From f2cf810e5f299ee3e1877af05cdbf4b94e10fc04 Mon Sep 17 00:00:00 2001 From: Sam Harwell Date: Fri, 1 Dec 2023 10:50:21 -0600 Subject: [PATCH] Run tests in parallel jobs --- .nuget/packages.config | 4 +- azure-pipelines.yml | 78 ++--------- build/build-and-test.yml | 280 +++++++++++++++++++++++++++++++++++++++ build/test.yml | 108 +++++++++++++++ 4 files changed, 398 insertions(+), 72 deletions(-) create mode 100644 build/build-and-test.yml create mode 100644 build/test.yml diff --git a/.nuget/packages.config b/.nuget/packages.config index f7010adb5..d409bf15b 100644 --- a/.nuget/packages.config +++ b/.nuget/packages.config @@ -1,7 +1,7 @@  - + - + \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bb3d4f15f..ef1f3f898 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,4 @@ -pool: +pool: name: Azure Pipelines vmImage: windows-latest demands: @@ -6,73 +6,11 @@ pool: - visualstudio - vstest -jobs: -- job: Build - variables: - BuildSolution: StyleCopAnalyzers.sln - BuildPlatform: Any CPU - strategy: - matrix: - Debug: - BuildConfiguration: Debug - _debugArg: '-Debug' - Release: - BuildConfiguration: Release - _debugArg: '' - steps: - - powershell: .\init.ps1 -NoRestore - displayName: Install .NET Core SDK +stages: +- template: build/build-and-test.yml + parameters: + BuildConfiguration: Debug - - task: NuGetToolInstaller@0 - displayName: 'Use NuGet 5.3.1' - inputs: - versionSpec: 5.3.1 - - - task: NuGetCommand@2 - displayName: 'NuGet restore' - inputs: - restoreSolution: '$(BuildSolution)' - feedsToUse: 'config' - nugetConfigPath: 'NuGet.config' - - - task: VSBuild@1 - displayName: 'Build solution StyleCopAnalyzers.sln' - inputs: - solution: '$(BuildSolution)' - platform: '$(BuildPlatform)' - configuration: '$(BuildConfiguration)' - msbuildArgs: '/bl:$(Build.SourcesDirectory)/msbuild.binlog' - - - task: PowerShell@2 - displayName: Run Tests - inputs: - workingDirectory: '$(Build.SourcesDirectory)/build' - filePath: build/opencover-report.ps1 - arguments: '$(_debugArg) -NoBuild -NoReport -Azure' - - - task: PublishTestResults@2 - displayName: Publish test results - condition: always() - inputs: - testResultsFormat: xUnit - testResultsFiles: 'build/*.xml' - mergeTestResults: true - testRunTitle: '$(BuildConfiguration)' - - - task: PowerShell@2 - displayName: Upload coverage reports to codecov.io - condition: eq(variables['BuildConfiguration'], 'Debug') - inputs: - workingDirectory: '$(Build.SourcesDirectory)/build' - targetType: inline - script: | - $packageConfig = [xml](Get-Content ..\.nuget\packages.config) - $codecov_version = $packageConfig.SelectSingleNode('/packages/package[@id="Codecov"]').version - $codecov = "..\packages\Codecov.$codecov_version\tools\codecov.exe" - &$codecov -f '..\build\OpenCover.Reports\OpenCover.StyleCopAnalyzers.xml' --required - - - task: PublishBuildArtifacts@1 - displayName: Publish build logs - inputs: - pathtoPublish: msbuild.binlog - condition: failed() +- template: build/build-and-test.yml + parameters: + BuildConfiguration: Release diff --git a/build/build-and-test.yml b/build/build-and-test.yml new file mode 100644 index 000000000..4638aeb99 --- /dev/null +++ b/build/build-and-test.yml @@ -0,0 +1,280 @@ +parameters: +- name: BuildConfiguration + displayName: Build Configuration + type: string + default: Debug + values: [ 'Debug', 'Release' ] +- name: BuildSolution + displayName: Solution + type: string + default: StyleCopAnalyzers.sln +- name: BuildPlatform + displayName: Platform + type: string + default: Any CPU + +stages: +- stage: Build_${{ parameters.BuildConfiguration }} + displayName: Build ${{ parameters.BuildConfiguration }} + dependsOn: [] + jobs: + - job: Build + steps: + - powershell: .\init.ps1 -NoRestore + displayName: Install .NET Core SDK + + - task: NuGetToolInstaller@0 + displayName: 'Use NuGet 5.3.1' + inputs: + versionSpec: 5.3.1 + + - task: NuGetCommand@2 + displayName: 'NuGet restore' + inputs: + restoreSolution: '${{ parameters.BuildSolution }}' + feedsToUse: 'config' + nugetConfigPath: 'NuGet.config' + + - task: VSBuild@1 + displayName: 'Build solution ${{ parameters.BuildSolution }}' + inputs: + solution: '${{ parameters.BuildSolution }}' + platform: '${{ parameters.BuildPlatform }}' + configuration: '${{ parameters.BuildConfiguration }}' + maximumCpuCount: false # AnnotatorBuildTask doesn't support parallel builds yet + msbuildArgs: '/v:minimal /bl:$(Build.SourcesDirectory)/msbuild.binlog' + +# - task: PowerShell@2 +# displayName: Upload coverage reports to codecov.io +# condition: eq(variables['BuildConfiguration'], 'Debug') +# inputs: +# workingDirectory: '$(Build.SourcesDirectory)/build' +# targetType: inline +# script: | +# $packageConfig = [xml](Get-Content ..\.nuget\packages.config) +# $codecov_version = $packageConfig.SelectSingleNode('/packages/package[@id="Codecov"]').version +# $codecov = "..\packages\Codecov.$codecov_version\tools\codecov.exe" +# &$codecov -f '..\build\OpenCover.Reports\OpenCover.StyleCopAnalyzers.xml' --required + + - task: PublishPipelineArtifact@1 + displayName: Publish build logs + inputs: + targetPath: msbuild.binlog + artifact: Build logs ${{ parameters.BuildConfiguration }} + condition: failed() + + - task: PublishPipelineArtifact@1 + displayName: Publish solution packages + inputs: + targetPath: $(Build.SourcesDirectory)/packages + artifact: slnPackages-${{ parameters.BuildConfiguration }} + + - task: PublishPipelineArtifact@1 + displayName: Publish build output (Test C# 6) + inputs: + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test/bin + artifact: buildTest-cs6-${{ parameters.BuildConfiguration }} + + - task: PublishPipelineArtifact@1 + displayName: Publish build output (Test C# 7) + inputs: + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp7/bin + artifact: buildTest-cs7-${{ parameters.BuildConfiguration }} + + - task: PublishPipelineArtifact@1 + displayName: Publish build output (Test C# 8) + inputs: + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp8/bin + artifact: buildTest-cs8-${{ parameters.BuildConfiguration }} + + - task: PublishPipelineArtifact@1 + displayName: Publish build output (Test C# 9) + inputs: + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp9/bin + artifact: buildTest-cs9-${{ parameters.BuildConfiguration }} + + - task: PublishPipelineArtifact@1 + displayName: Publish build output (Test C# 10) + inputs: + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp10/bin + artifact: buildTest-cs10-${{ parameters.BuildConfiguration }} + + - task: PublishPipelineArtifact@1 + displayName: Publish build output (Test C# 11) + inputs: + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp11/bin + artifact: buildTest-cs11-${{ parameters.BuildConfiguration }} + + - task: PublishPipelineArtifact@1 + displayName: Publish build output (Test C# 12) + inputs: + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp12/bin + artifact: buildTest-cs12-${{ parameters.BuildConfiguration }} + +- stage: Test_CSharp_6_${{ parameters.BuildConfiguration }} + displayName: Test C# 6 ${{ parameters.BuildConfiguration }} + dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ] + jobs: + - template: test.yml + parameters: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildSolution: ${{ parameters.BuildSolution }} + BuildPlatform: ${{ parameters.BuildPlatform }} + LangVersion: '6' + FrameworkVersion: 'net452' + +- stage: Test_CSharp_7_${{ parameters.BuildConfiguration }} + displayName: Test C# 7 ${{ parameters.BuildConfiguration }} + dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ] + jobs: + - template: test.yml + parameters: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildSolution: ${{ parameters.BuildSolution }} + BuildPlatform: ${{ parameters.BuildPlatform }} + LangVersion: '7' + FrameworkVersion: 'net46' + +- stage: Test_CSharp_8_${{ parameters.BuildConfiguration }} + displayName: Test C# 8 ${{ parameters.BuildConfiguration }} + dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ] + jobs: + - template: test.yml + parameters: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildSolution: ${{ parameters.BuildSolution }} + BuildPlatform: ${{ parameters.BuildPlatform }} + LangVersion: '8' + FrameworkVersion: 'net472' + +- stage: Test_CSharp_9_${{ parameters.BuildConfiguration }} + displayName: Test C# 9 ${{ parameters.BuildConfiguration }} + dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ] + jobs: + - template: test.yml + parameters: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildSolution: ${{ parameters.BuildSolution }} + BuildPlatform: ${{ parameters.BuildPlatform }} + LangVersion: '9' + FrameworkVersion: 'net472' + +- stage: Test_CSharp_10_${{ parameters.BuildConfiguration }} + displayName: Test C# 10 ${{ parameters.BuildConfiguration }} + dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ] + jobs: + - template: test.yml + parameters: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildSolution: ${{ parameters.BuildSolution }} + BuildPlatform: ${{ parameters.BuildPlatform }} + LangVersion: '10' + FrameworkVersion: 'net472' + +- stage: Test_CSharp_11_${{ parameters.BuildConfiguration }} + displayName: Test C# 11 ${{ parameters.BuildConfiguration }} + dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ] + jobs: + - template: test.yml + parameters: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildSolution: ${{ parameters.BuildSolution }} + BuildPlatform: ${{ parameters.BuildPlatform }} + LangVersion: '11' + FrameworkVersion: 'net472' + +- stage: Test_CSharp_12_${{ parameters.BuildConfiguration }} + displayName: Test C# 12 ${{ parameters.BuildConfiguration }} + dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ] + jobs: + - template: test.yml + parameters: + BuildConfiguration: ${{ parameters.BuildConfiguration }} + BuildSolution: ${{ parameters.BuildSolution }} + BuildPlatform: ${{ parameters.BuildPlatform }} + LangVersion: '12' + FrameworkVersion: 'net472' + +- stage: Publish_Code_Coverage_${{ parameters.BuildConfiguration }} + displayName: Publish Code Coverage + condition: eq('${{ parameters.BuildConfiguration }}', 'Debug') + dependsOn: + - Test_CSharp_6_${{ parameters.BuildConfiguration }} + - Test_CSharp_7_${{ parameters.BuildConfiguration }} + - Test_CSharp_8_${{ parameters.BuildConfiguration }} + - Test_CSharp_9_${{ parameters.BuildConfiguration }} + - Test_CSharp_10_${{ parameters.BuildConfiguration }} + - Test_CSharp_11_${{ parameters.BuildConfiguration }} + - Test_CSharp_12_${{ parameters.BuildConfiguration }} + jobs: + - job: WrapUp + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - task: DownloadPipelineArtifact@2 + displayName: 🔻 Download solution packages + continueOnError: true + inputs: + buildType: current + artifactName: slnPackages-${{ parameters.BuildConfiguration }} + targetPath: $(Build.SourcesDirectory)/packages + - download: current + artifact: coverageResults-cs6 + displayName: 🔻 Download C# 6 code coverage results + continueOnError: true + - download: current + artifact: coverageResults-cs7 + displayName: 🔻 Download C# 7 code coverage results + continueOnError: true + - download: current + artifact: coverageResults-cs8 + displayName: 🔻 Download C# 8 code coverage results + continueOnError: true + - download: current + artifact: coverageResults-cs9 + displayName: 🔻 Download C# 9 code coverage results + continueOnError: true + - download: current + artifact: coverageResults-cs10 + displayName: 🔻 Download C# 10 code coverage results + continueOnError: true + - download: current + artifact: coverageResults-cs11 + displayName: 🔻 Download C# 11 code coverage results + continueOnError: true + - download: current + artifact: coverageResults-cs12 + displayName: 🔻 Download C# 12 code coverage results + continueOnError: true + - task: PowerShell@2 + displayName: ⚙ Merge coverage + timeoutInMinutes: 20 + inputs: + workingDirectory: $(Build.SourcesDirectory) + targetType: inline + script: | + $packageConfig = [xml](Get-Content .\.nuget\packages.config) + $packages_folder = '.\packages' + $reportgenerator_version = $packageConfig.SelectSingleNode('/packages/package[@id="ReportGenerator"]').version + $report_generator = "$packages_folder\ReportGenerator.$reportgenerator_version\tools\net47\ReportGenerator.exe" + &$report_generator -targetdir:$(Pipeline.Workspace)/coverageResults-final -reporttypes:Cobertura "-reports:$(Pipeline.Workspace)/coverageResults-*/OpenCover.*.xml" + + - task: PowerShell@2 + displayName: Public code coverage to codecov.io + timeoutInMinutes: 20 + inputs: + workingDirectory: $(Build.SourcesDirectory) + targetType: inline + script: | + $packageConfig = [xml](Get-Content .\.nuget\packages.config) + $packages_folder = '.\packages' + $codecov_version = $packageConfig.SelectSingleNode('/packages/package[@id="CodecovUploader"]').version + $codecov = "$packages_folder\CodecovUploader.$codecov_version\tools\codecov.exe" + &$codecov -f '$(Pipeline.Workspace)/coverageResults-final/Cobertura.xml' --required + + - task: PublishPipelineArtifact@1 + displayName: Publish merged coverage + inputs: + targetPath: $(Pipeline.Workspace)/coverageResults-final + artifact: coverageResults-final$(System.JobAttempt)-${{ parameters.BuildConfiguration }} diff --git a/build/test.yml b/build/test.yml new file mode 100644 index 000000000..e1491acdf --- /dev/null +++ b/build/test.yml @@ -0,0 +1,108 @@ +parameters: +- name: BuildConfiguration + displayName: Build Configuration + type: string + default: Debug + values: [ 'Debug', 'Release' ] +- name: LangVersion + displayName: C# Language Version + type: string + default: '6' + values: [ '6', '7', '8', '9', '10', '11', '12' ] +- name: FrameworkVersion + displayName: .NET Framework Version + type: string + default: 'net472' + values: [ 'net452', 'net46', 'net472' ] +- name: BuildSolution + displayName: Solution + type: string + default: StyleCopAnalyzers.sln +- name: BuildPlatform + displayName: Platform + type: string + default: Any CPU + +jobs: +- job: Test_CSharp_${{ parameters.LangVersion }} + displayName: Test C# ${{ parameters.LangVersion }} + steps: + - powershell: .\init.ps1 -NoRestore + displayName: Install .NET Core SDK + + - task: NuGetToolInstaller@0 + displayName: 'Use NuGet 5.3.1' + inputs: + versionSpec: 5.3.1 + + - task: DownloadPipelineArtifact@2 + displayName: 🔻 Download solution packages + continueOnError: true + inputs: + buildType: current + artifactName: slnPackages-${{ parameters.BuildConfiguration }} + targetPath: $(Build.SourcesDirectory)/packages + + - ${{ if eq(parameters.LangVersion, '6') }}: + - task: DownloadPipelineArtifact@2 + displayName: 🔻 Download build output + continueOnError: true + inputs: + buildType: current + artifactName: buildTest-cs${{ parameters.LangVersion }}-${{ parameters.BuildConfiguration }} + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test/bin + + - ${{ if ne(parameters.LangVersion, '6') }}: + - task: DownloadPipelineArtifact@2 + displayName: 🔻 Download build output + continueOnError: true + inputs: + buildType: current + artifactName: buildTest-cs${{ parameters.LangVersion }}-${{ parameters.BuildConfiguration }} + targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp${{ parameters.LangVersion }}/bin + + - task: PowerShell@2 + displayName: 'Run tests' + timeoutInMinutes: 20 + inputs: + workingDirectory: '$(Build.SourcesDirectory)/build' + targetType: inline + script: | + $packageConfig = [xml](Get-Content ..\.nuget\packages.config) + $opencover_version = $packageConfig.SelectSingleNode('/packages/package[@id="OpenCover"]').version + $xunitrunner_version = $packageConfig.SelectSingleNode('/packages/package[@id="xunit.runner.console"]').version + + $packages_folder = '..\packages' + $opencover_console = "$packages_folder\OpenCover.$opencover_version\tools\OpenCover.Console.exe" + $xunit_runner_console_${{ parameters.FrameworkVersion }} = "$packages_folder\xunit.runner.console.$xunitrunner_version\tools\${{ parameters.FrameworkVersion }}\xunit.console.x86.exe" + $report_folder = '.\OpenCover.Reports' + mkdir $report_folder + $target_dll_name = If ('${{ parameters.LangVersion }}' -Eq '6') { "StyleCop.Analyzers.Test" } Else { "StyleCop.Analyzers.Test.CSharp${{ parameters.LangVersion }}" } + $target_dll_csharp${{ parameters.LangVersion }} = "..\StyleCop.Analyzers\$target_dll_name\bin\${{ parameters.BuildConfiguration }}\${{ parameters.FrameworkVersion }}\$target_dll_name.dll" + &$opencover_console ` + -register:Path32 ` + -threshold:1 -oldStyle ` + -returntargetcode ` + -hideskipped:All ` + -filter:"+[StyleCop*]*" ` + -excludebyattribute:*.ExcludeFromCodeCoverage* ` + -excludebyfile:*\*Designer.cs ` + -output:"$report_folder\OpenCover.StyleCopAnalyzers.CSharp${{ parameters.LangVersion }}.xml" ` + -target:"$xunit_runner_console_${{ parameters.FrameworkVersion }}" ` + -targetargs:"$target_dll_csharp${{ parameters.LangVersion }} -noshadow -xml StyleCopAnalyzers.CSharp${{ parameters.LangVersion }}.xunit.xml" + + - task: PublishTestResults@2 + displayName: 📢 Publish test results + condition: always() + inputs: + testResultsFormat: xUnit + testResultsFiles: 'build/*.xml' + mergeTestResults: true + testRunTitle: 'C# ${{ parameters.LangVersion }} ${{ parameters.BuildConfiguration }}' + + - ${{ if eq(parameters.BuildConfiguration, 'Debug') }}: + - task: PublishPipelineArtifact@1 + displayName: Publish code coverage + inputs: + targetPath: $(Build.SourcesDirectory)/build/OpenCover.Reports + artifact: coverageResults-cs${{ parameters.LangVersion }}