From 0364ee2da2dc6da2e73f10752560f0725b3997d4 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 21 Jan 2025 15:22:19 -0800 Subject: [PATCH 01/36] Remove prepare-signed-artifacts and publish from each leg directly. --- Build.proj | 2 +- eng/Publishing.props | 143 +++--------- eng/Signing.props | 17 +- eng/pipelines/common/global-build-job.yml | 16 +- .../common/templates/global-build-step.yml | 2 +- .../jobs/prepare-signed-artifacts.yml | 66 ------ eng/pipelines/official/stages/publish.yml | 57 ----- eng/pipelines/runtime-official.yml | 205 +++++------------- 8 files changed, 103 insertions(+), 405 deletions(-) delete mode 100644 eng/pipelines/official/jobs/prepare-signed-artifacts.yml delete mode 100644 eng/pipelines/official/stages/publish.yml diff --git a/Build.proj b/Build.proj index 2f687610b4a93a..0957ae6846e573 100644 --- a/Build.proj +++ b/Build.proj @@ -1,6 +1,6 @@ - + diff --git a/eng/Publishing.props b/eng/Publishing.props index da3d606ed683c4..abed2e967c1b85 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -30,124 +30,53 @@ IsShipping="false" /> - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - + $(ArtifactsPackagesDir)**\runtime.native.*.nupkg + $(ArtifactsPackagesDir)**\*.Transport.*.nupkg" /> + - - - - - + + - - - - - - - - - - - + TODO: Some packages aren't produced in the windows_x64 jobs. In particular, packages matching the following patterns: + - Microsoft.NET.Workload.Mono.Toolchain.*Manifest-* + - Microsoft.NET.Sdk.WebAssembly.Pack.* + + We need to determine where to ship these from and how to specify it in the official build. + + Additionally, some packages matching the above RID-specific patterns aren't actually RID-specific and need to be deduplicated: + - Microsoft.NET.Runtime.*.Sample.Mono + - Microsoft.NET.Runtime.LibraryBuilder.Sdk + - Microsoft.NET.Runtime.MonoAOTCompiler.Task + - Microsoft.NET.Runtime.MonoTargets.Sdk + - Microsoft.NET.Runtime.WebAssembly.Sdk + - Microsoft.NET.Runtime.WebAssembly.Templates.net10 + - Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk + - Microsoft.NET.Runtime.WorkloadTesting.Internal + --> + + + + + diff --git a/eng/Signing.props b/eng/Signing.props index c1ff2d8ba476b0..07761eda85163e 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -1,8 +1,4 @@ - - false - - @@ -45,8 +41,7 @@ - - + - - - \ No newline at end of file diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 6bdd96038acf75..e6fc0db08d3908 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -33,6 +33,7 @@ parameters: postBuildSteps: [] extraVariablesTemplates: [] preBuildSteps: [] + PublishRidAgnosticPackagesFromPlatform: windows_x64 templatePath: 'templates' templateContext: '' disableComponentGovernance: '' @@ -52,6 +53,8 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} enablePublishTestResults: ${{ parameters.enablePublishTestResults }} testResultsFormat: ${{ parameters.testResultsFormat }} + enableMicrobuild: ${{ parameters.isOfficialBuild }} + enableMicrobuildForMacAndLinux: ${{ parameters.isOfficialBuild }} ${{ if ne(parameters.templateContext, '') }}: templateContext: ${{ parameters.templateContext }} @@ -123,7 +126,7 @@ jobs: - name: _officialBuildParameter ${{ if eq(parameters.isOfficialBuild, true) }}: - value: /p:OfficialBuildId=$(Build.BuildNumber) + value: /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetPublishUsingPipelines=true ${{ if ne(parameters.isOfficialBuild, true) }}: value: '' @@ -132,6 +135,17 @@ jobs: value: /p:BuildDarwinFrameworks=true ${{ if notin(parameters.osGroup, 'ios', 'tvos', 'maccatalyst')}}: value: '' + + - name: _assetManifestFileNameParameter + ${{ if eq(parameters.isOfficialBuild, true) }}: + value: /p:AssetManifestFileName="${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{parameters.archType}}_${{parameters.nameSuffix }}" + ${{ if ne(parameters.isOfficialBuild, true) }}: + value: '' + - name: _publishRidAgnosticArtifactsParameter + ${{ if and(eq(parameters.isOfficialBuild, true), eq(format('{0}{1}_{2}', parameters.osGroup, parameters.osSubGroup, parameters.archType), parameters.PublishRidAgnosticPackagesFromPlatform)) }}: + value: /p:PublishRidAgnosticPackages=true + ${{ if ne(parameters.isOfficialBuild, true) }}: + value: '' # Set no native sanitizers by default - name: _nativeSanitizersArg diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 2a2262d9a4ead4..75cfd580952add 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -10,7 +10,7 @@ parameters: condition: succeeded() steps: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) $(_assetManifestFileNameParameter) $(_publishRidAgnosticArtifactsParameter) displayName: ${{ parameters.displayName }} ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml deleted file mode 100644 index 1440a7a1dfdc35..00000000000000 --- a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml +++ /dev/null @@ -1,66 +0,0 @@ -parameters: - PublishRidAgnosticPackagesFromPlatform: '' - isOfficialBuild: false - logArtifactName: 'Logs-PrepareSignedArtifacts_Attempt$(System.JobAttempt)' - -jobs: -- template: /eng/common/templates-official/job/job.yml - parameters: - name: 'PrepareSignedArtifacts' - displayName: 'Prepare Signed Artifacts' - - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - - # Double the default timeout. - timeoutInMinutes: 240 - - workspace: - clean: all - - enableMicrobuild: true - - variables: - - name: '_SignType' - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - templateContext: - inputs: - - input: checkout - repository: self - clean: true - fetchDepth: 20 - - input: pipelineArtifact - artifactName: IntermediateArtifacts - targetPath: $(Build.SourcesDirectory)\artifacts\PackageDownload\IntermediateArtifacts - outputs: - - output: pipelineArtifact - displayName: 'Publish BuildLogs' - condition: succeededOrFailed() - targetPath: '$(Build.StagingDirectory)\BuildLogs' - artifactName: ${{ parameters.logArtifactName }} - - steps: - - script: >- - build.cmd -restore -sign -publish -ci -configuration Release - /p:RestoreToolsetOnly=true - /p:PublishRidAgnosticPackagesFromPlatform=${{ parameters.PublishRidAgnosticPackagesFromPlatform }} - /p:DownloadDirectory=$(Build.SourcesDirectory)\artifacts\PackageDownload\ - /p:OfficialBuildId=$(Build.BuildNumber) - /p:SignType=$(_SignType) - /p:DotNetSignType=$(_SignType) - /p:DotNetPublishUsingPipelines=true - /bl:$(Build.SourcesDirectory)\prepare-artifacts.binlog - displayName: Prepare artifacts and upload to build - - - task: CopyFiles@2 - displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)\BuildLogs' - continueOnError: true - condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/pipelines/official/stages/publish.yml b/eng/pipelines/official/stages/publish.yml deleted file mode 100644 index 83b059ffc32a65..00000000000000 --- a/eng/pipelines/official/stages/publish.yml +++ /dev/null @@ -1,57 +0,0 @@ -parameters: - PublishRidAgnosticPackagesFromPlatform: windows_x64 - -stages: - -- stage: PrepareForPublish - displayName: Prepare for Publish - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing. - - template: /eng/pipelines/official/jobs/prepare-signed-artifacts.yml - parameters: - PublishRidAgnosticPackagesFromPlatform: ${{ parameters.PublishRidAgnosticPackagesFromPlatform }} - - # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. - - template: /eng/common/templates-official/job/publish-build-assets.yml - parameters: - publishUsingPipelines: true - publishAssetsImmediately: true - dependsOn: PrepareSignedArtifacts - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-windows-2022 - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' - -# Stages-based publishing entry point -- template: /eng/common/templates-official/post-build/post-build.yml - parameters: - validateDependsOn: - - PrepareForPublish - # The following checks are run after the build in the validation and release pipelines - # And thus are not enabled here. They can be enabled for dev builds for spot testing if desired - enableSymbolValidation: false - enableSigningValidation: false - enableNugetValidation: false - enableSourceLinkValidation: false - publishAssetsImmediately: true - SDLValidationParameters: - enable: false - artifactNames: - - PackageArtifacts - - BlobArtifacts - params: >- - -SourceToolsList @("policheck","credscan") - -TsaInstanceURL "$(TsaInstanceURL)" - -TsaProjectName "$(TsaProjectName)" - -TsaNotificationEmail "$(TsaNotificationEmail)" - -TsaCodebaseAdmin "$(TsaCodebaseAdmin)" - -TsaBugAreaPath "$(TsaBugAreaPath)" - -TsaIterationPath "$(TsaIterationPath)" - -TsaRepositoryName "$(TsaRepositoryName)" - -TsaCodebaseName "$(TsaCodebaseName)" - -TsaPublish $True - symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' - # Publish to blob storage. - publishInstallersAndChecksums: true diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 4604248840acb0..ac8792df4e0200 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -85,14 +85,9 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs - # Upload the results. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) - # # Build all runtime packs # Mac x64/arm64 @@ -136,14 +131,9 @@ extends: # Now that we've entitled and signed createdump, we can build the rest. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs - # Upload the results. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) - # # Build all runtime packs for Linux and Linux musl # Upload CoreCLR runtime for CrossDac packaging @@ -161,7 +151,7 @@ extends: - linux_musl_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -179,11 +169,6 @@ extends: PathtoPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts PublishLocation: Container ArtifactName: CoreCLRCrossDacArtifacts - - # Upload the results. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) extraVariablesTemplates: - template: /eng/pipelines/coreclr/templates/crossdac-hostarch.yml @@ -198,7 +183,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -sign -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -226,12 +211,6 @@ extends: basePath: $(crossDacArtifactsPath) isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 30 - postBuildSteps: - # Save packages using the prepare-signed-artifacts format. - # CrossDac packages are expected to be in the windows_x64 folder. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: windows_x64 dependsOn: - build_linux_x64_release_AllRuntimes - build_linux_arm_release_AllRuntimes @@ -272,22 +251,9 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: MobileRuntimePacks - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -298,21 +264,9 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: MobileRuntimePacks - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -323,22 +277,10 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: MobileRuntimePacks # # Build Mono LLVM runtime packs @@ -354,23 +296,11 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - # delete duplicate RIDless packages to prevent upload conflict - - task: DeleteFiles@1 - displayName: 'Delete Microsoft.NETCore.App.Ref and Microsoft.NETCore.App.HostModel package' - inputs: - SourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)/Shipping - Contents: | - 'Microsoft.NETCore.App.Ref.*.nupkg' - 'Microsoft.NET.HostModel.*.nupkg' - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: $(osGroup)$(osSubgroup)_$(archType) # # Build libraries (all TFMs) and packages @@ -383,13 +313,9 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -pack -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} - postBuildSteps: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: Libraries_WithPackages timeoutInMinutes: 95 # # Build SourceBuild packages @@ -417,13 +343,9 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO - postBuildSteps: - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: PGO timeoutInMinutes: 95 # @@ -441,39 +363,39 @@ extends: preBuildSteps: - task: DownloadPipelineArtifact@2 inputs: - artifact: 'IntermediateArtifacts' + artifact: 'PackageArtifacts' path: $(Build.SourcesDirectory)/artifacts/workloadPackages patterns: | - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg - IntermediateArtifacts/windows_arm64/Shipping/Microsoft.NETCore.App.Runtime.win-arm64*.nupkg - IntermediateArtifacts/windows_x64/Shipping/Microsoft.NETCore.App.Runtime.win-x64*.nupkg - IntermediateArtifacts/windows_x86/Shipping/Microsoft.NETCore.App.Runtime.win-x86*.nupkg - IntermediateArtifacts/MobileRuntimePacks/Shipping/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.android-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.android-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross.wasi-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.AOT.win-arm64.Cross.wasi-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.android-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.ios-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.iossimulator-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.tvos-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.tvossimulator-*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.Mono.wasi-wasm*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.Current.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net6.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net7.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net8.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Workload.Mono.ToolChain.net9.Manifest*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.MonoTargets.Sdk*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.MonoAOTCompiler.Task*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Sdk*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Wasi*.nupkg + PackageArtifacts/Microsoft.NET.Runtime.WebAssembly.Templates*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.win-arm64*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x64*.nupkg + PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg + PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - task: CopyFiles@2 displayName: Flatten packages @@ -484,40 +406,7 @@ extends: targetFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages flattenFolders: true - buildArgs: -s mono.workloads -c $(_BuildConfig) /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads - - postBuildSteps: - # Prepare packages wrapping msis - - task: CopyFiles@2 - displayName: Prepare package artifacts - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/packages/$(_BuildConfig)' - Contents: | - Shipping/**/* - NonShipping/**/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts1/workloads' - CleanTargetFolder: true - - # Prepare artifacts to be used for generating VS components - - task: CopyFiles@2 - displayName: Prepare VS Insertion artifacts - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/VSSetup/$(_BuildConfig)' - Contents: | - Insertion/**/* - TargetFolder: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts2/workloads-vs' - CleanTargetFolder: true - - templateContext: - outputs: - - output: buildArtifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts1' - ArtifactName: IntermediateArtifacts - displayName: 'Publish workload packages' - - output: buildArtifacts - PathtoPublish: '$(Build.ArtifactStagingDirectory)/IntermediateArtifacts2' - ArtifactName: IntermediateArtifacts - displayName: 'Publish workload VS Insertion artifacts' + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -540,7 +429,11 @@ extends: - Build_windows_x86_release_AllRuntimes - Build_windows_arm64_release_AllRuntimes - - ${{ if eq(variables.isOfficialBuild, true) }}: - - template: /eng/pipelines/official/stages/publish.yml + - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + publishUsingPipelines: true + publishAssetsImmediately: true + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' \ No newline at end of file From 7e284a45dcd94be860b8718617796d55b674e634 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 21 Jan 2025 15:27:59 -0800 Subject: [PATCH 02/36] Publish in a separate stage to make dependency tracking easy --- eng/pipelines/runtime-official.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index ac8792df4e0200..068fe007996f59 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -429,6 +429,8 @@ extends: - Build_windows_x86_release_AllRuntimes - Build_windows_arm64_release_AllRuntimes + - stage: Publish + jobs: - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: publishUsingPipelines: true From d2c9cf864e1dcf5e5b6e66b0011ef26c6784a93d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 21 Jan 2025 16:48:03 -0800 Subject: [PATCH 03/36] Set _SignType so we can pass -sign --- eng/pipelines/common/global-build-job.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index e6fc0db08d3908..d9211734804ae3 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -93,6 +93,9 @@ jobs: - name: _archParameter value: -arch ${{ parameters.archType }} + - name: _SignType + value: $[ coalesce(variables.OfficialSignType, 'real') ] + - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubGroup, '_bionic')) }}: - name: _osParameter value: -os linux-bionic @@ -126,7 +129,7 @@ jobs: - name: _officialBuildParameter ${{ if eq(parameters.isOfficialBuild, true) }}: - value: /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetPublishUsingPipelines=true + value: /p:OfficialBuildId=$(Build.BuildNumber) /p:DotNetPublishUsingPipelines=true /p:SignType=$(_SignType) /p:DotNetSignType=$(_SignType) ${{ if ne(parameters.isOfficialBuild, true) }}: value: '' From 3338d4d50088ae69c6146dcc8bcd0f135a7c5ba5 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 13:34:31 -0800 Subject: [PATCH 04/36] Remove old validation target --- eng/Publishing.props | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index abed2e967c1b85..b5a2c8e336b6ab 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -140,18 +140,4 @@ RelativeBlobPath="Runtime/$(NonStableProductVersion)/%(Artifact.SubBlobFolder)%(Filename)%(Extension)" /> - - - - - - - - - From 01bca6023d4144f37b0067410fc643594a698593 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 13:35:19 -0800 Subject: [PATCH 05/36] !drop remove -sign so we can figure out publishing first --- eng/pipelines/runtime-official.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 068fe007996f59..d0c3e58a21be62 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -85,7 +85,7 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs # @@ -131,7 +131,7 @@ extends: # Now that we've entitled and signed createdump, we can build the rest. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs # @@ -151,7 +151,7 @@ extends: - linux_musl_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -183,7 +183,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -sign -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -251,7 +251,7 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} - template: /eng/pipelines/common/platform-matrix.yml @@ -264,7 +264,7 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -277,7 +277,7 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread @@ -296,7 +296,7 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT @@ -313,7 +313,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -343,7 +343,7 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO timeoutInMinutes: 95 @@ -406,7 +406,7 @@ extends: targetFolder: $(Build.SourcesDirectory)/artifacts/workloadPackages flattenFolders: true - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 88ee36c4fad2e40545ad3e2e4ccfedf453b3ee07 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 14:04:30 -0800 Subject: [PATCH 06/36] Remove InitialTargets --- eng/Publishing.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index b5a2c8e336b6ab..b2d05d00626342 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -1,4 +1,4 @@ - + true From d2fa2444b6e289c6c6e9eef549c46a4c4d18554f Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 22 Jan 2025 15:42:56 -0800 Subject: [PATCH 07/36] Only generate product version files once. Fix asset manifest extension. Don't copy workloads packages as they're already in the right format now. --- eng/Publishing.props | 5 ++--- eng/pipelines/common/global-build-job.yml | 2 +- eng/pipelines/runtime-official.yml | 11 +---------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index b2d05d00626342..ec84d512e06d26 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -98,14 +98,13 @@ but we do produce them in both the VMR and the runtime official build. --> - <_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' == 'true' and '$(DotNetBuildOrchestrator)' == 'true'">true - <_ShouldGenerateProductVersionFiles Condition="'$(DotNetBuildRepo)' != 'true' and '$(DotNetBuildOrchestrator)' != 'true'">true + true + Condition="'$(ShouldGenerateProductVersionFiles)' == 'true'"> Date: Wed, 22 Jan 2025 16:34:13 -0800 Subject: [PATCH 08/36] Completely restructure how we publish to be more explicit that specific jobs publish specific packages --- Directory.Build.props | 99 +---------------- eng/Publishing.props | 68 ++++++------ eng/RuntimeIdentifier.props | 100 ++++++++++++++++++ .../common/templates/packages-to-publish.yml | 35 ++++++ eng/pipelines/runtime-official.yml | 19 ++-- 5 files changed, 181 insertions(+), 140 deletions(-) create mode 100644 eng/RuntimeIdentifier.props create mode 100644 eng/pipelines/common/templates/packages-to-publish.yml diff --git a/Directory.Build.props b/Directory.Build.props index 79ec992824c389..052f4e532e3a12 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -201,104 +201,7 @@ $(MonoCrossAOTTargetOS)+tvos+ios+maccatalyst - - false - true - - - - - - - <_portableOS>$(TargetOS.ToLowerInvariant()) - <_portableOS Condition="'$(_portableOS)' == 'windows'">win - - - <_portableOS Condition="'$(_portableOS)' == 'anyos'">$(__PortableTargetOS) - - - <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-musl'">linux-musl - <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-bionic'">linux-bionic - - - <_portableOS Condition="'$(HostOS)' == 'win' and '$(TargetsMobile)' != 'true'">win - - - - - <_packageOS>$(_portableOS) - - <_packageOS Condition="'$(CrossBuild)' == 'true' and '$(_portableOS)' != 'linux-musl' and '$(_portableOS)' != 'linux-bionic'">$(_hostOS) - - - $(PackageOS)-$(TargetArchitecture) - $(_packageOS)-$(TargetArchitecture) - - - - - - <_portableHostOS>$(_hostOS) - <_portableHostOS Condition="'$(_portableHostOS)' == 'windows'">win - <_portableHostOS Condition="'$(CrossBuild)' != 'true' and '$(_portableOS)' == 'linux-musl'">linux-musl - - - $(ToolsOS)-$(_hostArch) - $(_portableHostOS)-$(_hostArch) - - - $(ToolsRID) - - - - - <_hostRid Condition="'$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) - <_hostRid Condition="'$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant) - - <_parseDistroRid>$(__DistroRid) - <_parseDistroRid Condition="'$(_parseDistroRid)' == ''">$(_hostRid) - <_distroRidIndex>$(_parseDistroRid.LastIndexOf('-')) - - <_outputOS>$(_parseDistroRid.SubString(0, $(_distroRidIndex))) - <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_portableOS) - - $(_outputOS)-$(TargetArchitecture) - - - - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - true - + $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'microsoft.netcore.app.ref')) diff --git a/eng/Publishing.props b/eng/Publishing.props index ec84d512e06d26..1f4caa1eaca12f 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -8,6 +8,8 @@ $(TargetArchitecture) + + @@ -30,50 +32,44 @@ IsShipping="false" /> - + - - - + + + + + + + - - + + + - + + + + + + + + + diff --git a/eng/RuntimeIdentifier.props b/eng/RuntimeIdentifier.props new file mode 100644 index 00000000000000..1090d58b5c6187 --- /dev/null +++ b/eng/RuntimeIdentifier.props @@ -0,0 +1,100 @@ + + + false + true + + + + + + + <_portableOS>$(TargetOS.ToLowerInvariant()) + <_portableOS Condition="'$(_portableOS)' == 'windows'">win + + + <_portableOS Condition="'$(_portableOS)' == 'anyos'">$(__PortableTargetOS) + + + <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-musl'">linux-musl + <_portableOS Condition="'$(_portableOS)' == 'linux' and '$(__PortableTargetOS)' == 'linux-bionic'">linux-bionic + + + <_portableOS Condition="'$(HostOS)' == 'win' and '$(TargetsMobile)' != 'true'">win + + + + + <_packageOS>$(_portableOS) + + <_packageOS Condition="'$(CrossBuild)' == 'true' and '$(_portableOS)' != 'linux-musl' and '$(_portableOS)' != 'linux-bionic'">$(_hostOS) + + + $(PackageOS)-$(TargetArchitecture) + $(_packageOS)-$(TargetArchitecture) + + + + + + <_portableHostOS>$(_hostOS) + <_portableHostOS Condition="'$(_portableHostOS)' == 'windows'">win + <_portableHostOS Condition="'$(CrossBuild)' != 'true' and '$(_portableOS)' == 'linux-musl'">linux-musl + + + $(ToolsOS)-$(_hostArch) + $(_portableHostOS)-$(_hostArch) + + + $(ToolsRID) + + + + + <_hostRid Condition="'$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) + <_hostRid Condition="'$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant) + + <_parseDistroRid>$(__DistroRid) + <_parseDistroRid Condition="'$(_parseDistroRid)' == ''">$(_hostRid) + <_distroRidIndex>$(_parseDistroRid.LastIndexOf('-')) + + <_outputOS>$(_parseDistroRid.SubString(0, $(_distroRidIndex))) + <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_portableOS) + + $(_outputOS)-$(TargetArchitecture) + + + + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + true + + \ No newline at end of file diff --git a/eng/pipelines/common/templates/packages-to-publish.yml b/eng/pipelines/common/templates/packages-to-publish.yml new file mode 100644 index 00000000000000..6bcc28265f85e9 --- /dev/null +++ b/eng/pipelines/common/templates/packages-to-publish.yml @@ -0,0 +1,35 @@ +parameters: + # Parameters forwarded from global-build-job.yml + targetRid: '' + # Default artifact sets for specific RIDs + ios_arm64AdditionalPackages: + - Microsoft.NET.Runtime.iOS.Sample.Mono + - Microsoft.NET.Runtime.LibraryBuilder.Sdk + - Microsoft.NET.Runtime.MonoAOTCompiler.Task + - Microsoft.NET.Runtime.MonoTargets.Sdk + android_arm64AdditionalPackages: + - Microsoft.NET.Runtime.Android.Sample.Mono + wasi_wasmAdditionalPackages: + - Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk + browser_wasmAdditionalPackages: + - Microsoft.NET.Runtime.WebAssembly.Sdk + - Microsoft.NET.Runtime.WebAssembly.Templates.net10 + - Microsoft.NET.Sdk.WebAssembly.Pack + - Microsoft.NET.Workload.Mono.ToolChain + +# TODO-Publish: Move this into Publishing.props except for the win-x64 case. + +variables: + - name: _PublishAdditionalPackagesArg + ${{ if eq(parameters.targetRid, 'ios-arm64') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.ios_arm64AdditionalPackages) }} + ${{ elif eq(parameters.targetRid, 'android-arm64') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.android_arm64AdditionalPackages) }} + ${{ elif eq(parameters.targetRid, 'wasi-wasm') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.wasi_wasmAdditionalPackages) }} + ${{ elif eq(parameters.targetRid, 'browser-wasm') }}: + value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.browser_wasmAdditionalPackages) }} + ${{ if eq(parameters.targetRid, 'windows_x64') }}: + value: /p:PublishAllPackages=true + ${{ else }}: + value: '' diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 6c2c9da1cea6fe..d9bf067231fe0a 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -85,8 +85,10 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/packages-to-publish.yml # # Build all runtime packs @@ -183,7 +185,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -251,9 +253,12 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/packages-to-publish.yml + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -264,9 +269,11 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) $(_PublishAdditionalPackagesArg) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/packages-to-publish.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -313,7 +320,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -397,7 +404,7 @@ extends: PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 8ee9ac1b5a750f4e9e8195376ecd7d9d49d5e343 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 23 Jan 2025 11:51:05 -0800 Subject: [PATCH 09/36] Move "non-RID-specific" special packages logic into Publishing.props --- eng/Publishing.props | 33 +++++++++++++---- .../common/templates/packages-to-publish.yml | 35 ------------------- eng/pipelines/runtime-official.yml | 11 +++--- 3 files changed, 31 insertions(+), 48 deletions(-) delete mode 100644 eng/pipelines/common/templates/packages-to-publish.yml diff --git a/eng/Publishing.props b/eng/Publishing.props index 1f4caa1eaca12f..1579fa98616c43 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -38,8 +38,7 @@ For non-RID-specific packages, we have various rules: - A job can specify PublishAllPackages=true to publish all packages it produces. We have specific jobs that produce RID-agnostic packages or packages for multiple RIDs set this property. - - A job can specify PublishAdditionalSpecificPackages="Package1;Package2" to publish the specified packages. We use this for specific jobs were we produce the same package in multiple jobs, - but we don't produce the package in any jobs where we set PublishAllPackages. + - For some target RIDs, we also include specific RID-agnostic packages. --> @@ -49,6 +48,32 @@ + + + + + + + + + + + + + + + + diff --git a/eng/pipelines/common/templates/packages-to-publish.yml b/eng/pipelines/common/templates/packages-to-publish.yml deleted file mode 100644 index 6bcc28265f85e9..00000000000000 --- a/eng/pipelines/common/templates/packages-to-publish.yml +++ /dev/null @@ -1,35 +0,0 @@ -parameters: - # Parameters forwarded from global-build-job.yml - targetRid: '' - # Default artifact sets for specific RIDs - ios_arm64AdditionalPackages: - - Microsoft.NET.Runtime.iOS.Sample.Mono - - Microsoft.NET.Runtime.LibraryBuilder.Sdk - - Microsoft.NET.Runtime.MonoAOTCompiler.Task - - Microsoft.NET.Runtime.MonoTargets.Sdk - android_arm64AdditionalPackages: - - Microsoft.NET.Runtime.Android.Sample.Mono - wasi_wasmAdditionalPackages: - - Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk - browser_wasmAdditionalPackages: - - Microsoft.NET.Runtime.WebAssembly.Sdk - - Microsoft.NET.Runtime.WebAssembly.Templates.net10 - - Microsoft.NET.Sdk.WebAssembly.Pack - - Microsoft.NET.Workload.Mono.ToolChain - -# TODO-Publish: Move this into Publishing.props except for the win-x64 case. - -variables: - - name: _PublishAdditionalPackagesArg - ${{ if eq(parameters.targetRid, 'ios-arm64') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.ios_arm64AdditionalPackages) }} - ${{ elif eq(parameters.targetRid, 'android-arm64') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.android_arm64AdditionalPackages) }} - ${{ elif eq(parameters.targetRid, 'wasi-wasm') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.wasi_wasmAdditionalPackages) }} - ${{ elif eq(parameters.targetRid, 'browser-wasm') }}: - value: /p:PublishAdditionalSpecificPackages=${{ join(',', parameters.browser_wasmAdditionalPackages) }} - ${{ if eq(parameters.targetRid, 'windows_x64') }}: - value: /p:PublishAllPackages=true - ${{ else }}: - value: '' diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index d9bf067231fe0a..1ee6513c279e99 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -70,6 +70,9 @@ extends: - windows_x64 - windows_x86 - windows_arm64 + variables: + - name: PublishAllPackages + value: $[eq(variables.targetRid, 'win-x64')] jobParameters: templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog @@ -85,10 +88,8 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true /p:PublishAllPackages=$(PublishAllPackages) displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/packages-to-publish.yml # # Build all runtime packs @@ -256,8 +257,6 @@ extends: buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/packages-to-publish.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -272,8 +271,6 @@ extends: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) $(_PublishAdditionalPackagesArg) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/packages-to-publish.yml - template: /eng/pipelines/common/platform-matrix.yml parameters: From 44553015230b1fcc2c3e8d33725f89c145d215fe Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 23 Jan 2025 13:24:39 -0800 Subject: [PATCH 10/36] Make sure we build the list of skipped packages after we add the RID-specific packages --- eng/Publishing.props | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 1579fa98616c43..f7a867f7169083 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -43,11 +43,9 @@ - + - - @@ -74,6 +72,7 @@ + false - - $(TargetArchitecture) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 0dfa1809a53486..6cbc0c09eeda37 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -33,7 +33,6 @@ parameters: postBuildSteps: [] extraVariablesTemplates: [] preBuildSteps: [] - PublishRidAgnosticPackagesFromPlatform: windows_x64 templatePath: 'templates' templateContext: '' disableComponentGovernance: '' @@ -92,6 +91,9 @@ jobs: value: -os ${{ parameters.osGroup }} - name: _archParameter value: -arch ${{ parameters.archType }} + + - name: _AssetManifestName + value: ${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.nameSuffix }} - name: _SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] @@ -138,17 +140,6 @@ jobs: value: /p:BuildDarwinFrameworks=true ${{ if notin(parameters.osGroup, 'ios', 'tvos', 'maccatalyst')}}: value: '' - - - name: _assetManifestFileNameParameter - ${{ if eq(parameters.isOfficialBuild, true) }}: - value: /p:AssetManifestFileName="${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{parameters.archType}}_${{parameters.nameSuffix }}.xml" - ${{ if ne(parameters.isOfficialBuild, true) }}: - value: '' - - name: _publishRidAgnosticArtifactsParameter - ${{ if and(eq(parameters.isOfficialBuild, true), eq(format('{0}{1}_{2}', parameters.osGroup, parameters.osSubGroup, parameters.archType), parameters.PublishRidAgnosticPackagesFromPlatform)) }}: - value: /p:PublishRidAgnosticPackages=true - ${{ if ne(parameters.isOfficialBuild, true) }}: - value: '' # Set no native sanitizers by default - name: _nativeSanitizersArg diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml index 75cfd580952add..2a2262d9a4ead4 100644 --- a/eng/pipelines/common/templates/global-build-step.yml +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -10,7 +10,7 @@ parameters: condition: succeeded() steps: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) $(_assetManifestFileNameParameter) $(_publishRidAgnosticArtifactsParameter) + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) ${{ parameters.crossArg }} ${{ parameters.buildArgs }} ${{ parameters.targetCxxLibraryConfigurationArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) displayName: ${{ parameters.displayName }} ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 1ee6513c279e99..b655fb5da8c373 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -281,7 +281,7 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) /p:AssetManifestFileName=browser_wasm_Multithreaded.xml nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread @@ -300,8 +300,8 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish - /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AssetManifestFileName=$(_AssetManifestName).xml + /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -317,7 +317,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true + buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true /p:AssetManifestFileName=LibrariesPackages.xml nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -347,7 +347,7 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true /p:AssetManifestFileName=$(_AssetManifestName).xml isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO timeoutInMinutes: 95 @@ -401,7 +401,7 @@ extends: PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true /p:AssetManifestFileName=Workloads.xml isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 747bffebbf21190199e9e455a77f8e2d22cfc29d Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 23 Jan 2025 16:32:27 -0800 Subject: [PATCH 12/36] Extract handling hostOS and hostArch into another props file --- Directory.Build.props | 35 ++--------------------------------- eng/OSArch.props | 35 +++++++++++++++++++++++++++++++++++ eng/Publishing.props | 1 + 3 files changed, 38 insertions(+), 33 deletions(-) create mode 100644 eng/OSArch.props diff --git a/Directory.Build.props b/Directory.Build.props index 052f4e532e3a12..1c2c021f5d7633 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -17,39 +17,8 @@ false - - <_hostOS>linux - <_hostOS Condition="$([MSBuild]::IsOSPlatform('OSX'))">osx - <_hostOS Condition="$([MSBuild]::IsOSPlatform('FREEBSD'))">freebsd - <_hostOS Condition="$([MSBuild]::IsOSPlatform('NETBSD'))">netbsd - <_hostOS Condition="$([MSBuild]::IsOSPlatform('ILLUMOS'))">illumos - <_hostOS Condition="$([MSBuild]::IsOSPlatform('SOLARIS'))">solaris - <_hostOS Condition="$([MSBuild]::IsOSPlatform('HAIKU'))">haiku - <_hostOS Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">windows - $(_hostOS) - browser - $(_hostOS) - true - true - - - - - <_hostArch>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) - $(_hostArch) - wasm - wasm - arm - armv6 - armel - arm64 - loongarch64 - s390x - ppc64le - x64 - x64 - $(TargetArchitecture) - + + + + <_hostArch>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) + $(_hostArch) + wasm + wasm + arm + armv6 + armel + arm64 + loongarch64 + s390x + ppc64le + x64 + x64 + $(TargetArchitecture) + + \ No newline at end of file diff --git a/eng/Publishing.props b/eng/Publishing.props index 3d3fd70e8ff2e5..2ca4d5db89a6ff 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -6,6 +6,7 @@ false + - + @@ -44,7 +46,7 @@ - + From 6bbbc4cbf39a79ef4c9a8e89186c53f7c9287436 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 24 Jan 2025 13:43:29 -0800 Subject: [PATCH 14/36] Fix asset manifest name uniqueness to ensue we don't overwrite asset manifests. --- eng/Publishing.props | 7 ++++--- eng/pipelines/runtime-official.yml | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 917bd4fd2d6d09..0347fea9c02864 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -1,14 +1,15 @@ + + true false + + $(PackageRID).xml - - - diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index b655fb5da8c373..35ceca9ad22d9b 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -186,7 +186,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true /p:AssetManifestFileName=CrossDac.xml nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -254,7 +254,7 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true $(_PublishAdditionalPackagesArg) + buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -268,7 +268,7 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) $(_PublishAdditionalPackagesArg) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} From c1068eb6b3a325776e3ea920f6af435efc3e2bc4 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Fri, 24 Jan 2025 15:38:15 -0800 Subject: [PATCH 15/36] Set AssetManifestFilePath again --- eng/Publishing.props | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/Publishing.props b/eng/Publishing.props index 0347fea9c02864..251fdcec547269 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -8,6 +8,8 @@ false $(PackageRID).xml + + $(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName) - + diff --git a/eng/Signing.props b/eng/Signing.props index 07761eda85163e..2dfee7d17628c8 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -23,10 +23,10 @@ - @@ -48,7 +48,8 @@ $(ArtifactsPackagesDir)**\*.rpm; $(ArtifactsPackagesDir)**\*.pkg; $(ArtifactsPackagesDir)**\*.exe; - $(ArtifactsPackagesDir)**\*.msi" + $(ArtifactsPackagesDir)**\*.msi + $(ArtifactsDir)VSSetup\**\*.zip;" Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"> @@ -56,4 +57,4 @@ - \ No newline at end of file + From 552bb1d9eb1eb961cfb1c28d62baa4efcdc627c8 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 27 Jan 2025 16:53:45 -0800 Subject: [PATCH 20/36] Add missing semicolon --- eng/Signing.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Signing.props b/eng/Signing.props index 2dfee7d17628c8..9e8512c3355eda 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -48,7 +48,7 @@ $(ArtifactsPackagesDir)**\*.rpm; $(ArtifactsPackagesDir)**\*.pkg; $(ArtifactsPackagesDir)**\*.exe; - $(ArtifactsPackagesDir)**\*.msi + $(ArtifactsPackagesDir)**\*.msi; $(ArtifactsDir)VSSetup\**\*.zip;" Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"> From 92470278b04e97cffd34f730dfdf7676eb5204e7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 27 Jan 2025 16:57:29 -0800 Subject: [PATCH 21/36] Include VSSetup artifacts separately --- eng/Publishing.props | 5 ----- eng/Signing.props | 10 ++++++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index c58d8cca34d625..9332041a8d6687 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -101,11 +101,6 @@ - - - - - diff --git a/eng/Signing.props b/eng/Signing.props index 9e8512c3355eda..78688762d0dfcc 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -48,13 +48,19 @@ $(ArtifactsPackagesDir)**\*.rpm; $(ArtifactsPackagesDir)**\*.pkg; $(ArtifactsPackagesDir)**\*.exe; - $(ArtifactsPackagesDir)**\*.msi; - $(ArtifactsDir)VSSetup\**\*.zip;" + $(ArtifactsPackagesDir)**\*.msi;" Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz" IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))"> %(FullPath).sha512 + + + %(FullPath).sha512 + workloads/ + From 1eb6a857e15fbd3aaf5861e14e6439d49438ffd7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Tue, 28 Jan 2025 14:25:17 -0800 Subject: [PATCH 22/36] Re-enable signing (depends on Microbuild signing on Linux/Mac) --- eng/pipelines/runtime-official.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index b3c211bd6792a3..2afda15fa1ebdd 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -88,7 +88,7 @@ extends: # Now that we've signed the diagnostic files, do the rest of the build. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true /p:PublishAllPackages=$(PublishAllPackages) + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true /p:PublishAllPackages=$(PublishAllPackages) displayName: Build managed CoreCLR components, Mono, all libraries, hosts, and packs # @@ -134,7 +134,7 @@ extends: # Now that we've entitled and signed createdump, we can build the rest. - template: /eng/pipelines/common/templates/global-build-step.yml parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs # @@ -154,7 +154,7 @@ extends: - linux_musl_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -186,7 +186,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true /p:AssetManifestFileName=CrossDac.xml + buildArgs: -s crossdacpack -c $(_BuildConfig) -restore -build -sign -publish /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) /p:PublishAllPackages=true /p:AssetManifestFileName=CrossDac.xml nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 @@ -254,7 +254,7 @@ extends: - linux_bionic_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -c $(_BuildConfig) -restore -build -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true + buildArgs: -c $(_BuildConfig) -restore -build -sign -publish /p:BuildMonoAOTCrossCompiler=false /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -268,7 +268,7 @@ extends: - wasi_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -281,7 +281,7 @@ extends: - browser_wasm jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) /p:AssetManifestFileName=browser_wasm_Multithreaded.xml + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) /p:AssetManifestFileName=browser_wasm_Multithreaded.xml nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread @@ -300,7 +300,7 @@ extends: runtimeFlavor: mono jobParameters: templatePath: 'templates-official' - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -publish /p:AssetManifestFileName=$(_AssetManifestName).xml + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish /p:AssetManifestFileName=$(_AssetManifestName).xml /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT @@ -317,7 +317,7 @@ extends: - windows_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s tools+libs -restore -build -pack -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true /p:AssetManifestFileName=LibrariesPackages.xml + buildArgs: -s tools+libs -restore -build -pack -sign -publish -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true /p:PublishAllPackages=true /p:AssetManifestFileName=LibrariesPackages.xml nameSuffix: Libraries_WithPackages isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 95 @@ -347,7 +347,7 @@ extends: - linux_arm64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true /p:AssetManifestFileName=$(_AssetManifestName).xml + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -restore -build -sign -publish -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true /p:AssetManifestFileName=$(_AssetManifestName).xml isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO timeoutInMinutes: 95 @@ -401,7 +401,7 @@ extends: PackageArtifacts/Microsoft.NETCore.App.Runtime.win-x86*.nupkg PackageArtifacts/Microsoft.NET.Sdk.WebAssembly.Pack*.nupkg - buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true /p:AssetManifestFileName=Workloads.xml + buildArgs: -s mono.workloads -c $(_BuildConfig) -restore -build -sign -publish /p:PackageSource=$(Build.SourcesDirectory)/artifacts/workloadPackages /p:WorkloadOutputPath=$(Build.SourcesDirectory)/artifacts/workloads /p:ShouldGenerateProductVersionFiles=true /p:PublishAllPackages=true /p:AssetManifestFileName=Workloads.xml isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 From 8ce2e0992466436e43d2330ab5c060055757e5b2 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Wed, 29 Jan 2025 11:12:37 -0800 Subject: [PATCH 23/36] PR feedback --- eng/Publishing.props | 10 +--------- eng/Signing.props | 7 +++---- eng/pipelines/runtime-official.yml | 12 ++++++------ 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 9332041a8d6687..f324969ef3f8d4 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -7,7 +7,7 @@ false - $(PackageRID).xml + $(AGENT_JOBNAME) $(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName) @@ -92,14 +92,6 @@ - - - - - - diff --git a/eng/Signing.props b/eng/Signing.props index 54aa91c7ede5af..253ede751b44e4 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -1,4 +1,7 @@ + + + @@ -10,6 +13,9 @@ + + + diff --git a/eng/pipelines/common/macos-sign-with-entitlements.yml b/eng/pipelines/common/macos-sign-with-entitlements.yml deleted file mode 100644 index 72a03b90f340d6..00000000000000 --- a/eng/pipelines/common/macos-sign-with-entitlements.yml +++ /dev/null @@ -1,77 +0,0 @@ -parameters: - filesToSign: [] - timeoutInMinutes: '30' - -steps: - - task: UseDotNet@2 - displayName: Install .NET 6 SDK for signing. - inputs: - packageType: 'sdk' - version: '6.0.x' - installationPath: '$(Agent.TempDirectory)/dotnet' - - - ${{ each file in parameters.filesToSign }}: - - task: CopyFiles@2 - displayName: 'Copy entitled file ${{ file.name }}' - inputs: - contents: '${{ file.path }}/${{ file.name }}' - targetFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled' - overWrite: true - - - task: ArchiveFiles@2 - displayName: 'Zip MacOS files for signing' - inputs: - rootFolderOrFile: '$(Build.ArtifactStagingDirectory)/mac_entitled' - archiveFile: '$(Build.ArtifactStagingDirectory)/mac_entitled_to_sign.zip' - archiveType: zip - includeRootFolder: true - replaceExistingArchive: true - - - task: EsrpCodeSigning@5 - displayName: 'ESRP CodeSigning' - inputs: - ConnectedServiceName: 'DotNet-Engineering-Services_KeyVault' - AppRegistrationClientId: '28ec6507-2167-4eaa-a294-34408cf5dd0e' - AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - AuthAKVName: 'EngKeyVault' - AuthCertName: 'DotNetCore-ESRP-AuthCert' - AuthSignCertName: 'DotNetCore-ESRP-AuthSignCert' - FolderPath: '$(Build.ArtifactStagingDirectory)/' - Pattern: 'mac_entitled_to_sign.zip' - UseMinimatch: true - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "keyCode": "CP-401337-Apple", - "operationCode": "MacAppDeveloperSign", - "parameters" : { - "hardening": "Enable" - }, - "toolName": "sign", - "toolVersion": "1.0" - } - ] - SessionTimeout: ${{ parameters.timeoutInMinutes }} - MaxConcurrency: '50' - MaxRetryAttempts: '5' - PendingAnalysisWaitTimeoutMinutes: '5' - env: - DOTNET_MULTILEVEL_LOOKUP: 0 - DOTNET_ROOT: '$(Agent.TempDirectory)/dotnet' - DOTNET_MSBUILD_SDK_RESOLVER_CLI_DIR: '$(Agent.TempDirectory)/dotnet' - - - task: ExtractFiles@1 - displayName: 'Extract MacOS after signing' - inputs: - archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/mac_entitled_to_sign.zip' - destinationFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled_signed' - - - ${{ each file in parameters.filesToSign }}: - - task: CopyFiles@2 - displayName: 'Copy ${{ file.name }} to destination' - inputs: - contents: ${{ file.name }} - sourceFolder: '$(Build.ArtifactStagingDirectory)/mac_entitled_signed' - targetFolder: '${{ file.path }}' - overWrite: true diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 1385372daefe3d..ac169e34d6dfba 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -109,37 +109,10 @@ extends: - osx_x64 jobParameters: templatePath: 'templates-official' - buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + buildArgs: -s -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true nameSuffix: AllRuntimes isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 - postBuildSteps: - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/macos-sign-with-entitlements.yml - parameters: - filesToSign: - - name: createdump - path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) - - name: corerun - path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) - - name: dotnet - path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost - - name: apphost - path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost - - - task: CopyFiles@2 - displayName: 'Copy signed createdump to sharedFramework' - inputs: - contents: createdump - sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) - targetFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig)/sharedFramework - overWrite: true - - # Now that we've entitled and signed createdump, we can build the rest. - - template: /eng/pipelines/common/templates/global-build-step.yml - parameters: - buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+mono+libs+host.tools+host.pkg+packs -c $(_BuildConfig) -restore -build -sign -publish /p:DotNetBuildAllRuntimePacks=true - displayName: Build managed CoreCLR and host components, Mono, all libraries, and packs # # Build all runtime packs for Linux and Linux musl From 220a7dc4baafe28249d6dc21b705e7485083a6d7 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 30 Jan 2025 11:29:19 -0800 Subject: [PATCH 25/36] In a VMR-style build, produce the product version file in either the win-x64 leg (for UB) or in all legs (for SB) --- eng/DotNetBuild.props | 1 + eng/Publishing.props | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props index 223eae78ef3ee3..8e3a42b5c7b5b4 100644 --- a/eng/DotNetBuild.props +++ b/eng/DotNetBuild.props @@ -78,6 +78,7 @@ $(InnerBuildArgs) /p:MonoBundleLLVMOptimizer=$(DotNetBuildMonoBundleLLVMOptimizer) $(InnerBuildArgs) /p:DotNetBuildMonoCrossAOT=$(DotNetBuildMonoCrossAOT) $(InnerBuildArgs) /p:DotNetBuildAllRuntimePacks=$(DotNetBuildAllRuntimePacks) + $(InnerBuildArgs) /p:DotNetBuildPass=$(DotNetBuildPass) $(InnerBuildArgs) $(FlagParameterPrefix)pgoinstrument diff --git a/eng/Publishing.props b/eng/Publishing.props index 95ee767f3d2d91..62d8fc0929b2ed 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -85,11 +85,10 @@ In non-VMR builds, we can skip publishing RID-agnostic packages entirely when we're not the lane that is supposed to publish them. In VMR builds, we need to publish them for upstack jobs to consume. --> - + - @@ -108,8 +107,9 @@ This ensures that we don't produce these files in the "Repo source build" builds, but we do produce them in both the VMR and the runtime official build. --> - - true + + true + true Date: Fri, 31 Jan 2025 15:10:30 -0800 Subject: [PATCH 26/36] Change where we set PublishAllPackages to make things automatically work when this change flows to the VMR. --- eng/DotNetBuild.props | 8 ++++++++ eng/Publishing.props | 3 +-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props index 8e3a42b5c7b5b4..746e74d075fc60 100644 --- a/eng/DotNetBuild.props +++ b/eng/DotNetBuild.props @@ -110,6 +110,14 @@ $(InnerBuildArgs) /p:NetCoreAppToolCurrentVersion=$(NetCoreAppToolCurrentVersion) + + + $(InnerBuildArgs) /p:PublishAllPackages=true diff --git a/eng/Publishing.props b/eng/Publishing.props index 62d8fc0929b2ed..987823290a9bcb 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -38,7 +38,6 @@ - A job can specify PublishAllPackages=true to publish all packages it produces. We have specific jobs that produce RID-agnostic packages or packages for multiple RIDs set this property. - In Source-Build scenarios, we want to publish all packages from here as source-build legs are fully self-contained. - For some target RIDs, we also include specific RID-agnostic packages. --> @@ -79,7 +78,7 @@ - + false - - $(AGENT_JOBNAME) - - $(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName) - $(InnerBuildArgs) /p:PublishAllPackages=true + $(InnerBuildArgs) /p:EnableDefaultArtifacts=true diff --git a/eng/Publishing.props b/eng/Publishing.props index 904caf1af838a8..55b6d2088f151d 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -3,86 +3,106 @@ true false - - - - - - - - + + false + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - $(InnerBuildArgs) /p:EnableDefaultArtifacts=true + $(InnerBuildArgs) /p:EnableDefaultArtifacts=false diff --git a/eng/Publishing.props b/eng/Publishing.props index 55b6d2088f151d..487c8abcc4a53a 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -13,9 +13,11 @@ We have specific jobs that produce RID-agnostic packages or packages for multiple RIDs set this property. - For some target RIDs, we also include specific RID-agnostic packages. + VMR jobs control whether or not a vertical has EnableDefaultArtifacts set to true or false in DotNetBuild.props. + Packages that do not meet the above rules are added with Vertical visibility in the VMR and excluded in non-VMR builds. --> - false + false false - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eng/Signing.props b/eng/Signing.props index 253ede751b44e4..6c78a667bdd91b 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -2,6 +2,23 @@ + + + false + + @@ -68,4 +85,86 @@ SubBlobFolder="workloads/" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From cbfbe4a426731f9b34e92aa3128094a2ed48b083 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 11:00:36 -0800 Subject: [PATCH 33/36] Only publish the LLVM runtime packs from LLVM legs --- eng/Signing.props | 61 +++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 6c78a667bdd91b..18d123f01d40b1 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -17,6 +17,10 @@ Packages that do not meet the above rules are added with Vertical visibility in the VMR and excluded in non-VMR builds. --> false + true + false + + true @@ -57,14 +61,7 @@ - - - - - - - - + + + + + + + + + + + + + + + + @@ -146,25 +168,6 @@ Kind="Package" Condition="'$(DotNetBuildOrchestrator)' == 'true'" /> - - - - - - - - - From 60b8af5ee39aca28cd472ed3bed15ddb8f089efe Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 11:45:19 -0800 Subject: [PATCH 34/36] Add the package artifacts to the right item group --- eng/Signing.props | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eng/Signing.props b/eng/Signing.props index 18d123f01d40b1..746078a6cecef5 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -115,6 +115,9 @@ + From 7b209dde3826cc43a823d94fa9ad94651fd9ff39 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 13:14:06 -0800 Subject: [PATCH 35/36] Fix package name format --- eng/Signing.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 746078a6cecef5..5b6717e0eff634 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -18,7 +18,7 @@ --> false true - false + false true @@ -112,9 +112,9 @@ IsShipping="false" /> - + - + From ca33ca51a16c3d1c97faf3fcb7c79403ae1ddbf1 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Thu, 6 Feb 2025 16:52:22 -0800 Subject: [PATCH 36/36] Fix handling host-targeting assets as NETCoreSdkRuntimeIdentifier is not available. --- eng/Signing.props | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/eng/Signing.props b/eng/Signing.props index 5b6717e0eff634..6b6b0d73a3c5d6 100644 --- a/eng/Signing.props +++ b/eng/Signing.props @@ -95,19 +95,24 @@ - - - - - + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILAsm.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.NETCore.ILDAsm.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.NETCore.ILDAsm.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\runtime.*.Microsoft.DotNet.ILCompiler.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\runtime.$(OutputRID).Microsoft.DotNet.ILCompiler.*.nupkg" /> + + <_HostArtifact Include="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.*.nupkg" + Exclude="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(OutputRID).*.nupkg" /> + +