Split generator pipelines and make E2E publish work (Azure#49321)

* tsp-client debug

* backtick

* workingdirectory

* add working directory

* source

* don't install globally

* outputdir

* sources

* fix path

* Pass version

* backtick

* use eng

* print emitter version

* fix print

* remove version

* fix

* relative path

* add back version

* Temp work for multiple-emitter-prs

* create dir

* pass name

* fix version

* fix file name

* fix filename

* fix build path

* fix working directory and always pass emitter path

* Use parameter default

* npmrc

* ci

* remove packagename

* remove pn

* make lockfiles

* skip mgmt for now

* fix outputdirectory build

* copy nmrc to source dir

* fix npmrc

* print the registry

* ugh

* fix overrides

* add logging

* fix escaped $

* fix $

* log

* print overrides

* fix

* fix override path

* actually commit changes

* buildartifactspath

* more logs

* fix path

* updates

* restore

* fix

* remove lockfilespath

* fix arg

* if

* logging

* Add package path

* sync

* move inside function

* check-codegen

* Revert unintended change

* revert doc

* delete artifacts

* revert

* delete

* revert

* revert

---------

Co-authored-by: Patrick Hallisey <[email protected]>

Author: JoshLove-msft

eng/packages/http-client-csharp-mgmt/ci.yml

@@ -0,0 +1,73 @@
+trigger:
+  branches:
+    include:
+      - main
+  paths:
+    include:
+      - eng/packages/http-client-csharp
+      - eng/packages/http-client-csharp-mgmt
+      - eng/scripts/typespec
+pr:
+  branches:
+    include:
+      - main
+      - feature/*
+      - hotfix/*
+      - release/*
+  paths:
+    include:
+      - eng/packages/http-client-csharp
+      - eng/packages/http-client-csharp-mgmt
+      - eng/scripts/typespec
+
+parameters:
+  - name: UseTypeSpecNext
+    displayName: "Use TypeSpec Next"
+    type: boolean
+    default: false
+  - name: ShouldPublish
+    displayName: "Publish packages"
+    type: boolean
+    default: false
+  - name: PublishPublic
+    displayName: "Publish to npmjs.org"
+    type: boolean
+    default: false
+  - name: ShouldRegenerate
+    displayName: "Regenerate clients"
+    type: boolean
+    default: false
+
+variables:
+  - template: /eng/pipelines/templates/variables/image.yml
+
+extends:
+  template: /eng/common/pipelines/templates/archetype-typespec-emitter.yml
+  parameters:
+    EmitterPackageJsonPath: eng/http-client-csharp-mgmt-emitter-package.json
+    EmitterPackagePath: eng/packages/http-client-csharp-mgmt
+    SparseCheckoutPaths:
+      - /sdk/core/Azure.Core/src/Shared
+      - /sdk/core/Azure.Core.TestFramework/src
+    BuildPrereleaseVersion: true
+    UseTypeSpecNext: ${{ parameters.UseTypeSpecNext }}
+    ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      ${{ if eq(variables['Build.Reason'], 'Manual') }}:
+        ShouldPublish: ${{ parameters.ShouldPublish }}
+        ${{ if startswith(variables['Build.SourceBranch'], 'refs/pull/') }}:  # PR ref's can't be published to public, even in a manual run
+          PublishPublic: false
+          PublishDependsOnTest: false
+        ${{ else }}: # Any other manual run can go to public
+           PublishPublic: ${{ parameters.PublishPublic }}
+           PublishDependsOnTest: ${{ parameters.PublishPublic }}
+      ${{ else }}: # All automatic runs publish, but only CI of main goes public
+        ShouldPublish: true
+        PublishDependsOnTest: false
+        PublishPublic: ${{ and(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) }}
+      RegenerationJobCount: 5
+      MinimumPerJob: 10
+      OnlyGenerateTypespec: true
+    TestMatrix:
+      All:
+        TestArguments: -UnitTests -GenerationChecks -Filter .NET
+        SpectorName: "http-client-csharp-mgmt"

eng/packages/http-client-csharp/ci.yml

@@ -5,7 +5,6 @@ trigger:
   paths:
     include:
       - eng/packages/http-client-csharp
-      - eng/packages/http-client-csharp-mgmt
       - eng/scripts/typespec
 pr:
   branches:
@@ -17,14 +16,9 @@ pr:
   paths:
     include:
       - eng/packages/http-client-csharp
-      - eng/packages/http-client-csharp-mgmt
       - eng/scripts/typespec
 
 parameters:
-  - name: BuildPrereleaseVersion
-    displayName: "Build prerelease version"
-    type: boolean
-    default: true
   - name: UseTypeSpecNext
     displayName: "Use TypeSpec Next"
     type: boolean
@@ -48,11 +42,12 @@ variables:
 extends:
   template: /eng/common/pipelines/templates/archetype-typespec-emitter.yml
   parameters:
-    EmitterPackageJsonPath: /eng/azure-typespec-emitter-package.json
+    EmitterPackageJsonOutputPath: eng/http-client-csharp-emitter-package.json
+    EmitterPackagePath: eng/packages/http-client-csharp
     SparseCheckoutPaths:
       - /sdk/core/Azure.Core/src/Shared
       - /sdk/core/Azure.Core.TestFramework/src
-    BuildPrereleaseVersion: ${{ parameters.BuildPrereleaseVersion }}
+    BuildPrereleaseVersion: true
     UseTypeSpecNext: ${{ parameters.UseTypeSpecNext }}
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:
       ${{ if eq(variables['Build.Reason'], 'Manual') }}:

eng/scripts/typespec/Build-Emitter.ps1

@@ -1,83 +1,76 @@
 #Requires -Version 7.0
 
 param(
-    [string] $BuildNumber,
     [string] $OutputDirectory,
-    [switch] $Prerelease,
-    [switch] $TargetNpmJsFeed
+    [switch] $TargetNpmJsFeed,
+    [string] $emitterPackagePath
 )
 
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version 3.0
 . "$PSScriptRoot/../../common/scripts/common.ps1"
 Set-ConsoleEncoding
 
-$packageRoot = Resolve-Path "$RepoRoot/eng/packages/http-client-csharp"
-$mgmtPackageRoot = Resolve-Path "$RepoRoot/eng/packages/http-client-csharp-mgmt"
-$outputPath = $OutputDirectory ? $OutputDirectory : (Join-Path $packageRoot "artifacts" "build")
-$mgmtOutputPath = $OutputDirectory ? $OutputDirectory : (Join-Path $mgmtPackageRoot "artifacts" "build")
-
 function Build-Emitter {
     param (
         [string]$packageRoot,
-        [string]$outputPath
+        [string]$outputPath,
+        [hashtable]$overrides
     )
 
     Push-Location $packageRoot
     try {
         Write-Host "Working in $PWD"
-        
-        $outputPath = New-Item -ItemType Directory -Force -Path $outputPath | Select-Object -ExpandProperty FullName
-
-        $emitterVersion = (npm pkg get version).Trim('"')
+        Write-Host "TargetNpmJsFeed: $TargetNpmJsFeed"
 
-        if ($BuildNumber) {
-            # set package versions
-            $versionTag = $Prerelease ? "-alpha" : "-beta"
-
-            $emitterVersion = "$($emitterVersion.Split('-')[0])$versionTag.$BuildNumber"
-            Write-Host "Setting output variable 'emitterVersion' to $emitterVersion"
-            Write-Host "##vso[task.setvariable variable=emitterVersion;isoutput=true]$emitterVersion"
-        }
+        $outputPath = New-Item -ItemType Directory -Force -Path $outputPath | Select-Object -ExpandProperty FullName
 
         # build and pack the emitter
-        Invoke-LoggedCommand "npm install @types/node --save-dev" -GroupOutput
         Invoke-LoggedCommand "npm run build" -GroupOutput
 
-        if ($BuildNumber) {
-            Write-Host "Updating version package.json to the new emitter version`n"
-            Invoke-LoggedCommand "npm pkg set version=$emitterVersion"
-        }
-
         # remove any existing tarballs
         Remove-Item -Path "./*.tgz" -Force | Out-Null
-        
+
         #pack the emitter
         Invoke-LoggedCommand "npm pack"
         $file = Get-ChildItem -Filter "*.tgz" | Select-Object -ExpandProperty FullName
 
-        # ensure the packages directory exists
-        New-Item -ItemType Directory -Force -Path "$outputPath/packages" | Out-Null
+        Write-Host "Copying $file to $outputPath"
+        Copy-Item $file -Destination $outputPath
 
-        Write-Host "Copying $file to $outputPath/packages"
-        Copy-Item $file -Destination "$outputPath/packages"
-
-        if ($TargetNpmJsFeed) {
-            $overrides = @{}
-        }
-        else {
+        if (!$TargetNpmJsFeed) {
             $feedUrl = "https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry"
 
-            $overrides = @{
-                "@azure-typespec/http-client-csharp" = "$feedUrl/@azure-typespec/http-client-csharp/-/http-client-csharp-$emitterVersion.tgz"
-            }
+            $packageJson = Get-Content -Path "./package.json" | ConvertFrom-Json
+            $packageVersion = $packageJson.version
+            $packageName = $packageJson.name
+
+            $unscopedName = $packageName.Split("/")[1]
+            $overrides[$packageName] = "$feedUrl/$packageName/-/$unscopedName-$packageVersion.tgz"
         }
+
+        # restore the package.json and package-lock.json files to their original state
+        Write-Host "Restoring package.json and package-lock.json to their original state"
+        Invoke-LoggedCommand "git restore package.json package-lock.json"
     }
     finally {
         Pop-Location
     }
 }
 
-Build-Emitter -packageRoot $packageRoot -outputPath $outputPath
+$overrides = @{}
+
+$outputPath = $OutputDirectory ? $OutputDirectory : (Join-Path $RepoRoot "artifacts" "emitters")
+New-Item -ItemType Directory -Force -Path $outputPath | Out-Null
+
+# strip leading slash from emitterPackagePath if it exists
+if ($emitterPackagePath.StartsWith("/")) {
+    $emitterPackagePath = $emitterPackagePath.Substring(1)
+}
+$packageRoot = Join-Path $RepoRoot $emitterPackagePath
+Build-Emitter -packageRoot $packageRoot -outputPath $outputPath -overrides $overrides
+
+Write-Host "Writing overrides to $outputPath/overrides.json"
 
-Build-Emitter -packageRoot $mgmtPackageRoot -outputPath $mgmtOutputPath
+$overrides | ConvertTo-Json | Out-File -FilePath "$outputPath/overrides.json" -Encoding utf8 -Force
+Get-Content -Path "$outputPath/overrides.json" | Out-Host

eng/scripts/typespec/Check-CodeGeneration.ps1

@@ -1,40 +1,25 @@
 #Requires -Version 7.0
 
+param(
+    [string] $EmitterPackagePath
+)
+
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version 3.0
 
-$packageRoot = Resolve-Path "$PSScriptRoot../../../packages/http-client-csharp"
-Push-Location $packageRoot
+Push-Location $EmitterPackagePath
 
 try {
-    & "$packageRoot/eng/scripts/Generate.ps1"
+    & "$EmitterPackagePath/eng/scripts/Generate.ps1"
 
     Write-Host 'Checking generated files difference...'
     git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code
     if ($LastExitCode -ne 0) {
-        Write-Error 'Generated codes of Azure generator are not up to date. Please run: eng/packages/http-client-csharp/eng/scripts/Generate.ps1'
+        Write-Error 'Generated code of Azure generator is not up to date. Please run: /eng/scripts/Generate.ps1'
         exit 1
     }
     Write-Host 'Done for Azure generator. No change is detected.'
 }
 finally {
     Pop-Location
 }
-
-$mgmtPackageRoot = Resolve-Path "$PSScriptRoot../../../packages/http-client-csharp-mgmt"
-Push-Location $mgmtPackageRoot
-
-try {
-    & "$mgmtPackageRoot/eng/scripts/Generate.ps1"
-
-    Write-Host 'Checking generated files difference...'
-    git -c core.safecrlf=false diff --ignore-space-at-eol --exit-code
-    if ($LastExitCode -ne 0) {
-        Write-Error 'Generated codes of mgmt generator are not up to date. Please run: eng/packages/http-client-csharp/eng/scripts/Generate.ps1'
-        exit 1
-    }
-    Write-Host 'Done for mgmt generator. No change is detected.'
-}
-finally {
-    Pop-Location
-}