Parallelize SQL E2E and Integration tests

build/build-variables.yml

@@ -16,21 +16,53 @@ variables:
   azureContainerRegistryName: 'healthplatformregistry'
   azureContainerRegistry: '$(azureContainerRegistryName).azurecr.io'
   DeploymentEnvironmentNameSql: '$(DeploymentEnvironmentName)-sql'
+  # Isolation endpoint for Stu3 SQL (mirrors the R4 SQL isolation endpoint): dedicated container app +
+  # dedicated database on the SAME app SQL server so RequiresIsolation E2E tests run concurrently with shards.
+  DeploymentEnvironmentNameSqlIso: '$(DeploymentEnvironmentNameSql)-iso'
   DeploymentEnvironmentNameR4: '$(DeploymentEnvironmentName)-r4'
   DeploymentEnvironmentNameR4Sql: '$(DeploymentEnvironmentNameR4)-sql'
+  # Isolation endpoint for R4 SQL: dedicated container app + dedicated database on the SAME app SQL
+  # server, so RequiresIsolation E2E tests (reindex, bulk delete/update, custom search params) run
+  # concurrently with the parallel shards instead of in a serialized terminal lane.
+  DeploymentEnvironmentNameR4SqlIso: '$(DeploymentEnvironmentNameR4Sql)-iso'
   DeploymentEnvironmentNameR4B: '$(DeploymentEnvironmentName)-r4b'
   DeploymentEnvironmentNameR4BSql: '$(DeploymentEnvironmentNameR4B)-sql'
   DeploymentEnvironmentNameR5: '$(DeploymentEnvironmentName)-r5'
   DeploymentEnvironmentNameR5Sql: '$(DeploymentEnvironmentNameR5)-sql'
+  # Isolation endpoint for R5 SQL (mirrors R4): dedicated container app + dedicated database on the shared server.
+  DeploymentEnvironmentNameR5SqlIso: '$(DeploymentEnvironmentNameR5Sql)-iso'
+  # Cosmos isolation endpoints (mirror the SQL isolation endpoints): each gets its OWN dedicated Cosmos
+  # account (the deploy script names the Cosmos account after the container app), so RequiresIsolation
+  # Cosmos E2E tests run concurrently with the parallel shards instead of in a serialized terminal lane.
+  # Use a compact '-ciso' (cosmos-iso) suffix; it is shorter than the proven-safe SQL '-sql-iso' suffix,
+  # so it stays within the ACA container-app (32 char) and Cosmos account (44 char) name limits.
+  DeploymentEnvironmentNameCosmosIso: '$(DeploymentEnvironmentName)-ciso'
+  DeploymentEnvironmentNameR4CosmosIso: '$(DeploymentEnvironmentNameR4)-ciso'
   AcaEnvironmentName: '$(DeploymentEnvironmentName)-acae'
   # Key Vault names (shorter due to 24 character limit)
   KeyVaultNameSql: '$(KeyVaultBaseName)-sql'
+  # Stu3 SQL isolation vault. Must be <= 24 chars. Use compact 'sqi' (sql-iso) token: '$(KeyVaultBaseName)-sqi'
+  # (base + 4 chars), well under the limit and distinct from the main Stu3 SQL vault.
+  KeyVaultNameSqlIso: '$(KeyVaultBaseName)-sqi'
   KeyVaultNameR4: '$(KeyVaultBaseName)-r4'
   KeyVaultNameR4Sql: '$(KeyVaultNameR4)-sql'
+  # Key Vault names must be <= 24 chars. KeyVaultNameR4Sql is already 'f<buildNumber>-r4-sql' (21 chars at the
+  # proven-safe max), so a '-iso' suffix would overflow. Use a compact 'sqi' (sql-iso) token to stay the same
+  # length as the main R4 SQL vault while remaining distinct (each ACA app provisions its own vault).
+  KeyVaultNameR4SqlIso: '$(KeyVaultNameR4)-sqi'
   KeyVaultNameR4B: '$(KeyVaultBaseName)-r4b'
   KeyVaultNameR4BSql: '$(KeyVaultNameR4B)-sql'
   KeyVaultNameR5: '$(KeyVaultBaseName)-r5'
   KeyVaultNameR5Sql: '$(KeyVaultNameR5)-sql'
+  # R5 SQL isolation vault. '$(KeyVaultNameR5)-sqi' = '<base>-r5-sqi' (base + 7 chars), same length as the
+  # proven-safe R4 SQL iso vault and under the 24-char limit.
+  KeyVaultNameR5SqlIso: '$(KeyVaultNameR5)-sqi'
+  # Cosmos isolation vaults. Each iso ACA app provisions its own vault. Use a compact 'csi' (cosmos-iso)
+  # token to stay <= 24 chars: '$(KeyVaultBaseName)-csi' = '<base>-csi' (base + 4 = 20 chars) for Stu3,
+  # and '$(KeyVaultNameR4)-csi' = '<base>-r4-csi' (base + 7 = 23 chars) for R4 — same lengths as the
+  # proven-safe SQL iso vaults.
+  KeyVaultNameCosmosIso: '$(KeyVaultBaseName)-csi'
+  KeyVaultNameR4CosmosIso: '$(KeyVaultNameR4)-csi'
   TestEnvironmentUrl: 'https://$(DeploymentEnvironmentName).azurewebsites.net'
   # These variables are not used in the deployment scripts, but are used in the E2E tests files.
   TestEnvironmentUrl_Sql: 'https://$(DeploymentEnvironmentName)-sql.azurewebsites.net'

build/jobs/e2e-tests.yml

@@ -5,9 +5,9 @@ parameters:
   type: string
 - name: appServiceType
   type: string
-- name: categoryFilter
+- name: testFilter
   type: string
-  default: 'Category!=ExportLongRunning'
+  default: ''
 - name: testRunTitleSuffix
   type: string
   default: ''
@@ -37,7 +37,11 @@ steps:
           throw "Could not find $testName.dll under $testRoot"
         }
 
-        $filter = "FullyQualifiedName~${{ parameters.appServiceType }}&${{ parameters.categoryFilter }}"
+        $filter = "${{ parameters.testFilter }}"
+        if ([string]::IsNullOrWhiteSpace($filter)) {
+          throw "testFilter is required and must not be empty."
+        }
+
         $args = @('--filter', $filter, '--retry-failed-tests', '3', '--report-trx')
 
         Write-Host "Running dotnet $($dll.FullName)"
@@ -95,3 +99,23 @@ steps:
       testRunTitle: '${{ parameters.version }} ${{parameters.appServiceType}}${{ parameters.testRunTitleSuffix }}'
       failTaskOnFailedTests: true
     condition: succeededOrFailed()
+
+  - task: PowerShell@2
+    displayName: 'Report E2E duration by class'
+    condition: succeededOrFailed()
+    inputs:
+      targetType: filePath
+      filePath: '$(System.DefaultWorkingDirectory)/build/scripts/Write-TestClassDurationReport.ps1'
+      pwsh: true
+      arguments: >-
+        -ResultsDirectory "$(Agent.TempDirectory)/testresults"
+        -OutputDirectory "$(Agent.TempDirectory)/test-class-duration"
+        -ReportName "E2E_${{ parameters.appServiceType }}_${{ parameters.version }}${{ replace(parameters.testRunTitleSuffix, ' ', '_') }}_ByClass"
+
+  - task: PublishBuildArtifacts@1
+    displayName: 'Publish E2E class duration report'
+    condition: succeededOrFailed()
+    inputs:
+      pathToPublish: '$(Agent.TempDirectory)/test-class-duration'
+      artifactName: "TestClassDuration_E2E_${{ parameters.appServiceType }}_${{ parameters.version }}${{ replace(parameters.testRunTitleSuffix, ' ', '_') }}"
+      artifactType: 'container'

build/jobs/integration-tests-run.yml

@@ -0,0 +1,144 @@
+parameters:
+- name: version
+  type: string
+- name: dataStore
+  type: string  # 'Sql' or 'Cosmos'
+- name: keyVaultName
+  type: string
+- name: testFilter
+  type: string
+- name: testRunTitle
+  type: string
+- name: artifactSuffix
+  type: string
+- name: containerAppName
+  type: string
+  default: ''
+- name: integrationSqlServerName
+  type: string
+  default: ''
+
+steps:
+# Keep integration tests on the project-based dotnet test path used by main. Directly invoking
+# prebuilt test DLLs works for execution, but coverage + retry TRX files are not merged correctly
+# by PublishTestResults. When the pipeline moves to .NET 10, revisit prebuilt execution with
+# `dotnet test --test-modules`.
+- task: DotNetCoreCLI@2
+  displayName: 'Build Integration Test Projects'
+  inputs:
+    command: build
+    projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj'
+    arguments: '--configuration $(buildConfiguration) -f $(defaultBuildFramework)'
+
+- task: AzureKeyVault@1
+  displayName: 'Azure Key Vault: ${{ parameters.keyVaultName }}'
+  inputs:
+    azureSubscription: $(ConnectedServiceName)
+    KeyVaultName: '${{ parameters.keyVaultName }}'
+
+- ${{ if eq(parameters.dataStore, 'Sql') }}:
+  - task: AzurePowerShell@5
+    displayName: 'Set Workload Identity Variables'
+    inputs:
+      azureSubscription: $(ConnectedServiceName)
+      azurePowerShellVersion: latestVersion
+      pwsh: true
+      ScriptType: inlineScript
+      Inline: |
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID"
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID"
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"
+
+- ${{ if eq(parameters.dataStore, 'Cosmos') }}:
+  - task: AzurePowerShell@5
+    displayName: 'Set Workload Identity Variables'
+    inputs:
+      azureSubscription: $(ConnectedServiceName)
+      azurePowerShellVersion: latestVersion
+      pwsh: true
+      ScriptType: inlineScript
+      Inline: |
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_CLIENT_ID]$env:AZURESUBSCRIPTION_CLIENT_ID"
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_TENANT_ID]$env:AZURESUBSCRIPTION_TENANT_ID"
+        Write-Host "##vso[task.setvariable variable=AZURESUBSCRIPTION_SERVICE_CONNECTION_ID]$env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID"
+
+        $containerAppName = '${{ parameters.containerAppName }}'
+        $containerApp = Get-AzResource -ResourceGroupName $(UniqueResourceGroupName) -ResourceType "Microsoft.App/containerApps" -Name $containerAppName -ApiVersion "2023-05-01" -ExpandProperties
+        if ($null -eq $containerApp) {
+            throw "Container App '$containerAppName' was not found in resource group '$(UniqueResourceGroupName)'"
+        }
+
+        $containerAppData = $containerApp | ConvertTo-Json -Depth 100 | ConvertFrom-Json
+        $envSettings = $containerAppData.Properties.template.containers[0].env
+
+        $dataStoreResourceId = ($envSettings | Where-Object { $_.name -eq "FhirServer__ResourceManager__DataStoreResourceId" } | Select-Object -First 1).value
+        Write-Host "$dataStoreResourceId"
+        Write-Host "##vso[task.setvariable variable=DataStoreResourceId]$($dataStoreResourceId)"
+
+- task: DotNetCoreCLI@2
+  displayName: 'Run ${{ parameters.dataStore }} Integration Tests ${{ parameters.testRunTitle }} with coverage'
+  inputs:
+    command: test
+    projects: '$(Pipeline.Workspace)/source/test/**/*${{ parameters.version }}.Tests.Integration.csproj'
+    arguments: '--configuration $(buildConfiguration) --no-build -f $(defaultBuildFramework) -- --filter "${{ parameters.testFilter }}" --retry-failed-tests 3 --coverage --coverage-output-format cobertura --coverage-settings "$(System.DefaultWorkingDirectory)/CodeCoverage.Mtp.settings.xml" --report-trx'
+    testRunTitle: '${{ parameters.testRunTitle }}'
+    # Disable the task's built-in (non-retry-aware) result publishing so the
+    # explicit retry-aware PublishTestResults@2 task below is the sole publisher.
+    publishTestResults: false
+  env:
+    platformOptions__resultDirectory: '$(Agent.TempDirectory)/coverage'
+    'AZURESUBSCRIPTION_CLIENT_ID': '$(AZURESUBSCRIPTION_CLIENT_ID)'
+    'AZURESUBSCRIPTION_TENANT_ID': '$(AZURESUBSCRIPTION_TENANT_ID)'
+    'AZURESUBSCRIPTION_SERVICE_CONNECTION_ID': '$(AZURESUBSCRIPTION_SERVICE_CONNECTION_ID)'
+    'SYSTEM_ACCESSTOKEN': $(System.AccessToken)
+    ${{ if eq(parameters.dataStore, 'Sql') }}:
+      'SqlServer:ConnectionString': 'Server=tcp:${{ parameters.integrationSqlServerName }}.database.windows.net,1433;Initial Catalog=master;Persist Security Info=False;Authentication=ActiveDirectoryWorkloadIdentity;MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;User Id=$(AZURESUBSCRIPTION_CLIENT_ID);'
+    ${{ if eq(parameters.dataStore, 'Cosmos') }}:
+      'CosmosDb__Host': $(CosmosDb--Host)
+      'FhirServer__ResourceManager__DataStoreResourceId': '$(DataStoreResourceId)'
+      'CosmosDb__UseManagedIdentity': true
+
+- task: PublishTestResults@2
+  displayName: 'Publish integration test results'
+  inputs:
+    testResultsFormat: 'VSTest'
+    testResultsFiles: '$(Agent.TempDirectory)/coverage/**/*.trx'
+    mergeTestResults: true
+    testRunTitle: '${{ parameters.testRunTitle }}'
+    failTaskOnFailedTests: true
+  condition: succeededOrFailed()
+
+- task: PowerShell@2
+  displayName: 'Report integration duration by class'
+  condition: succeededOrFailed()
+  inputs:
+    targetType: filePath
+    filePath: '$(System.DefaultWorkingDirectory)/build/scripts/Write-TestClassDurationReport.ps1'
+    pwsh: true
+    arguments: >-
+      -ResultsDirectory "$(Agent.TempDirectory)/coverage"
+      -OutputDirectory "$(Agent.TempDirectory)/test-class-duration"
+      -ReportName "${{ parameters.artifactSuffix }}_ByClass"
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish integration class duration report'
+  condition: succeededOrFailed()
+  inputs:
+    pathToPublish: '$(Agent.TempDirectory)/test-class-duration'
+    artifactName: 'TestClassDuration_IntegrationTests_${{ parameters.artifactSuffix }}'
+    artifactType: 'container'
+
+- task: reportgenerator@5
+  displayName: 'Aggregate integration test coverage'
+  condition: succeededOrFailed()
+  inputs:
+    reports: '$(Agent.TempDirectory)/coverage/**/*.cobertura.xml'
+    reporttypes: 'Cobertura'
+    targetdir: '$(Agent.TempDirectory)/coverage-aggregated'
+
+- task: PublishBuildArtifacts@1
+  displayName: 'Publish integration test coverage'
+  inputs:
+    pathToPublish: '$(Agent.TempDirectory)/coverage-aggregated'
+    artifactName: 'Coverage_IntegrationTests_${{ parameters.artifactSuffix }}'
+    artifactType: 'container'

build/jobs/provision-deploy.yml

@@ -22,6 +22,9 @@ parameters:
 - name: sqlServerName
   type: string
   default: ''
+- name: sqlDatabaseName
+  type: string
+  default: ''
 - name: sqlComputeTier
   type: string
   default: ''
@@ -70,6 +73,7 @@ jobs:
         -AzureContainerRegistry "$(azureContainerRegistry)"
         -TenantIdGuid "$(tenant-id-guid)"
         -SqlServerName "${{ parameters.sqlServerName }}"
+        -SqlDatabaseName "${{ parameters.sqlDatabaseName }}"
         -SqlElasticPoolName "${{ parameters.sqlElasticPoolName }}"
         -SchemaAutomaticUpdatesEnabled "${{ parameters.schemaAutomaticUpdatesEnabled }}"
         -ReindexEnabled "${{ parameters.reindexEnabled }}"