From be7fa3aad5ce05593b5e33fdd3a14ba2b21997ef Mon Sep 17 00:00:00 2001 From: qmuntal Date: Wed, 11 Nov 2026 13:55:52 +0100 Subject: [PATCH] [release-branch.go1.27] Retry Darwin tests on fresh hosted agents Backport of #2616. Retry failed Darwin test runs once on a fresh hosted agent, reusing the already-built toolchain, and publish per-attempt diagnostics and results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cmd/pipelineymlgen/run_stage_test.go | 282 ++++++++++++++++++ eng/pipeline/stages/builders-to-stages.yml | 4 +- eng/pipeline/stages/run-stage.yml | 147 +++++---- eng/pipeline/steps/run-builder-tests.yml | 68 +++++ 4 files changed, 437 insertions(+), 64 deletions(-) create mode 100644 eng/_util/cmd/pipelineymlgen/run_stage_test.go create mode 100644 eng/pipeline/steps/run-builder-tests.yml diff --git a/eng/_util/cmd/pipelineymlgen/run_stage_test.go b/eng/_util/cmd/pipelineymlgen/run_stage_test.go new file mode 100644 index 00000000000..b3e1ed9293b --- /dev/null +++ b/eng/_util/cmd/pipelineymlgen/run_stage_test.go @@ -0,0 +1,282 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "go.yaml.in/yaml/v4" +) + +func readPipeline(t *testing.T, name string) *yaml.Node { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "eng", "pipeline", name)) + if err != nil { + t.Fatal(err) + } + var doc yaml.Node + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + return &doc +} + +func mappingValue(n *yaml.Node, key string) *yaml.Node { + if n != nil && n.Kind == yaml.MappingNode { + for i := 0; i < len(n.Content); i += 2 { + if n.Content[i].Value == key { + return n.Content[i+1] + } + } + } + return nil +} + +func findMapping(t *testing.T, n *yaml.Node, key, value string) *yaml.Node { + t.Helper() + var matches []*yaml.Node + var walk func(*yaml.Node) + walk = func(n *yaml.Node) { + if v := mappingValue(n, key); v != nil && v.Value == value { + matches = append(matches, n) + } + for _, child := range n.Content { + walk(child) + } + } + walk(n) + if len(matches) != 1 { + t.Fatalf("found %d mappings with %s=%q, want one", len(matches), key, value) + } + return matches[0] +} + +func requireValue(t *testing.T, n *yaml.Node, key, want string) { + t.Helper() + got := mappingValue(n, key) + if got == nil || got.Value != want { + t.Fatalf("%s = %v, want %q", key, got, want) + } +} + +func TestDarwinFreshAgentRetry(t *testing.T) { + builders := readPipeline(t, "stages/builders-to-stages.yml") + unbroken := findMapping(t, builders, "${{ if not(builder.broken) }}", "") + darwin := mappingValue(unbroken, "${{ if not(builder.broken) }}") + darwin = mappingValue(darwin, "${{ if eq(builder.os, 'darwin') }}") + if darwin == nil { + t.Fatal("missing Darwin builder configuration") + } + testBuilders := mappingValue(darwin, "${{ if and(ne(builder.config, 'buildandpack'), not(startsWith(builder.config, 'codeql'))) }}") + if testBuilders == nil { + t.Fatal("fresh-agent retries must exclude builders without tests") + } + requireValue(t, testBuilders, "retryOnFreshAgent", "true") + + stage := readPipeline(t, "stages/run-stage.yml") + requireValue(t, findMapping(t, stage, "name", "retryOnFreshAgent"), "default", "false") + stages := mappingValue(stage.Content[0], "stages") + jobs := mappingValue(stages.Content[0], "jobs") + if len(jobs.Content) != 2 || mappingValue(jobs.Content[1], "${{ if parameters.retryOnFreshAgent }}") == nil { + t.Fatal("expected only the original job and one compile-time-conditional retry job") + } + retry := findMapping(t, stage, "job", "${{ parameters.builder.id }}_test_retry") + requireValue(t, retry, "dependsOn", "${{ parameters.builder.id }}") + // Missing output (including setup failure), successful tests, and cancellation must not provision a VM. + requireValue(t, retry, "condition", "and(succeeded(), eq(dependencies.${{ parameters.builder.id }}.outputs['retry.required'], 'true'))") + if mappingValue(retry, "continueOnError") != nil { + t.Fatal("a failed final retry must fail the stage") + } + + request := findMapping(t, stage, "name", "retry") + wantCondition := "and(succeeded(), ne(variables['TEST_BUILDER_SUCCESSFUL'], 'true'))" + requireValue(t, request, "condition", wantCondition) + publish := findMapping(t, stage, "displayName", "Publish toolchain for test retry") + requireValue(t, publish, "condition", wantCondition) + if publish.Line >= request.Line { + t.Fatal("retry must only be requested after artifact publication succeeds") + } + snapshot := findMapping(t, stage, "displayName", "Preserve toolchain for test retry") + first := findMapping(t, stage, "job", "${{ parameters.builder.id }}") + requireValue(t, retry, "displayName", mappingValue(first, "displayName").Value+" (test retry)") + fresh := findMapping(t, mappingValue(first, "steps"), "${{ if parameters.retryOnFreshAgent }}", "") + initialTest := findMapping(t, mappingValue(fresh, "${{ if parameters.retryOnFreshAgent }}"), "template", "../steps/run-builder-tests.yml") + initialAttempts := mappingValue(mappingValue(initialTest, "parameters"), "retryAttempts") + if initialAttempts == nil || len(initialAttempts.Content) != 1 || initialAttempts.Content[0].Value != "1" { + t.Fatal("build job must run exactly one provisional test attempt") + } + if snapshot.Line >= initialTest.Line || initialTest.Line >= publish.Line { + t.Fatal("toolchain snapshot must precede tests, and publication must follow them") + } + + steps := mappingValue(retry, "steps") + final := findMapping(t, steps, "template", "../steps/run-builder-tests.yml") + attempts := mappingValue(mappingValue(final, "parameters"), "retryAttempts") + if attempts == nil || len(attempts.Content) != 1 || attempts.Content[0].Value != "FINAL" { + t.Fatal("retry job must run exactly one final test attempt") + } + requireValue(t, mappingValue(final, "parameters"), "artifactSuffix", "-test-retry") + restore := findMapping(t, retry, "displayName", "Restore toolchain for test retry") + if restore.Line >= final.Line { + t.Fatal("toolchain must be restored before the final test attempt") + } + for _, step := range steps.Content { + if v := mappingValue(step, "pwsh"); v != nil && strings.Contains(v.Value, "run-builder -build") { + t.Fatal("retry job must not rebuild the toolchain") + } + } + variables := mappingValue(retry, "variables") + artifact := findMapping(t, variables, "name", "retryToolchainArtifact") + requireValue(t, artifact, "value", "$[ dependencies.${{ parameters.builder.id }}.outputs['retry.toolchainArtifact'] ]") + download := findMapping(t, retry, "task", "DownloadPipelineArtifact@2") + requireValue(t, mappingValue(download, "inputs"), "artifactName", "$(retryToolchainArtifact)") + if download.Line >= restore.Line { + t.Fatal("toolchain must be downloaded before restoration") + } + + tests := readPipeline(t, "steps/run-builder-tests.yml") + test := findMapping(t, tests, "name", "test_${{ attempt }}") + provisional := mappingValue(test, "${{ if ne(attempt, 'FINAL') }}") + requireValue(t, provisional, "ignoreLASTEXITCODE", "true") + if mappingValue(test, "ignoreLASTEXITCODE") != nil { + t.Fatal("final test attempt must propagate its exit code") + } + requireValue(t, mappingValue(mappingValue(test, "${{ if parameters.builder.nosystemcrypto }}"), "env"), "MS_GO_NOSYSTEMCRYPTO", "1") + command := mappingValue(test, "pwsh").Value + if !strings.Contains(command, "run-builder -test") || strings.Contains(command, "run-builder -build") { + t.Fatal("shared test step must only run tests") + } +} + +func TestRetryTestCommand(t *testing.T) { + pwsh, err := exec.LookPath("pwsh") + if err != nil { + t.Skip("PowerShell is not installed") + } + tests := readPipeline(t, "steps/run-builder-tests.yml") + test := findMapping(t, tests, "name", "test_${{ attempt }}") + for _, tt := range []struct { + name string + attempt string + testExit string + wantExit int + }{ + {"initial pass", "1", "0", 0}, + {"initial failure is provisional", "1", "1", 0}, + {"retry pass", "FINAL", "0", 0}, + {"retry failure is fatal", "FINAL", "1", 1}, + } { + t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "eng"), 0o755); err != nil { + t.Fatal(err) + } + stub := ` +Write-Host ("ARGS:" + (($args | ForEach-Object { $_ }) -join '|')) +if ($env:RETRY_TEST_EXIT -eq '0') { + Write-Host '##vso[task.setvariable variable=TEST_BUILDER_SUCCESSFUL]true' +} +exit ([int]$env:RETRY_TEST_EXIT) +` + if err := os.WriteFile(filepath.Join(root, "eng", "run.ps1"), []byte(stub), 0o644); err != nil { + t.Fatal(err) + } + command := strings.NewReplacer( + "$(Build.SourcesDirectory)", filepath.ToSlash(root), + "${{ parameters.builder.os }}", runtime.GOOS, + "${{ parameters.builder.arch }}", runtime.GOARCH, + "${{ parameters.builder.config }}", "test", + "${{ parameters.builder.experiment }}", "systemcrypto", + "${{ parameters.builder.fips }}", "True", + "${{ attempt }}", tt.attempt, + ).Replace(mappingValue(test, "pwsh").Value) + if tt.attempt == "FINAL" { + // PowerShell@2 appends this unless ignoreLASTEXITCODE is true. + command += "\nif (Test-Path variable:LASTEXITCODE) { exit $LASTEXITCODE }\n" + } + path := filepath.Join(root, "test.ps1") + if err := os.WriteFile(path, []byte(command), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command(pwsh, "-NoLogo", "-NoProfile", "-NonInteractive", "-File", path) + cmd.Dir = root + cmd.Env = append(os.Environ(), "RETRY_TEST_EXIT="+tt.testExit) + out, err := cmd.CombinedOutput() + if cmd.ProcessState == nil || cmd.ProcessState.ExitCode() != tt.wantExit { + t.Fatalf("test step: %v\n%s; want exit %d", err, out, tt.wantExit) + } + output := string(out) + for _, want := range []string{"|run-builder|-test|", "|-experiment|systemcrypto|", "|-fipsmode|"} { + if !strings.Contains(output, want) { + t.Fatalf("missing %q in command output:\n%s", want, output) + } + } + success := strings.Contains(output, "variable=TEST_BUILDER_SUCCESSFUL]true") + if success != (tt.testExit == "0") { + t.Fatalf("incorrect test success signal:\n%s", output) + } + }) + } +} + +func TestRetryToolchainArchive(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Darwin toolchain archive scripts require Unix permissions and symlinks") + } + stage := readPipeline(t, "stages/run-stage.yml") + snapshot := findMapping(t, stage, "displayName", "Preserve toolchain for test retry") + restore := findMapping(t, stage, "displayName", "Restore toolchain for test retry") + root := t.TempDir() + original := filepath.Join(root, "original") + retry := filepath.Join(root, "retry") + staging := filepath.Join(root, "staging") + for _, dir := range []string{filepath.Join(original, "go", "bin"), filepath.Join(retry, "go")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + write := func(name, contents string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(name, []byte(contents), mode); err != nil { + t.Fatal(err) + } + } + write(filepath.Join(original, "go", "bin", "go"), "#!/bin/sh\nprintf original-toolchain", 0o755) + write(filepath.Join(original, "go", "source.go"), "original source", 0o644) + write(filepath.Join(original, "go", ".git"), "original git metadata", 0o644) + write(filepath.Join(retry, "go", ".git"), "retry git metadata", 0o644) + if err := os.Symlink("go", filepath.Join(original, "go", "bin", "tool-link")); err != nil { + t.Fatal(err) + } + run := func(dir, script string) { + t.Helper() + cmd := exec.Command("bash", "-c", script) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("archive script failed: %v\n%s", err, out) + } + } + run(original, strings.ReplaceAll(mappingValue(snapshot, "bash").Value, "$(Build.ArtifactStagingDirectory)", staging)) + write(filepath.Join(original, "go", "source.go"), "modified by tests", 0o644) + run(retry, strings.ReplaceAll(mappingValue(restore, "bash").Value, "$(Pipeline.Workspace)", staging)) + for name, want := range map[string]string{ + "source.go": "original source", + ".git": "retry git metadata", + } { + got, err := os.ReadFile(filepath.Join(retry, "go", name)) + if err != nil || string(got) != want { + t.Fatalf("restored %s = %q, %v; want %q", name, got, err, want) + } + } + cmd := exec.Command(filepath.Join(retry, "go", "bin", "tool-link")) + if out, err := cmd.CombinedOutput(); err != nil || string(out) != "original-toolchain" { + t.Fatalf("restored executable/symlink: output %q, error %v", out, err) + } +} diff --git a/eng/pipeline/stages/builders-to-stages.yml b/eng/pipeline/stages/builders-to-stages.yml index af6c55c993d..04ddce4c011 100644 --- a/eng/pipeline/stages/builders-to-stages.yml +++ b/eng/pipeline/stages/builders-to-stages.yml @@ -65,7 +65,9 @@ stages: # Attempt to retry the build on macOS to mitigate flakiness: # "read: connection reset by peer" in cmd/go tests ${{ if eq(builder.os, 'darwin') }}: - retryAttempts: [1, 2, "FINAL"] + retryAttempts: [1, "FINAL"] + ${{ if and(ne(builder.config, 'buildandpack'), not(startsWith(builder.config, 'codeql'))) }}: + retryOnFreshAgent: true - ${{ if eq(parameters.sign, true) }}: - template: pool.yml diff --git a/eng/pipeline/stages/run-stage.yml b/eng/pipeline/stages/run-stage.yml index 004565dad0f..b721e6b0a32 100644 --- a/eng/pipeline/stages/run-stage.yml +++ b/eng/pipeline/stages/run-stage.yml @@ -34,6 +34,11 @@ parameters: type: object default: ["FINAL"] + # Keep the first test attempt in the build job and use one conditional fresh-VM retry. + - name: retryOnFreshAgent + type: boolean + default: false + stages: - stage: ${{ parameters.builder.id }} # For display name, try for readability. Use some parameters set by @@ -246,70 +251,35 @@ stages: MS_GO_NOSYSTEMCRYPTO: 1 displayName: Build - # Run each test retry attempt in its own step. Benefits over a single step: - # - # - The dev can immediately see whether any retries happened - # - Easier to navigate to a specific retry attempt - # - More stable viewing experience if job is ongoing - # - Automated tools can more easily check number of retries by relying on structure - # - # Test retries get their own steps but builds don't because builds are relatively quick, - # don't produce much output, and errors are easier to reproduce. - - ${{ each attempt in parameters.retryAttempts }}: - - pwsh: | - if ($IsWindows) { - Write-Host "Removing Git usr\bin from PATH to avoid running a Linux test that would fail, 'TestScript/script_wait'..." - Write-Host $env:PATH - $env:PATH = ( - $env:PATH -split ';' | Where-Object { $_ -ne 'C:\Program Files\Git\usr\bin' } - ) -join ';' - } + - ${{ if parameters.retryOnFreshAgent }}: + # Snapshot before tests can modify the toolchain. Upload only if tests fail. + - bash: | + set -euo pipefail + mkdir -p "$(Build.ArtifactStagingDirectory)/test-retry" + tar -czf "$(Build.ArtifactStagingDirectory)/test-retry/toolchain.tar.gz" --exclude=go/.git go + displayName: Preserve toolchain for test retry - # Test publishing currently used in our CI because this process seems to cut off some test output: - # https://github.com/microsoft/go/issues/1114. - eng/run.ps1 cmdscan -envprefix GO_CMDSCAN_RULE_ -successvar TEST_BUILDER_SUCCESSFUL -- ` - pwsh eng/run.ps1 run-builder -test ` - -builder '${{ parameters.builder.os }}-${{ parameters.builder.arch }}-${{ parameters.builder.config }}' ` - $(if ('${{ parameters.builder.experiment }}') { '-experiment'; '${{ parameters.builder.experiment }}' }) ` - $(if ('${{ parameters.builder.fips }}') { '-fipsmode' }) ` - -junitout '$(Build.SourcesDirectory)/eng/artifacts/RawTestOutput/TestResults-attempt-${{ attempt }}.xml' ` - -rawtestout '$(Build.SourcesDirectory)/eng/artifacts/RawTestOutput/raw-json-attempt-${{ attempt }}.txt' - ${{ if parameters.builder.nosystemcrypto }}: - env: - MS_GO_NOSYSTEMCRYPTO: 1 - ${{ if eq(length(parameters.retryAttempts), 1) }}: - displayName: Test - ${{ else }}: - displayName: Test (🔁 ${{ attempt }}) - name: test_${{ attempt }} - # Run unless a previous retry was successful or something catastrophic happens. - # Note: test failure returns success, so it doesn't count as catastrophic. - condition: and(ne(variables['TEST_BUILDER_SUCCESSFUL'], 'true'), succeeded()) - ${{ if ne(attempt, 'FINAL') }}: - ignoreLASTEXITCODE: true - ${{ elseif eq(parameters.builder.broken, true) }}: - # Don't fail the job if we are at the last retry and the builder is marked as broken. - continueOnError: true - - # Publish this retry attempt's test results as its own test run. - - task: PublishTestResults@2 - displayName: Publish test results (${{ attempt }}) - condition: and(ne(variables['TEST_BUILDER_SUCCESSFUL'], 'true'), succeededOrFailed()) - inputs: - testResultsFormat: JUnit - testResultsFiles: $(Build.SourcesDirectory)/eng/artifacts/RawTestOutput/TestResults-attempt-${{ attempt }}.xml - testRunTitle: $(System.JobDisplayName) (retry ${{ attempt }}, stage attempt $(System.StageAttempt)) - buildPlatform: ${{ parameters.builder.arch }} - buildConfiguration: ${{ parameters.builder.config }} - publishRunAttachments: true - - # Publish raw test JSON output and the published (or attempted to publish) JUnit XML. - - task: ${{ iif(eq(variables['System.TeamProject'], 'public'), '', '1ES.') }}PublishPipelineArtifact@1 - displayName: Publish raw test results - condition: succeededOrFailed() - inputs: - targetPath: $(Build.SourcesDirectory)/eng/artifacts/RawTestOutput - artifactName: raw-test-output-${{ parameters.builder.id }}-retry-$(System.StageAttempt)-job-$(System.JobAttempt) + - template: ../steps/run-builder-tests.yml + parameters: + builder: ${{ parameters.builder }} + retryAttempts: [1] + + - task: ${{ iif(eq(variables['System.TeamProject'], 'public'), '', '1ES.') }}PublishPipelineArtifact@1 + displayName: Publish toolchain for test retry + condition: and(succeeded(), ne(variables['TEST_BUILDER_SUCCESSFUL'], 'true')) + + - pwsh: | + Write-Host "##vso[task.setvariable variable=required;isOutput=true]true" + Write-Host "##vso[task.setvariable variable=toolchainArtifact;isOutput=true]test-retry-toolchain-${{ parameters.builder.id }}-stage-$(System.StageAttempt)-job-$(System.JobAttempt)" + name: retry + displayName: Request test retry on a fresh agent + condition: and(succeeded(), ne(variables['TEST_BUILDER_SUCCESSFUL'], 'true')) + + - ${{ else }}: + - template: ../steps/run-builder-tests.yml + parameters: + builder: ${{ parameters.builder }} + retryAttempts: ${{ parameters.retryAttempts }} - ${{ if eq(parameters.builder.os, 'linux') }}: # Files may be owned by root because builds don't set user ID. If this build is running on a @@ -322,3 +292,54 @@ stages: displayName: Update file ownership from root to build agent account continueOnError: true condition: succeededOrFailed() + + - ${{ if parameters.retryOnFreshAgent }}: + - job: ${{ parameters.builder.id }}_test_retry + displayName: ${{ parameters.builder.os }}-${{ parameters.builder.arch }} ${{ parameters.builder.hostParens}} ${{ parameters.builder.config }} ${{ parameters.builder.distroParens}} ${{ parameters.builder.experimentBrackets }} ${{ parameters.builder.nosystemcryptoBrackets }} ${{ parameters.builder.fipsAcronym }} (test retry) + dependsOn: ${{ parameters.builder.id }} + condition: and(succeeded(), eq(dependencies.${{ parameters.builder.id }}.outputs['retry.required'], 'true')) + pool: ${{ parameters.pool }} + workspace: + clean: all + ${{ if eq(parameters.builder.config, 'longtest') }}: + timeoutInMinutes: 240 + ${{ else }}: + timeoutInMinutes: 120 + variables: + - group: go-cmdscan-rules + - name: retryToolchainArtifact + value: $[ dependencies.${{ parameters.builder.id }}.outputs['retry.toolchainArtifact'] ] + steps: + - pwsh: | + . eng/utilities.ps1 + Download-Stage0 + displayName: Init stage 0 Go toolset + + - template: ../steps/init-submodule-task.yml + + - pwsh: eng/run.ps1 submodule-refresh + displayName: Apply patches + + - ${{ if eq(parameters.builder.config, 'longtest') }}: + - template: ../steps/set-bot-git-author-task.yml + + - ${{ if ne(parameters.builder.hostArch, parameters.builder.arch) }}: + - pwsh: Write-Host "##vso[task.setvariable variable=GOARCH]${{ parameters.builder.arch }}" + displayName: Set GOARCH for cross-compile + + - task: DownloadPipelineArtifact@2 + displayName: Download toolchain for test retry + inputs: + artifactName: $(retryToolchainArtifact) + targetPath: $(Pipeline.Workspace)/test-retry + + - bash: | + set -euo pipefail + tar -xzf "$(Pipeline.Workspace)/test-retry/toolchain.tar.gz" + displayName: Restore toolchain for test retry + + - template: ../steps/run-builder-tests.yml + parameters: + builder: ${{ parameters.builder }} + retryAttempts: ["FINAL"] + artifactSuffix: -test-retry diff --git a/eng/pipeline/steps/run-builder-tests.yml b/eng/pipeline/steps/run-builder-tests.yml new file mode 100644 index 00000000000..9575ac68494 --- /dev/null +++ b/eng/pipeline/steps/run-builder-tests.yml @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +parameters: + - name: builder + type: object + - name: retryAttempts + type: object + - name: artifactSuffix + type: string + default: '' + +steps: + - ${{ each attempt in parameters.retryAttempts }}: + - pwsh: | + New-Item '$(Build.SourcesDirectory)/eng/artifacts/RawTestOutput' -ItemType Directory -Force | Out-Null + + if ($IsWindows) { + Write-Host "Removing Git usr\bin from PATH to avoid running a Linux test that would fail, 'TestScript/script_wait'..." + Write-Host $env:PATH + $env:PATH = ( + $env:PATH -split ';' | Where-Object { $_ -ne 'C:\Program Files\Git\usr\bin' } + ) -join ';' + } + + # Test publishing currently used in our CI because this process seems to cut off some test output: + # https://github.com/microsoft/go/issues/1114. + eng/run.ps1 cmdscan -envprefix GO_CMDSCAN_RULE_ -successvar TEST_BUILDER_SUCCESSFUL -- ` + pwsh eng/run.ps1 run-builder -test ` + -builder '${{ parameters.builder.os }}-${{ parameters.builder.arch }}-${{ parameters.builder.config }}' ` + $(if ('${{ parameters.builder.experiment }}') { '-experiment'; '${{ parameters.builder.experiment }}' }) ` + $(if ('${{ parameters.builder.fips }}') { '-fipsmode' }) ` + -junitout '$(Build.SourcesDirectory)/eng/artifacts/RawTestOutput/TestResults-attempt-${{ attempt }}.xml' ` + -rawtestout '$(Build.SourcesDirectory)/eng/artifacts/RawTestOutput/raw-json-attempt-${{ attempt }}.txt' + ${{ if parameters.builder.nosystemcrypto }}: + env: + MS_GO_NOSYSTEMCRYPTO: 1 + ${{ if eq(length(parameters.retryAttempts), 1) }}: + displayName: Test + ${{ else }}: + displayName: Test (🔁 ${{ attempt }}) + name: test_${{ attempt }} + condition: and(ne(variables['TEST_BUILDER_SUCCESSFUL'], 'true'), succeeded()) + ${{ if ne(attempt, 'FINAL') }}: + ignoreLASTEXITCODE: true + ${{ elseif eq(parameters.builder.broken, true) }}: + continueOnError: true + + - task: PublishTestResults@2 + displayName: Publish test results (${{ attempt }}) + condition: and(ne(variables['TEST_BUILDER_SUCCESSFUL'], 'true'), succeededOrFailed()) + ${{ if or(ne(attempt, 'FINAL'), eq(parameters.builder.broken, true)) }}: + continueOnError: true + inputs: + testResultsFormat: JUnit + testResultsFiles: $(Build.SourcesDirectory)/eng/artifacts/RawTestOutput/TestResults-attempt-${{ attempt }}.xml + testRunTitle: $(System.JobDisplayName) (retry ${{ attempt }}, stage attempt $(System.StageAttempt)) + buildPlatform: ${{ parameters.builder.arch }} + buildConfiguration: ${{ parameters.builder.config }} + publishRunAttachments: true + + - task: ${{ iif(eq(variables['System.TeamProject'], 'public'), '', '1ES.') }}PublishPipelineArtifact@1 + displayName: Publish raw test results + condition: succeededOrFailed() + inputs: + targetPath: $(Build.SourcesDirectory)/eng/artifacts/RawTestOutput + artifactName: raw-test-output-${{ parameters.builder.id }}-retry-$(System.StageAttempt)-job-$(System.JobAttempt)${{ parameters.artifactSuffix }}