From 0e5c3e4213461fb0e85109b950e88098743ccf86 Mon Sep 17 00:00:00 2001 From: Lionello Lunesu Date: Thu, 9 Jul 2026 11:53:42 -0700 Subject: [PATCH] fix: detect CD image pull failures early --- src/pkg/cli/client/byoc/azure/byoc.go | 166 +++++++++++--------- src/pkg/cli/client/byoc/azure/byoc_test.go | 26 ++- src/pkg/clouds/aws/codebuild/run.go | 18 +++ src/pkg/clouds/aws/codebuild/run_test.go | 38 +++++ src/pkg/clouds/aws/codebuild/status.go | 29 ++++ src/pkg/clouds/aws/codebuild/status_test.go | 75 +++++++++ src/pkg/clouds/azure/aca/job.go | 76 ++++++--- src/pkg/clouds/azure/aca/job_test.go | 17 ++ 8 files changed, 351 insertions(+), 94 deletions(-) diff --git a/src/pkg/cli/client/byoc/azure/byoc.go b/src/pkg/cli/client/byoc/azure/byoc.go index 7087624ce..62b729e03 100644 --- a/src/pkg/cli/client/byoc/azure/byoc.go +++ b/src/pkg/cli/client/byoc/azure/byoc.go @@ -26,6 +26,7 @@ import ( azuredns "github.com/DefangLabs/defang/src/pkg/clouds/azure/dns" "github.com/DefangLabs/defang/src/pkg/clouds/azure/keyvault" defanghttp "github.com/DefangLabs/defang/src/pkg/http" + "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/term" "github.com/DefangLabs/defang/src/pkg/tokenstore" "github.com/DefangLabs/defang/src/pkg/types" @@ -809,103 +810,122 @@ func (b *ByocAzure) QueryLogs(ctx context.Context, req *defangv1.TailRequest) (i return nil, err } - // Resolve the CD job execution for this request. The deploying process caches - // the run ID in memory; a standalone `defang logs` has none, so recover it by - // matching the request etag against each execution's recorded ETAG env var. - runID := b.cdRunID + // Honor the requested log types (a bitmask of CD/RUN/BUILD), like the other + // providers do. In particular `cd down`/refresh request CD logs only, so the + // service and build watchers below must not run against the project resource + // group — it is being (or already) torn down and would return ResourceGroupNotFound. + logType := logs.LogType(req.LogType) + + // The etag labels CD, service, and build entries alike, so resolve it before + // branching on log type. etag := b.cdEtag if req.Etag != "" && req.Etag != b.cdEtag { - runID, etag = "", req.Etag - } - if runID == "" && etag != "" { - found, err := b.job.FindExecutionByEtag(ctx, etag) - if err != nil { - return nil, fmt.Errorf("failed to find CD deployment for etag %q: %w", etag, err) - } - runID = found - } - if runID == "" { - // Unknown or empty etag: no CD run to tail. Service and build logs are keyed - // by resource group rather than the CD execution, so still surface those. - term.Warnf("No CD logs found for etag %q; showing service and build logs only", req.Etag) + etag = req.Etag } - // CD logs. cdCh stays nil when there's no CD run, which makes the select below skip - // it and treat the CD source as already drained. Follow streams line-by-line; the - // snapshot reads the whole buffered run content once. + // CD logs. cdCh stays nil when CD logs aren't requested or there's no CD run, + // which makes the select below skip it and treat the CD source as already + // drained. Follow streams line-by-line; the snapshot reads the whole buffered + // run content once. type cdLogEntry struct { line string err error } var cdCh chan cdLogEntry - if runID != "" { - cdCh = make(chan cdLogEntry) - if req.Follow { - logIter, err := b.job.TailJobLogs(ctx, runID) + if logType.Has(logs.LogTypeCD) { + // Resolve the CD job execution for this request. The deploying process caches + // the run ID in memory; a standalone `defang logs` has none, so recover it by + // matching the request etag against each execution's recorded ETAG env var. + runID := b.cdRunID + if req.Etag != "" && req.Etag != b.cdEtag { + runID = "" + } + if runID == "" && etag != "" { + found, err := b.job.FindExecutionByEtag(ctx, etag) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to find CD deployment for etag %q: %w", etag, err) } - go func() { - defer close(cdCh) - for line, err := range logIter { + runID = found + } + if runID == "" { + term.Warnf("No CD logs found for etag %q", req.Etag) + } else { + cdCh = make(chan cdLogEntry) + if req.Follow { + logIter, err := b.job.TailJobLogs(ctx, runID) + if err != nil { + return nil, err + } + go func() { + defer close(cdCh) + for line, err := range logIter { + select { + case cdCh <- cdLogEntry{line: line, err: err}: + case <-ctx.Done(): + return + } + } + }() + } else { + content, err := b.job.ReadJobLogs(ctx, runID) + if err != nil { + return nil, err + } + go func() { + defer close(cdCh) + if content == "" { + return + } select { - case cdCh <- cdLogEntry{line: line, err: err}: + case cdCh <- cdLogEntry{line: content}: case <-ctx.Done(): - return } - } - }() - } else { - content, err := b.job.ReadJobLogs(ctx, runID) - if err != nil { - return nil, err + }() } - go func() { - defer close(cdCh) - if content == "" { - return - } - select { - case cdCh <- cdLogEntry{line: content}: - case <-ctx.Done(): - } - }() } } projectRG := b.projectResourceGroupName(req.Project) - // Service logs from the project's Container Apps and build logs from ACR both - // live in the PROJECT resource group (independent of the CD run). For a one-shot - // query, if that group doesn't exist the project isn't deployed under this - // name/stack — surface one clear message instead of letting both watchers fail - // with raw "ResourceGroupNotFound" errors. In follow mode we skip the check: the - // group may be created mid-session (deploy-then-tail), so the watchers poll for it. + // Service logs from the project's Container Apps (RUN) and build logs from ACR + // (BUILD) both live in the PROJECT resource group (independent of the CD run). + // For a one-shot query, if that group doesn't exist the project isn't deployed + // under this name/stack — surface one clear message instead of letting the + // watchers fail with raw "ResourceGroupNotFound" errors. In follow mode we skip + // the check: the group may be created mid-session (deploy-then-tail), so the + // watchers poll for it. var acaCh <-chan aca.ServiceLogEntry var buildCh <-chan acr.BuildLogEntry - startWatchers := true - if !req.Follow { - exists, err := b.driver.ResourceGroupExists(ctx, projectRG) - if err != nil { - return nil, err - } - if !exists { - term.Warnf("No deployed services found for project %q on stack %q (resource group %q not found)", req.Project, b.PulumiStack, projectRG) - startWatchers = false - } - } - if startWatchers { - acaClient := &aca.ContainerApp{ - Azure: b.driver.Azure, - ResourceGroup: projectRG, + wantRun := logType.Has(logs.LogTypeRun) + wantBuild := logType.Has(logs.LogTypeBuild) + if wantRun || wantBuild { + startWatchers := true + if !req.Follow { + exists, err := b.driver.ResourceGroupExists(ctx, projectRG) + if err != nil { + return nil, err + } + if !exists { + term.Warnf("No deployed services found for project %q on stack %q (resource group %q not found)", req.Project, b.PulumiStack, projectRG) + startWatchers = false + } } - acaCh = acaClient.WatchLogs(ctx, req.Follow) - - buildWatcher := &acr.BuildLogWatcher{ - Azure: b.driver.Azure, - ResourceGroup: projectRG, + if startWatchers { + if wantRun { + acaClient := &aca.ContainerApp{ + Azure: b.driver.Azure, + ResourceGroup: projectRG, + } + acaCh = acaClient.WatchLogs(ctx, req.Follow) + } + if wantBuild { + buildWatcher := &acr.BuildLogWatcher{ + Azure: b.driver.Azure, + ResourceGroup: projectRG, + } + buildCh = buildWatcher.WatchBuildLogs(ctx, req.Follow) + } } - buildCh = buildWatcher.WatchBuildLogs(ctx, req.Follow) } return func(yield func(*defangv1.TailResponse, error) bool) { diff --git a/src/pkg/cli/client/byoc/azure/byoc_test.go b/src/pkg/cli/client/byoc/azure/byoc_test.go index 10949bcbf..6e2e6cd02 100644 --- a/src/pkg/cli/client/byoc/azure/byoc_test.go +++ b/src/pkg/cli/client/byoc/azure/byoc_test.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/DefangLabs/defang/src/pkg/cli/client" cloudazure "github.com/DefangLabs/defang/src/pkg/clouds/azure" + "github.com/DefangLabs/defang/src/pkg/logs" defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1" composeTypes "github.com/compose-spec/compose-go/v2/types" ) @@ -267,7 +268,7 @@ func TestQueryLogsDiscoversRunByEtag(t *testing.T) { b := newTestProvider(t, cloudazure.LocationEastUS, "sub") ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - _, err := b.QueryLogs(ctx, &defangv1.TailRequest{Etag: "some-etag"}) + _, err := b.QueryLogs(ctx, &defangv1.TailRequest{Etag: "some-etag", LogType: uint32(logs.LogTypeCD)}) if err == nil || !strings.Contains(err.Error(), "failed to find CD deployment for etag") { t.Errorf("expected etag-lookup failure, got: %v", err) } @@ -282,12 +283,31 @@ func TestQueryLogsEtagMismatchTriggersLookup(t *testing.T) { b.cdEtag = "etag-A" ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - _, err := b.QueryLogs(ctx, &defangv1.TailRequest{Etag: "etag-B"}) + _, err := b.QueryLogs(ctx, &defangv1.TailRequest{Etag: "etag-B", LogType: uint32(logs.LogTypeCD)}) if err == nil || !strings.Contains(err.Error(), "failed to find CD deployment for etag") { t.Errorf("expected mismatched-etag lookup failure, got: %v", err) } } +func TestQueryLogsCDOnlySkipsProjectResourceGroup(t *testing.T) { + // `cd down`/refresh tail only CD logs. With no CD run to tail, QueryLogs must + // not start the service/build watchers nor probe the project resource group — + // which is being torn down and would fail with ResourceGroupNotFound. A working + // credential is provided so any stray project-RG call would reach Azure (and + // hang/fail) instead of short-circuiting on missing creds. + useFakeCred(t, "tok", nil) + b := newTestProvider(t, cloudazure.LocationEastUS, "sub") + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + it, err := b.QueryLogs(ctx, &defangv1.TailRequest{Project: "proj", Follow: false, LogType: uint32(logs.LogTypeCD)}) + if err != nil { + t.Fatalf("QueryLogs returned error, want none: %v", err) + } + for _, err := range it { + t.Fatalf("expected no log entries for CD-only request without a run, got err=%v", err) + } +} + func TestAuthenticateNonInteractiveFailsWithoutCreds(t *testing.T) { // Point the SDK at an ARM endpoint that returns 401 so DefaultAzureCredential's // token always fails validation — no real Azure call is made by our code beyond @@ -415,7 +435,7 @@ func TestQueryLogsNonFollow(t *testing.T) { // an error (not panic). ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - _, err := b.QueryLogs(ctx, &defangv1.TailRequest{Etag: "etag", Follow: false}) + _, err := b.QueryLogs(ctx, &defangv1.TailRequest{Etag: "etag", Follow: false, LogType: uint32(logs.LogTypeCD)}) if err == nil { t.Error("QueryLogs non-follow should fail without real Azure workspace") } diff --git a/src/pkg/clouds/aws/codebuild/run.go b/src/pkg/clouds/aws/codebuild/run.go index a18eefcd0..37efd5fdc 100644 --- a/src/pkg/clouds/aws/codebuild/run.go +++ b/src/pkg/clouds/aws/codebuild/run.go @@ -26,6 +26,23 @@ type buildspecBuild struct { Commands []string `yaml:"commands"` } +// environmentTypeForImage returns the CodeBuild environment type required to run +// the given container image. The project is created as LINUX_CONTAINER (x86_64), +// but some CD images are arm64 (e.g. the legacy public-cd-image-*-arm64 tag). +// Running an arm64 image on an x86_64 environment fails at runtime with +// "node: not found" / exit 127, so the environment type must match the image. +// +// The image architecture is inferred from the reference: a sha256 digest is +// hexadecimal and cannot contain the letters in "arm64"/"aarch64", so a simple +// substring check is unambiguous. Unknown/x86_64 images keep the project default. +func environmentTypeForImage(image string) cbtypes.EnvironmentType { + lower := strings.ToLower(image) + if strings.Contains(lower, "arm64") || strings.Contains(lower, "aarch64") { + return cbtypes.EnvironmentTypeArmContainer + } + return cbtypes.EnvironmentTypeLinuxContainer +} + func buildspec(workingDir string, cmd ...string) (string, error) { if workingDir == "" { return "", errors.New("workingDir must not be empty") @@ -82,6 +99,7 @@ func (a *AwsCodeBuild) Run(ctx context.Context, workingDir, image string, env ma input := &codebuild.StartBuildInput{ ProjectName: ptr.String(a.ProjectName), ImageOverride: ptr.String(image), + EnvironmentTypeOverride: environmentTypeForImage(image), EnvironmentVariablesOverride: envOverrides, BuildspecOverride: ptr.String(spec), } diff --git a/src/pkg/clouds/aws/codebuild/run_test.go b/src/pkg/clouds/aws/codebuild/run_test.go index 1c88c00b1..411c31a77 100644 --- a/src/pkg/clouds/aws/codebuild/run_test.go +++ b/src/pkg/clouds/aws/codebuild/run_test.go @@ -4,9 +4,47 @@ import ( "strings" "testing" + cbtypes "github.com/aws/aws-sdk-go-v2/service/codebuild/types" "go.yaml.in/yaml/v4" ) +func TestEnvironmentTypeForImage(t *testing.T) { + tests := []struct { + name string + image string + want cbtypes.EnvironmentType + }{ + { + name: "arm64 public CD image with digest", + image: "public.ecr.aws/defang-io/cd:public-cd-image-fb20b70e-arm64@sha256:6cdb3f11e548700a673098642ed25e3fb50ebf3b39d533364f75d207bf66ea9b", + want: cbtypes.EnvironmentTypeArmContainer, + }, + { + name: "x86_64 private CD image with digest", + image: "426819183542.dkr.ecr.us-west-2.amazonaws.com/cd:nodejs-cd-image-e87eefcf-x86_64@sha256:7efe4a59ed9d1c06f4bf396d76dbfb0f1d4784c289999bf88664fabe43409793", + want: cbtypes.EnvironmentTypeLinuxContainer, + }, + { + name: "aarch64 spelling", + image: "public.ecr.aws/defang-io/cd:some-tag-aarch64", + want: cbtypes.EnvironmentTypeArmContainer, + }, + { + name: "unknown arch defaults to linux", + image: "aws/codebuild/amazonlinux2-x86_64-standard:5.0", + want: cbtypes.EnvironmentTypeLinuxContainer, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := environmentTypeForImage(tt.image); got != tt.want { + t.Errorf("environmentTypeForImage(%q) = %q, want %q", tt.image, got, tt.want) + } + }) + } +} + func TestBuildspec(t *testing.T) { tests := []struct { name string diff --git a/src/pkg/clouds/aws/codebuild/status.go b/src/pkg/clouds/aws/codebuild/status.go index ee571a1f5..aa4fd601e 100644 --- a/src/pkg/clouds/aws/codebuild/status.go +++ b/src/pkg/clouds/aws/codebuild/status.go @@ -2,6 +2,7 @@ package codebuild import ( "context" + "fmt" "io" "strings" "time" @@ -32,6 +33,14 @@ func GetBuildStatus(ctx context.Context, cfg aws.Config, buildID BuildID) (bool, func buildStatus(build cbtypes.Build) (bool, error) { switch build.BuildStatus { case cbtypes.StatusTypeInProgress: + // The top-level BuildStatus can lag a phase that has already failed: when + // the CodeBuild agent faults before our command runs (e.g. it cannot exec + // the shell), the build can sit at IN_PROGRESS until the project timeout + // (up to an hour) even though it is already doomed. Phases are sequential + // and never recover, so a failed phase means the build is done. + if err := failedPhaseError(build); err != nil { + return true, err + } return false, nil case cbtypes.StatusTypeSucceeded: return true, io.EOF @@ -45,6 +54,26 @@ func buildStatus(build cbtypes.Build) (bool, error) { } } +// failedPhaseError returns a BuildFailure when any build phase has reached a +// terminal failure state, or nil when none has. It lets us surface a doomed +// build while its top-level BuildStatus still reads IN_PROGRESS, rather than +// waiting for CodeBuild to finalize (or time out). +func failedPhaseError(build cbtypes.Build) error { + for _, phase := range build.Phases { + switch phase.PhaseStatus { + case cbtypes.StatusTypeFailed, cbtypes.StatusTypeFault, + cbtypes.StatusTypeTimedOut, cbtypes.StatusTypeStopped: + reason := getBuildPhaseErrorContexts(build) + if reason == "" { + // No context message (e.g. an agent fault): fall back to naming the phase. + reason = fmt.Sprintf("build %s phase %s", phase.PhaseType, strings.ToLower(string(phase.PhaseStatus))) + } + return BuildFailure{Reason: reason} + } + } + return nil +} + func getBuildPhaseErrorContexts(build cbtypes.Build) string { var messages []string for _, phase := range build.Phases { diff --git a/src/pkg/clouds/aws/codebuild/status_test.go b/src/pkg/clouds/aws/codebuild/status_test.go index dc34e5545..de2a74547 100644 --- a/src/pkg/clouds/aws/codebuild/status_test.go +++ b/src/pkg/clouds/aws/codebuild/status_test.go @@ -131,6 +131,81 @@ func TestBuildStatus_InProgress(t *testing.T) { } } +func TestBuildStatus_InProgressWithFailedPhase(t *testing.T) { + // The CodeBuild agent faulted mid-build but the top-level BuildStatus still + // reads IN_PROGRESS. We must report the build as done+failed so the tail + // doesn't hang until the project timeout. + build := cbtypes.Build{ + BuildStatus: cbtypes.StatusTypeInProgress, + Phases: []cbtypes.BuildPhase{ + {PhaseType: cbtypes.BuildPhaseTypeDownloadSource, PhaseStatus: cbtypes.StatusTypeSucceeded}, + { + PhaseType: cbtypes.BuildPhaseTypeBuild, + PhaseStatus: cbtypes.StatusTypeFailed, + Contexts: []cbtypes.PhaseContext{ + {StatusCode: aws.String("COMMAND_EXECUTION_ERROR"), Message: aws.String("fork/exec /bin/bash: no such file or directory")}, + }, + }, + {PhaseType: cbtypes.BuildPhaseTypeCompleted}, + }, + } + + done, err := buildStatus(build) + if !done { + t.Error("expected done=true for in-progress build with a failed phase") + } + bf, ok := err.(BuildFailure) + if !ok { + t.Fatalf("expected BuildFailure, got %T", err) + } + if bf.Reason != "fork/exec /bin/bash: no such file or directory" { + t.Errorf("reason = %q, want the phase context message", bf.Reason) + } +} + +func TestBuildStatus_InProgressWithFaultedPhaseNoContext(t *testing.T) { + // An agent fault may carry no context message; fall back to naming the phase + // rather than returning an empty reason. + build := cbtypes.Build{ + BuildStatus: cbtypes.StatusTypeInProgress, + Phases: []cbtypes.BuildPhase{ + {PhaseType: cbtypes.BuildPhaseTypeProvisioning, PhaseStatus: cbtypes.StatusTypeFault}, + {PhaseType: cbtypes.BuildPhaseTypeCompleted}, + }, + } + + done, err := buildStatus(build) + if !done { + t.Error("expected done=true for in-progress build with a faulted phase") + } + bf, ok := err.(BuildFailure) + if !ok { + t.Fatalf("expected BuildFailure, got %T", err) + } + if bf.Reason != "build PROVISIONING phase fault" { + t.Errorf("reason = %q, want the phase-name fallback", bf.Reason) + } +} + +func TestBuildStatus_InProgressHealthy(t *testing.T) { + // A normal in-progress build (no failed phase) must keep waiting. + build := cbtypes.Build{ + BuildStatus: cbtypes.StatusTypeInProgress, + Phases: []cbtypes.BuildPhase{ + {PhaseType: cbtypes.BuildPhaseTypeDownloadSource, PhaseStatus: cbtypes.StatusTypeSucceeded}, + {PhaseType: cbtypes.BuildPhaseTypeBuild, PhaseStatus: cbtypes.StatusTypeInProgress}, + }, + } + + done, err := buildStatus(build) + if done { + t.Error("expected done=false for a healthy in-progress build") + } + if err != nil { + t.Errorf("expected nil error, got %v", err) + } +} + func TestBuildStatus_TimedOut(t *testing.T) { done, err := buildStatus(cbtypes.Build{BuildStatus: cbtypes.StatusTypeTimedOut}) if !done { diff --git a/src/pkg/clouds/azure/aca/job.go b/src/pkg/clouds/azure/aca/job.go index 31a0b2cb0..18d9b386c 100644 --- a/src/pkg/clouds/azure/aca/job.go +++ b/src/pkg/clouds/azure/aca/job.go @@ -822,35 +822,75 @@ func (j *Job) getCDContainerLogStreamURL(ctx context.Context, executionName stri } // cdReplicaStatus derives a terminal JobStatus from the execution's replica -// container when it has terminated, or returns nil when the container is still -// running or not present yet. The execution-level Status (JobsExecutions list) -// can lag the container indefinitely, leaving the CLI polling forever; the -// container's own Terminated state plus its exit code is the timely signal. +// container, or returns nil when the container is still legitimately starting or +// not present yet. The execution-level Status (JobsExecutions list) can lag the +// container indefinitely, leaving the CLI polling forever; the container's own +// state is the timely signal. Two terminal cases are recognized: +// - Terminated: the container ran and exited (classified by exit code). +// - Waiting on a permanent error (image pull backoff, crash loop, etc.): the +// container will never reach Running, so report it as failed now instead of +// hanging until the job's timeout. Transient waiting (pulling the image, +// container creating) returns nil so healthy startups aren't misreported. func (j *Job) cdReplicaStatus(ctx context.Context, executionName string) (*JobStatus, error) { containers, err := j.listCDReplicaContainers(ctx, executionName) if err != nil { return nil, err } for _, c := range containers { - if c.RunningState != "Terminated" { - continue - } - status := &JobStatus{ExecutionName: executionName} - // Classify by the container's exit code. A clean success normally - // surfaces via the execution-level Status, so an exit code we cannot - // parse here is treated as a failure and surfaced with its details - // rather than being silently reported as success. - if code, ok := parseExitCode(c.RunningStateDetails); ok && code == 0 { - status.Status = armappcontainersv3.JobExecutionRunningStateSucceeded - } else { - status.Status = armappcontainersv3.JobExecutionRunningStateFailed - status.ErrorMessage = c.RunningStateDetails + switch c.RunningState { + case "Terminated": + status := &JobStatus{ExecutionName: executionName} + // Classify by the container's exit code. A clean success normally + // surfaces via the execution-level Status, so an exit code we cannot + // parse here is treated as a failure and surfaced with its details + // rather than being silently reported as success. + if code, ok := parseExitCode(c.RunningStateDetails); ok && code == 0 { + status.Status = armappcontainersv3.JobExecutionRunningStateSucceeded + } else { + status.Status = armappcontainersv3.JobExecutionRunningStateFailed + status.ErrorMessage = c.RunningStateDetails + } + return status, nil + case "Waiting": + if isTerminalWaitingFailure(c.RunningStateDetails) { + return &JobStatus{ + ExecutionName: executionName, + Status: armappcontainersv3.JobExecutionRunningStateFailed, + ErrorMessage: c.RunningStateDetails, + }, nil + } } - return status, nil } return nil, nil } +// waitingFailureMarkers are substrings that appear in a Waiting container's +// runningStateDetails when it has hit a permanent startup failure and will never +// reach Running. Matched case-insensitively. Transient reasons (ContainerCreating, +// PodInitializing, plain "pulling image") are deliberately excluded. +var waitingFailureMarkers = []string{ + "ImagePullBackOff", + "ErrImagePull", + "Back-off pulling image", + "CrashLoopBackOff", + "CreateContainerError", + "CreateContainerConfigError", + "InvalidImageName", + "ErrImageNeverPull", +} + +// isTerminalWaitingFailure reports whether a Waiting container's details indicate +// a permanent failure rather than normal in-progress startup. +func isTerminalWaitingFailure(details string) bool { + lower := strings.ToLower(details) + for _, m := range waitingFailureMarkers { + if strings.Contains(lower, strings.ToLower(m)) { + return true + } + } + return false +} + // exitCodeRe extracts the numeric exit code from a replica container's free-form // runningStateDetails (e.g. "Container 'defang-cd' was terminated with exit code '1'"). var exitCodeRe = regexp.MustCompile(`(?i)exit code:?\s*'?(-?\d+)'?`) diff --git a/src/pkg/clouds/azure/aca/job_test.go b/src/pkg/clouds/azure/aca/job_test.go index 6f5f85823..f2bb2761c 100644 --- a/src/pkg/clouds/azure/aca/job_test.go +++ b/src/pkg/clouds/azure/aca/job_test.go @@ -355,6 +355,23 @@ func TestCDReplicaStatus(t *testing.T) { wantSuccess: false, wantMsg: "OOMKilled", }, + { + name: "waiting while pulling the image yields no terminal status", + resp: `{"value":[{"properties":{"containers":[{"name":"defang-cd","runningState":"Waiting","runningStateDetails":"ContainerCreating"}]}}]}`, + wantNil: true, + }, + { + name: "waiting on image pull backoff is a failure", + resp: `{"value":[{"properties":{"containers":[{"name":"defang-cd","runningState":"Waiting","runningStateDetails":"Back-off pulling image \"defang-cd\": ImagePullBackOff"}]}}]}`, + wantSuccess: false, + wantMsg: `Back-off pulling image "defang-cd": ImagePullBackOff`, + }, + { + name: "waiting on crash loop is a failure", + resp: `{"value":[{"properties":{"containers":[{"name":"defang-cd","runningState":"Waiting","runningStateDetails":"CrashLoopBackOff"}]}}]}`, + wantSuccess: false, + wantMsg: "CrashLoopBackOff", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {