diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 16ce710..f28423c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -150,6 +150,22 @@ jobs: --push \ . + - name: Smoke-check pg_dump is on PATH in the built image + # P0 regression gate (incident 2026-05-30): customer_backup_runner.go + # shells out to `pg_dump`. The previous distroless base had no + # pg_dump and every Pro+ scheduled backup failed silently with + # `executable file not found in $PATH`. Hard-fail the deploy if + # the binary regresses, and assert the major version is >= the + # customer-pg server major (currently postgres:16-alpine). + run: | + IMAGE="${IMAGE_REPO}:${{ steps.meta.outputs.version }}" + echo "Probing ${IMAGE} for pg_dump" + docker pull "${IMAGE}" + PG_VER=$(docker run --rm --entrypoint sh "${IMAGE}" -c 'pg_dump --version') + echo "pg_dump: ${PG_VER}" + echo "${PG_VER}" | grep -qE '^pg_dump \(PostgreSQL\) 1[6-9]' \ + || { echo "::error::pg_dump missing or wrong major version (need >=16): ${PG_VER}"; exit 1; } + - name: Set up kubectl uses: azure/setup-kubectl@v5 with: diff --git a/Dockerfile b/Dockerfile index bb2af79..0101cdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,21 @@ RUN CGO_ENABLED=0 go build \ -ldflags "-X instant.dev/common/buildinfo.GitSHA=${GIT_SHA} -X instant.dev/common/buildinfo.BuildTime=${BUILD_TIME} -X instant.dev/common/buildinfo.Version=${VERSION}" \ -o /worker . -FROM gcr.io/distroless/static-debian12 +# Runtime: postgres:16-alpine ships pg_dump matching the customer-pg server +# version (infra/k8s/postgres-customers.yaml uses postgres:16-alpine). The +# previous distroless/static-debian12 base had NO shell, NO package manager, +# and NO pg_dump — which silently broke customer_backup_runner.go's +# `exec.CommandContext(ctx, "pg_dump", ...)` for every Pro+ tier customer's +# scheduled backup (data-loss risk: P0 incident 2026-05-30). +# +# Our worker binary is built with CGO_ENABLED=0 so it's fully static and +# runs unmodified on alpine. Alpine also gives us sh+wget+curl for the +# in-pod /healthz SHA check at the end of deploy.yml (was a warning-only +# fallback on distroless; now a hard gate). +# +# When the customer-pg image bumps to postgres:17-alpine, bump this tag in +# the same PR — pg_dump major version must be >= server major version, and +# matching exactly keeps the dump format ABI predictable. +FROM postgres:16-alpine COPY --from=builder /worker /worker ENTRYPOINT ["/worker"] diff --git a/internal/jobs/deploy_probe.go b/internal/jobs/deploy_probe.go index 44f0377..f7bc328 100644 --- a/internal/jobs/deploy_probe.go +++ b/internal/jobs/deploy_probe.go @@ -129,19 +129,38 @@ const ( // deployProbeResult* are the outcome enum values emitted as the `result` // label. // -// pass — leg met all assertions inside its latency budget. -// fail — leg failed an assertion (wrong status, build timeout, -// 5xx from the serving URL). Triggers audit_log row + -// ERROR slog line + NR alert. -// degraded — leg passed assertions but crossed a soft threshold OR -// is configured-off (e.g. probe bearer missing). Tracked -// separately so a slow-but-working leg doesn't page. +// pass — leg met all assertions inside its latency budget. +// fail — leg failed an assertion (wrong status, build timeout, +// 5xx from the serving URL). Triggers audit_log row + +// ERROR slog line + NR alert. +// degraded — leg passed assertions but crossed a soft threshold OR +// is configured-off (e.g. probe bearer missing). Tracked +// separately so a slow-but-working leg doesn't page. +// bootstrap — leg-1-only: the first tick of a probe's lifetime saw a +// canonical 404 `no_existing_deployment_to_redeploy` from +// /deploy/new (the probe-app row doesn't exist yet), then +// transparently retried without `redeploy=true` and that +// second call succeeded. Distinct from `pass` so the +// dashboard surfaces "we self-healed once" as its own +// event. Subsequent ticks should report `pass`, not +// `bootstrap`. A `bootstrap` outcome does NOT page (it +// is the prober working as designed). const ( - deployProbeResultPass = "pass" - deployProbeResultFail = "fail" - deployProbeResultDegraded = "degraded" + deployProbeResultPass = "pass" + deployProbeResultFail = "fail" + deployProbeResultDegraded = "degraded" + deployProbeResultBootstrap = "bootstrap" ) +// deployProbeRedeployMissingCode is the api's canonical error_code +// (api/internal/handlers/deploy.go) returned with HTTP 404 when +// /deploy/new is called with redeploy=true and no matching app row +// exists for (team, env, name). The probe relies on this EXACT string +// to distinguish "first tick, need to bootstrap" from any other 404 +// (auth, routing). Locked in by api#206 as the typed-error contract; +// changing it on the api side must change this constant in lockstep. +const deployProbeRedeployMissingCode = "no_existing_deployment_to_redeploy" + // deployProbeStatusBudget is the leg-2 budget — wall-clock time the api // has to flip the row from `building` to `healthy`. 90s comfortably // exceeds the observed end-to-end k8s build for the minimal nginx image @@ -370,11 +389,14 @@ func (w *DeployProbeWorker) Work(ctx context.Context, job *river.Job[DeployProbe w.recordLeg(ctx, deployProbeLegSubmit, submitRes) var statusRes, serveRes deployProbeLegResult - if submitRes.result != deployProbeResultPass { + // Both `pass` and `bootstrap` mean leg-1 produced a usable app_id — + // downstream legs must run on the freshly-bootstrapped row so the + // next tick has somewhere to redeploy into. + if submitRes.result != deployProbeResultPass && submitRes.result != deployProbeResultBootstrap { // Leg-1 didn't produce a usable app_id — short-circuit the - // downstream legs. result=skipped is its own enum value so the - // dashboard distinguishes "we didn't try" from "we tried and - // failed". + // downstream legs. result=degraded with a "skipped" reason so + // the dashboard distinguishes "we didn't try" from "we tried + // and failed". statusRes = deployProbeLegResult{ result: deployProbeResultDegraded, reason: "submit_leg_failed — status leg skipped", @@ -443,6 +465,19 @@ func (w *DeployProbeWorker) recordLeg(ctx context.Context, leg string, r deployP ) return } + if r.result == deployProbeResultBootstrap { + // Self-heal success — log at INFO (not WARN; bootstrap is not + // a degradation) and skip audit_log. The metric counter at + // result=bootstrap is the operator-facing surface; this log + // line is the human-readable confirmation in the worker stream. + slog.Info("deploy_probe_bootstrap", + "leg", leg, + "reason", r.reason, + "latency_ms", r.latency.Milliseconds(), + "http_status", r.httpStatus, + ) + return + } slog.Debug("deploy_probe_pass", "leg", leg, "latency_ms", r.latency.Milliseconds(), @@ -491,10 +526,55 @@ func (w *DeployProbeWorker) emitDeployProbeFailed(ctx context.Context, leg strin // api expects (tarball + name + port + env + redeploy=true). Returns the // result plus the app_id pulled from the response envelope. The app_id // is the key the next two legs depend on. +// +// Bootstrap retry: the persistent probe-app row doesn't exist on the +// FIRST tick of a probe's lifetime, so the redeploy=true POST hits +// /deploy/new's "no_existing_deployment_to_redeploy" 404. legSubmit +// detects that exact canonical error code (NOT any 404 — see api#206 +// for the typed-error contract) and transparently retries once with +// redeploy omitted (create semantics). The retry's outcome is reported +// as `result=bootstrap` rather than `pass` so the dashboard can +// distinguish "we self-healed once" from "steady-state working". Every +// tick thereafter sees the row, gets a 2xx on the first call, and +// reports `pass`. func (w *DeployProbeWorker) legSubmit(ctx context.Context) (deployProbeLegResult, string) { + r, appID, errCode := w.legSubmitOnce(ctx, true /* redeploy */) + if r.result != deployProbeResultFail || r.httpStatus != http.StatusNotFound || errCode != deployProbeRedeployMissingCode { + return r, appID + } + // Canonical "first tick, app doesn't exist yet" → retry as a create. + // The second call carries the same tarball + name + env, just without + // redeploy=true. Anti-design: this is the ONLY 404 we retry on — a + // non-canonical 404 (auth, routing, an api-side regression that + // dropped the error_code field) still fails the leg, so we never + // mask a real outage as a bootstrap. + slog.Info("jobs.deploy_probe.bootstrap_retry", + "reason", "first_tick: api returned "+deployProbeRedeployMissingCode+" on redeploy=true", + "app_name", w.cfg.AppName, + "env", w.cfg.Env, + ) + r2, appID2, _ := w.legSubmitOnce(ctx, false /* redeploy */) + if r2.result == deployProbeResultPass { + // Successful self-heal: relabel as `bootstrap` so a steady-state + // dashboard tile shows the one-time bootstrap event distinctly + // from the per-tick pass. + r2.result = deployProbeResultBootstrap + r2.reason = "first-tick bootstrap: api returned " + deployProbeRedeployMissingCode + " on redeploy=true; retried without redeploy" + } + return r2, appID2 +} + +// legSubmitOnce performs a single POST /deploy/new. `redeploy` controls +// whether the request body carries `redeploy=true` (steady-state) or +// omits the field (first-tick bootstrap). Returns the leg result, the +// app_id (empty unless result=pass), and the api's canonical `error` +// string when the response was a non-2xx with a parseable JSON envelope +// (empty otherwise). The error_code is what the caller uses to decide +// whether a 404 is the bootstrap path or a real outage. +func (w *DeployProbeWorker) legSubmitOnce(ctx context.Context, redeploy bool) (deployProbeLegResult, string, string) { // buildDeployProbeMultipart writes to an in-memory bytes.Buffer and // cannot fail — see the helper's docstring. No err to check here. - body, contentType := buildDeployProbeMultipart(w.cfg.AppName, w.cfg.Env) + body, contentType := buildDeployProbeMultipart(w.cfg.AppName, w.cfg.Env, redeploy) target := w.cfg.BaseURL + "/deploy/new" req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, body) @@ -502,7 +582,7 @@ func (w *DeployProbeWorker) legSubmit(ctx context.Context) (deployProbeLegResult return deployProbeLegResult{ result: deployProbeResultFail, reason: "build_request: " + err.Error(), - }, "" + }, "", "" } req.Header.Set("Content-Type", contentType) req.Header.Set("Authorization", "Bearer "+w.cfg.BearerToken) @@ -516,7 +596,7 @@ func (w *DeployProbeWorker) legSubmit(ctx context.Context) (deployProbeLegResult result: deployProbeResultFail, reason: "http_error: " + err.Error(), latency: latency, - }, "" + }, "", "" } defer func() { _ = resp.Body.Close() }() @@ -533,7 +613,15 @@ func (w *DeployProbeWorker) legSubmit(ctx context.Context) (deployProbeLegResult if resp.StatusCode < 200 || resp.StatusCode >= 300 { r.result = deployProbeResultFail r.reason = fmt.Sprintf("status=%d (want 2xx); body=%s", resp.StatusCode, truncateForLog(string(respBody), 256)) - return r, "" + // Best-effort parse of the api's typed error envelope so the + // caller can branch on the canonical error code. A parse failure + // here is non-fatal — the leg still fails, the caller just sees + // an empty errCode and will not take the bootstrap retry path. + var errEnv struct { + ErrorCode string `json:"error"` + } + _ = json.Unmarshal(respBody, &errEnv) + return r, "", errEnv.ErrorCode } var parsed struct { OK bool `json:"ok"` @@ -544,26 +632,26 @@ func (w *DeployProbeWorker) legSubmit(ctx context.Context) (deployProbeLegResult if err := json.Unmarshal(respBody, &parsed); err != nil { r.result = deployProbeResultFail r.reason = "body_parse: " + err.Error() + "; raw=" + truncateForLog(string(respBody), 256) - return r, "" + return r, "", "" } if !parsed.OK { r.result = deployProbeResultFail r.reason = "body ok=false; raw=" + truncateForLog(string(respBody), 256) - return r, "" + return r, "", "" } if parsed.Item.AppID == "" { r.result = deployProbeResultFail r.reason = "body missing item.app_id; raw=" + truncateForLog(string(respBody), 256) - return r, "" + return r, "", "" } submitBudget := w.effectiveSubmitBudget() if latency > submitBudget { r.result = deployProbeResultDegraded r.reason = fmt.Sprintf("latency=%dms over budget=%dms", latency.Milliseconds(), submitBudget.Milliseconds()) - return r, parsed.Item.AppID + return r, parsed.Item.AppID, "" } r.result = deployProbeResultPass - return r, parsed.Item.AppID + return r, parsed.Item.AppID, "" } // legStatus drives leg 2: poll GET /deploy/ until status is @@ -736,16 +824,17 @@ func (w *DeployProbeWorker) legServe(ctx context.Context, appID string) deployPr // buildDeployProbeMultipart constructs the multipart body POSTed to // /deploy/new. The api requires `tarball` + `name` + `port` + `env`; -// `redeploy=true` is what makes the probe-app row reusable across -// ticks. Extracted so tests can re-use the exact same shape the prober -// puts on the wire. +// `redeploy` is sent as `"true"` when reusing an existing app row +// (steady-state ticks), or omitted entirely on the first-tick +// bootstrap retry so the api takes the create path. Extracted so +// tests can re-use the exact same shape the prober puts on the wire. // // Returns the buffer + Content-Type. No error path: every underlying // operation writes to an in-memory bytes.Buffer (CreateFormFile, // WriteField, part.Write, mw.Close) which cannot fail — same pattern // as auth_probe.legEmailStart's `_ = json.Marshal(...)`. Removing the // defensive branches keeps the patch-coverage gate at 100%. -func buildDeployProbeMultipart(name, env string) (*bytes.Buffer, string) { +func buildDeployProbeMultipart(name, env string, redeploy bool) (*bytes.Buffer, string) { var buf bytes.Buffer mw := multipart.NewWriter(&buf) @@ -758,13 +847,18 @@ func buildDeployProbeMultipart(name, env string) (*bytes.Buffer, string) { part, _ := mw.CreateFormFile("tarball", "app.tar.gz") _, _ = part.Write(tarball) - // Required + optional scalar fields. Loop rather than four duplicated + // Required + optional scalar fields. Loop rather than three+ duplicated // WriteField calls — keeps the field ordering matrix visible at a glance. + // `redeploy=true` is OMITTED on the bootstrap retry — the api treats + // the absence of the field as create-semantics, which is the only way + // to mint the probe-app row on the first-ever tick. fields := [][2]string{ {"name", name}, {"port", "80"}, {"env", env}, - {"redeploy", "true"}, + } + if redeploy { + fields = append(fields, [2]string{"redeploy", "true"}) } for _, f := range fields { _ = mw.WriteField(f[0], f[1]) diff --git a/internal/jobs/deploy_probe_internal_test.go b/internal/jobs/deploy_probe_internal_test.go index 1c7f10b..8431e0a 100644 --- a/internal/jobs/deploy_probe_internal_test.go +++ b/internal/jobs/deploy_probe_internal_test.go @@ -55,12 +55,13 @@ func TestBuildDeployProbeNginxTarball_RoundTrips(t *testing.T) { } // TestBuildDeployProbeMultipart_Shape — buildDeployProbeMultipart's -// success path. Asserts the multipart body carries `tarball`, `name`, -// `port=80`, `env`, and `redeploy=true` — the exact field set the api's -// /deploy/new handler reads. A missing field here would silently flip -// the prober to "fresh deploy" (no redeploy) and burn one slot per tick. +// success path with redeploy=true. Asserts the multipart body carries +// `tarball`, `name`, `port=80`, `env`, and `redeploy=true` — the exact +// field set the api's /deploy/new handler reads. A missing field here +// would silently flip the prober to "fresh deploy" (no redeploy) and +// burn one slot per tick. func TestBuildDeployProbeMultipart_Shape(t *testing.T) { - body, contentType := buildDeployProbeMultipart("probe-name", "development") + body, contentType := buildDeployProbeMultipart("probe-name", "development", true) if !strings.HasPrefix(contentType, "multipart/form-data; boundary=") { t.Errorf("contentType: %q", contentType) } @@ -79,6 +80,30 @@ func TestBuildDeployProbeMultipart_Shape(t *testing.T) { } } +// TestBuildDeployProbeMultipart_BootstrapShape — the bootstrap retry +// passes redeploy=false; the body must NOT carry a `redeploy` field at +// all, so the api takes the create-semantics path. Required so a future +// refactor that defaults the form field can't quietly re-introduce the +// "first-tick burns a slot per tick" regression. +func TestBuildDeployProbeMultipart_BootstrapShape(t *testing.T) { + body, _ := buildDeployProbeMultipart("probe-name", "development", false) + raw := body.String() + if strings.Contains(raw, `name="redeploy"`) { + t.Errorf("bootstrap body unexpectedly carries name=\"redeploy\" field: %s", raw) + } + // Other required fields must still be present. + for _, want := range []string{ + `name="tarball"`, + `name="name"`, "probe-name", + `name="port"`, "80", + `name="env"`, "development", + } { + if !strings.Contains(raw, want) { + t.Errorf("bootstrap body missing %q", want) + } + } +} + // TestRecordLeg_AllBranches — recordLeg's three branches (pass / // degraded / fail) emit different slog levels. The fail branch on a // nil-DB worker exercises the audit-skip path. We can't observe the diff --git a/internal/jobs/deploy_probe_test.go b/internal/jobs/deploy_probe_test.go index 54bd0a7..b1b3c22 100644 --- a/internal/jobs/deploy_probe_test.go +++ b/internal/jobs/deploy_probe_test.go @@ -16,6 +16,7 @@ package jobs_test import ( "context" "database/sql" + "io" "net/http" "net/http/httptest" "strings" @@ -727,6 +728,151 @@ func TestDeployProbe_ServeBuildRequestErr(t *testing.T) { } } +// TestDeployProbe_Bootstrap_FirstTick404RetriesAsCreate — the headline +// fix for the synthetic prober's first-tick wedge (worker#69 → worker#71). +// The first POST /deploy/new with redeploy=true must hit the canonical +// 404 `no_existing_deployment_to_redeploy` from the api. legSubmit MUST +// transparently retry without redeploy=true and report the second +// call's outcome as result=bootstrap (NOT pass), so the dashboard +// distinguishes "we self-healed once" from "steady-state working". +// Downstream legs (status + serve) MUST still run against the +// bootstrapped app_id — they get skipped on submit fail, but bootstrap +// is a success category for the purposes of leg dependency. +// +// Counts POSTs per body so the test asserts EXACTLY one bootstrap +// retry was made (no infinite loop / duplicate slot burn). +func TestDeployProbe_Bootstrap_FirstTick404RetriesAsCreate(t *testing.T) { + var ( + mu sync.Mutex + postsWithReply int + postsCreatePath int + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/deploy/new" && r.Method == http.MethodPost: + body, _ := io.ReadAll(r.Body) + mu.Lock() + hasRedeploy := strings.Contains(string(body), `name="redeploy"`) + if hasRedeploy { + postsWithReply++ + mu.Unlock() + // First-tick canonical 404 — exact body shape the api returns. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"ok":false,"error":"no_existing_deployment_to_redeploy","message":"No active deployment named \"deploy-probe-test\" was found in env=development."}`)) + return + } + postsCreatePath++ + mu.Unlock() + // Bootstrap path — create returns 202 + app_id. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"ok":true,"item":{"app_id":"bootstrap-app-1"}}`)) + case strings.HasPrefix(r.URL.Path, "/deploy/") && r.Method == http.MethodGet: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"ok":true,"item":{"app_id":"bootstrap-app-1","status":"healthy"}}`)) + case r.URL.Path == "/": + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("deploy-probe-ok")) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + // No ExpectExec — a bootstrap is NOT a fail; the audit_log path must + // not fire. mock.ExpectationsWereMet() at the end asserts this. + + fm := &fakeDeployProbeMetrics{} + w := jobs.NewDeployProbeWorker(db, httpClientRetargetingHTTPSToServer(srv), fm, deployProbeBaseConfig(t, srv)) + if err := w.Work(context.Background(), fakeJob[jobs.DeployProbeArgs]()); err != nil { + t.Fatalf("Work: %v", err) + } + + mu.Lock() + gotRedeploy, gotCreate := postsWithReply, postsCreatePath + mu.Unlock() + if gotRedeploy != 1 { + t.Errorf("POSTs with redeploy=true: want 1, got %d", gotRedeploy) + } + if gotCreate != 1 { + t.Errorf("POSTs without redeploy (bootstrap): want 1, got %d", gotCreate) + } + if got := fm.outcomeFor("submit"); got != "bootstrap" { + t.Errorf("submit outcome: want bootstrap, got %q", got) + } + // Downstream legs must run on the bootstrapped row. + if got := fm.outcomeFor("status"); got != "pass" { + t.Errorf("status outcome (after bootstrap): want pass, got %q", got) + } + if got := fm.outcomeFor("serve"); got != "pass" { + t.Errorf("serve outcome (after bootstrap): want pass, got %q", got) + } + // No audit_log row was expected — bootstrap is not a failure. + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("unexpected DB activity on bootstrap: %v", err) + } +} + +// TestDeployProbe_Bootstrap_NonCanonical404StillFails — coverage gate +// against the "any 404 is bootstrap" anti-pattern. A 404 whose body +// does NOT carry the canonical error_code (e.g. an auth misroute, a +// reverse-proxy 404, a future api-side regression that drops the +// typed-error field) MUST NOT trigger the retry — it stays a hard fail +// with an audit_log row. Otherwise a real outage gets silently masked +// as a self-heal. +func TestDeployProbe_Bootstrap_NonCanonical404StillFails(t *testing.T) { + var posts int + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/deploy/new" && r.Method == http.MethodPost { + mu.Lock() + posts++ + mu.Unlock() + // 404 with a DIFFERENT error code — looks like a routing 404, + // not the canonical missing-row signal. Must NOT trigger + // bootstrap retry. + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"ok":false,"error":"route_not_found","message":"unknown route"}`)) + } + })) + defer srv.Close() + + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + // audit_log row IS expected — this is a real failure, not a self-heal. + mock.ExpectExec(`INSERT INTO audit_log`).WillReturnResult(sqlmock.NewResult(1, 1)) + + fm := &fakeDeployProbeMetrics{} + w := jobs.NewDeployProbeWorker(db, httpClientRetargetingHTTPSToServer(srv), fm, deployProbeBaseConfig(t, srv)) + if err := w.Work(context.Background(), fakeJob[jobs.DeployProbeArgs]()); err != nil { + t.Fatalf("Work: %v", err) + } + + mu.Lock() + got := posts + mu.Unlock() + // EXACTLY one POST — the non-canonical 404 must not retry. + if got != 1 { + t.Errorf("POSTs to /deploy/new: want 1 (no retry on non-canonical 404), got %d", got) + } + if got := fm.outcomeFor("submit"); got != "fail" { + t.Errorf("submit outcome: want fail, got %q", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("audit_log expectation: %v", err) + } +} + // guardCompileTime ensures the fakeDeployProbeMetrics conforms to the // DeployProbeMetrics interface. var _ jobs.DeployProbeMetrics = (*fakeDeployProbeMetrics)(nil)