fix(ci): open the protected immutable cockpit preview with its own automation bypass (#984)

With #983 the deploy job reached "Exhaustively verify immutable cockpit
preview" for the first time, and it failed on its first probe:

    [preview] root default redirect: expected 308, received 302.

The 302 is Vercel deployment protection sending every path on the unaliased
artifact to vercel.com/sso-api — the same wall #974 removed for the Website
preview. Bypass secrets are issued per Vercel project, so the Website secret
cannot open the cockpit deployment.

- deploy-smoke.ts sends `x-vercel-protection-bypass` on every probe when
  VERCEL_AUTOMATION_BYPASS_SECRET is set, read from the environment so the
  value never lands in argv or step logs. A 302 to the SSO endpoint now names
  deployment protection and the missing secret instead of reporting a bare
  status mismatch.
- The workflow step supplies the secret from
  VERCEL_COCKPIT_AUTOMATION_BYPASS_SECRET and fails with a provisioning
  message when it is unset.
- Unit tests cover the header on every probe (including the hostile-header
  case), its absence without a secret, and the SSO hint; the workflow guard
  asserts both preview checks carry their own project's secret.

Requires the repository secret VERCEL_COCKPIT_AUTOMATION_BYPASS_SECRET.

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

Author: blove

.github/workflows/ci.yml

@@ -1111,7 +1111,17 @@ jobs:
       - name: Exhaustively verify immutable cockpit preview
         if: steps.freshness.outputs.stale != 'true' && steps.affected.outputs.cockpit == 'true'
         run: |
+          # Deployment protection answers every path on the unaliased artifact
+          # with 302 -> vercel.com/sso-api, which the smoke reports as
+          # "expected 308, received 302". Bypass secrets are issued per Vercel
+          # project, so the Website secret cannot open this one. Say so.
+          if [ -z "${VERCEL_AUTOMATION_BYPASS_SECRET}" ]; then
+            echo "::error::VERCEL_COCKPIT_AUTOMATION_BYPASS_SECRET is unset — the protected immutable cockpit preview cannot be verified. Enable 'Protection Bypass for Automation' on the Vercel threadplane-cockpit project and store the value as this repository secret."
+            exit 1
+          fi
           npx tsx apps/cockpit/scripts/deploy-smoke.ts --url "${{ steps.deploy_cockpit.outputs.deployment_url }}" --mode preview --retries 20 --retry-delay-ms 5000
+        env:
+          VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_COCKPIT_AUTOMATION_BYPASS_SECRET }}
       - name: Check this commit is still the tip before cockpit promotion
         if: steps.freshness.outputs.stale != 'true' && steps.affected.outputs.cockpit == 'true'
         id: cockpit_promotion_freshness

apps/cockpit/scripts/deploy-smoke.spec.ts

@@ -332,6 +332,73 @@ describe('redirect deploy smoke contract', () => {
     ).rejects.toThrow(/WAF Raw Path prerequisite/);
   });
 
+  it('sends the automation bypass on every probe only when a secret is supplied', async () => {
+    // Vercel deployment protection answers every path on an unaliased
+    // deployment with 302 -> vercel.com/sso-api, so the immutable cockpit
+    // artifact can only be verified with the project's automation bypass.
+    const cases = buildRedirectSmokeCases('preview');
+    const withSecret = vi.fn(async (request: RedirectSmokeRequest) => {
+      const { 'x-vercel-protection-bypass': bypass, ...rest } =
+        request.headers ?? {};
+      if (bypass !== 'cockpit-bypass-sentinel') {
+        throw new Error(`Missing bypass on ${request.path}`);
+      }
+      return responseFor(
+        { ...request, headers: Object.keys(rest).length ? rest : undefined },
+        cases
+      );
+    });
+
+    await expect(
+      runDeploySmoke({
+        url: previewUrl,
+        mode: 'preview',
+        requestImpl: withSecret,
+        bypassSecret: 'cockpit-bypass-sentinel',
+      })
+    ).resolves.toBe(`pass:preview:${previewUrl}:${cases.length}`);
+    expect(withSecret).toHaveBeenCalledTimes(cases.length);
+    expect(withSecret).toHaveBeenCalledWith(
+      expect.objectContaining({
+        path: '/langgraph/core-capabilities/streaming/overview/python',
+        headers: expect.objectContaining({
+          'x-forwarded-host': 'attacker.test',
+          'x-vercel-protection-bypass': 'cockpit-bypass-sentinel',
+        }),
+      })
+    );
+
+    const withoutSecret = vi.fn(async (request: RedirectSmokeRequest) =>
+      responseFor(request, cases)
+    );
+    await runDeploySmoke({
+      url: previewUrl,
+      mode: 'preview',
+      requestImpl: withoutSecret,
+    });
+    for (const [request] of withoutSecret.mock.calls) {
+      expect(request.headers ?? {}).not.toHaveProperty(
+        'x-vercel-protection-bypass'
+      );
+    }
+  });
+
+  it('names Vercel deployment protection when a probe lands on the SSO redirect', async () => {
+    const requestImpl = vi.fn(async () => ({
+      status: 302,
+      headers: {
+        location:
+          'https://vercel.com/sso-api?url=https%3A%2F%2Fimmutable-preview.vercel.app%2F&nonce=abc',
+      },
+    }));
+
+    await expect(
+      runDeploySmoke({ url: previewUrl, mode: 'preview', requestImpl })
+    ).rejects.toThrow(
+      /expected 308, received 302.*deployment protection.*automation bypass/i
+    );
+  });
+
   it('formats dry-run output with the selected mode and case count', async () => {
     await expect(
       runDeploySmoke({ url: previewUrl, mode: 'preview', dryRun: true })

apps/cockpit/scripts/deploy-smoke.ts

@@ -46,6 +46,14 @@ export interface DeploySmokeOptions {
   readonly retryDelayMs?: number;
   readonly requestImpl?: RedirectSmokeRequestImpl;
   readonly sleep?: (delayMs: number) => Promise<void>;
+  /**
+   * Vercel "Protection Bypass for Automation" secret for the project that
+   * owns the deployment. Deployment protection answers every path on an
+   * unaliased deployment with 302 -> vercel.com/sso-api, so the immutable
+   * artifact can only be probed when each request carries this header. The
+   * secret is issued per project: the cockpit one is not the Website one.
+   */
+  readonly bypassSecret?: string;
 }
 
 export interface ParsedDeploySmokeArgs {
@@ -57,6 +65,8 @@ export interface ParsedDeploySmokeArgs {
 }
 
 const WEBSITE_ORIGIN = 'https://threadplane.ai';
+const BYPASS_HEADER = 'x-vercel-protection-bypass';
+const BYPASS_SECRET_ENV = 'VERCEL_AUTOMATION_BYPASS_SECRET';
 const DEFAULT_RETRIES = 0;
 const DEFAULT_RETRY_DELAY_MS = 2000;
 const ALL_MODES: readonly WorkspaceMode[] = ['Docs', 'Run', 'Code', 'API'];
@@ -383,8 +393,15 @@ const verifyCase = (
     ? ' Raw Path rejection failed; verify the Vercel project WAF Raw Path prerequisite before promotion.'
     : '';
   if (response.status !== smokeCase.expectedStatus) {
+    const protectionHint =
+      response.status === 302 &&
+      (response.headers.location ?? '').startsWith(
+        'https://vercel.com/sso-api'
+      )
+        ? ` The deployment answered with Vercel deployment protection, not the redirect service; supply the owning project's automation bypass secret via ${BYPASS_SECRET_ENV}.`
+        : '';
     throw new RedirectContractError(
-      `[${mode}] ${smokeCase.name}: expected ${smokeCase.expectedStatus}, received ${response.status}.${rawGateHint}`
+      `[${mode}] ${smokeCase.name}: expected ${smokeCase.expectedStatus}, received ${response.status}.${rawGateHint}${protectionHint}`
     );
   }
   const location = response.headers.location;
@@ -411,6 +428,7 @@ export const runDeploySmoke = async ({
   retryDelayMs = DEFAULT_RETRY_DELAY_MS,
   requestImpl = requestExactTarget,
   sleep = defaultSleep,
+  bypassSecret,
 }: DeploySmokeOptions): Promise<string> => {
   const target = new URL(url);
   if (target.pathname !== '/' || target.search || target.hash) {
@@ -420,14 +438,21 @@ export const runDeploySmoke = async ({
   const cases = buildRedirectSmokeCases(mode);
   if (dryRun) return `dry-run:${mode}:${origin}:${cases.length}`;
 
+  const bypassHeaders: Readonly<Record<string, string>> | undefined =
+    bypassSecret ? { [BYPASS_HEADER]: bypassSecret } : undefined;
+
   for (const smokeCase of cases) {
+    const headers =
+      smokeCase.headers || bypassHeaders
+        ? { ...smokeCase.headers, ...bypassHeaders }
+        : undefined;
     let attempt = 0;
     while (true) {
       try {
         const response = await requestImpl({
           origin,
           path: smokeCase.path,
-          ...(smokeCase.headers ? { headers: smokeCase.headers } : {}),
+          ...(headers ? { headers } : {}),
         });
         verifyCase(mode, smokeCase, response);
         break;
@@ -454,7 +479,10 @@ if (
 ) {
   try {
     const options = parseDeploySmokeArgs(process.argv.slice(2));
-    runDeploySmoke(options)
+    // Read the secret from the environment, never argv, so it stays out of
+    // process listings and CI step logs.
+    const bypassSecret = process.env[BYPASS_SECRET_ENV] || undefined;
+    runDeploySmoke({ ...options, ...(bypassSecret ? { bypassSecret } : {}) })
       .then((result) => process.stdout.write(`${result}\n`))
       .catch((error: unknown) => {
         const message = error instanceof Error ? error.message : String(error);

scripts/ci-workflow.spec.mjs

@@ -556,6 +556,41 @@ describe('CI workflow', () => {
     );
   });
 
+  it('verifies every protected immutable preview with its own automation bypass', async () => {
+    // Vercel deployment protection answers every path on an unaliased
+    // deployment with 302 -> vercel.com/sso-api. Bypass secrets are issued per
+    // project, so the Website and cockpit checks each need their own, and a
+    // missing one must fail with a message that says what to provision rather
+    // than as an opaque "expected 308, received 302".
+    const deployJob = await readDeployJob();
+    const websiteStep = readNamedStep(
+      deployJob,
+      'Verify Website preview runtime embedding policy'
+    );
+    const cockpitStep = readNamedStep(
+      deployJob,
+      'Exhaustively verify immutable cockpit preview'
+    );
+
+    assert.match(
+      websiteStep,
+      /VERCEL_AUTOMATION_BYPASS_SECRET:\s*\$\{\{ secrets\.VERCEL_AUTOMATION_BYPASS_SECRET \}\}/
+    );
+    assert.match(websiteStep, /-z "\$\{VERCEL_AUTOMATION_BYPASS_SECRET\}"/);
+    assert.match(
+      cockpitStep,
+      /VERCEL_AUTOMATION_BYPASS_SECRET:\s*\$\{\{ secrets\.VERCEL_COCKPIT_AUTOMATION_BYPASS_SECRET \}\}/
+    );
+    assert.match(cockpitStep, /-z "\$\{VERCEL_AUTOMATION_BYPASS_SECRET\}"/);
+    assert.match(cockpitStep, /::error::[^\n]*threadplane-cockpit/);
+    assert.match(cockpitStep, /exit 1/);
+    assert.doesNotMatch(
+      cockpitStep,
+      /secrets\.VERCEL_AUTOMATION_BYPASS_SECRET/,
+      'the cockpit preview must not reuse the Website project secret'
+    );
+  });
+
   it('gates Cockpit deployment on the production Website smoke even for Cockpit-only changes', async () => {
     const deployJob = await readDeployJob();
     const websiteOrCockpit =