feat(ci): working aimock drift detection (#846)

* docs: design spec for working aimock drift detection

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* docs: implementation plan for aimock drift detection

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* feat(examples-chat): record mode for the aimock e2e runner

* feat(examples-chat): AIMOCK_MODE=record wiring in e2e global-setup

* feat(examples-chat): tag the contract-only @drift e2e subset

* feat(examples-chat): structural fixture differ replacing the broken drift script

* feat(ci): weekly aimock drift run — record-proxy e2e subset + structural diff

* feat(examples-chat): differ reports incomplete recordings separately from drift

The aimock recorder emits { content: '' } with a 'fixture may be
incomplete' warning when it cannot parse tool-call deltas from a stream.
Observed live: both tool-calling entries in the first record run collapsed
to empty text while the @drift specs passed. A recorder artifact, not
model drift — reported in its own category so it cannot masquerade as a
kind change.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

Author: blove

.github/workflows/aimock-drift.yml

@@ -2,10 +2,12 @@ name: aimock fixture drift
 
 on:
   workflow_dispatch:
-  # NOTE: scheduled cron is intentionally omitted. The committed fixtures are
-  # handwritten seeds — drift detection only becomes meaningful once a phase
-  # lands recorded fixtures. Re-enable a `schedule:` trigger here at that
-  # point (weekly cron is a reasonable starting cadence).
+  schedule:
+    # Weekly, Monday 09:00 UTC. Advisory only — never a merge gate. The @drift
+    # e2e subset runs against the LIVE provider through aimock's record proxy;
+    # a red run means today's model no longer satisfies our contract
+    # assertions. Recordings are uploaded as an artifact either way.
+    - cron: '0 9 * * 1'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -27,37 +29,61 @@ jobs:
         with:
           node-version: 22
           cache: npm
+      - name: Install uv
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
       - run: npm ci
-      - name: Run drift check
+      - name: Cache examples-chat python venv
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: examples/chat/python/.venv
+          key: uv-examples-chat-${{ runner.os }}-${{ hashFiles('examples/chat/python/uv.lock') }}
+      - name: Sync examples-chat python
+        working-directory: examples/chat/python
+        run: uv sync
+      - run: npx playwright install --with-deps chromium
+      - name: Run @drift subset against the live provider
+        id: drift-run
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        run: npx nx run examples-chat-angular:drift --skip-nx-cache
+          AIMOCK_MODE: record
+          AIMOCK_RECORD_DIR: ${{ runner.temp }}/recordings
+        working-directory: examples/chat/angular
+        run: npx playwright test --config e2e/playwright.config.ts --grep @drift
+      - name: Structural diff vs committed fixtures
+        if: always()
+        working-directory: examples/chat/angular
+        run: |
+          mkdir -p "${{ runner.temp }}/recordings"
+          npx tsx e2e/scripts/drift.ts "${{ runner.temp }}/recordings" | tee "${{ runner.temp }}/drift-report.json"
+      - name: Upload recordings artifact
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: aimock-recordings
+          path: ${{ runner.temp }}/recordings
+          if-no-files-found: warn
       - name: Open issue on drift
         if: failure()
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           script: |
+            const fs = require('fs');
             const { owner, repo } = context.repo;
-            const today = new Date().toISOString().slice(0, 10);
-            const title = `aimock fixture drift detected — ${today}`;
-            const body = [
-              'The scheduled fixture drift check failed.',
-              '',
-              `Workflow run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
-              '',
-              'Investigate which fixture drifted and either refresh it intentionally',
-              'or open a PR fixing the regression on the LLM side.',
-            ].join('\n');
-            const existing = await github.rest.issues.listForRepo({
-              owner, repo, labels: 'aimock-drift', state: 'open',
+            let report = '(drift report unavailable)';
+            try { report = fs.readFileSync(process.env.RUNNER_TEMP + '/drift-report.json', 'utf8'); } catch {}
+            const trigger = context.eventName === 'schedule' ? 'scheduled' : 'manually dispatched';
+            await github.rest.issues.create({
+              owner, repo,
+              title: 'aimock drift: @drift subset failed against the live provider',
+              body: [
+                `The ${trigger} fixture drift check failed.`,
+                '',
+                `Run: ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
+                '',
+                'Structural diff of recordings vs committed fixtures:',
+                '```json',
+                report.slice(0, 6000),
+                '```',
+              ].join('\n'),
+              labels: ['drift'],
             });
-            if (existing.data.length === 0) {
-              await github.rest.issues.create({
-                owner, repo, title, body, labels: ['aimock-drift'],
-              });
-            } else {
-              await github.rest.issues.createComment({
-                owner, repo, issue_number: existing.data[0].number,
-                body: `Drift again on ${today}. ${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`,
-              });
-            }