review/content-lane: duplicates.ts's block-scalar detection lags source-evidence.ts's hardened version (comment-tolerance, digit order) #72
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Calibration advisory checks — the logic/regex-change backtest (#8139, epic #8082) and the | |
| # counterfactual prompt replay (#8222, sub-epic #8218, epic #8211 track C), merged into ONE workflow | |
| # 2026-07-24: they shared identical scaffolding (dual base/head checkout, npm ci, engine build, D1 | |
| # corpus export, fail-open posture, fork-notice pairing) and overlapping path filters | |
| # (src/services/ai-review.ts triggered BOTH — two runners doing the same npm ci on every judge-prompt | |
| # PR), so one job runs both halves for one setup cost and one runner slot. | |
| # | |
| # BACKTEST half: replays linked_issue_scope_mismatch's recorded raw-context history (#8129/#8130 + | |
| # the #8139 model-response capture) through BOTH the PR's head checkout and its base checkout — | |
| # actually executing the two versions of the detection code, which is why this lives in CI and not in | |
| # ORB's live Worker (the Worker holds credentials and must never execute PR-supplied logic; a CI | |
| # checkout running a PR's own code is the exact trust boundary validate-code/validate-tests already | |
| # use for every PR). | |
| # | |
| # REPLAY half: replays the CANONICAL judge prompt from both checkouts (scripts/print-review-prompt.ts) | |
| # against the recorded raw-context fixture corpus and posts the Pareto-floored comparison. It | |
| # self-gates twice: on the COUNTERFACTUAL_REPLAY_BUDGET spend guard (#8222's requirement — unset, it | |
| # posts a visible skip notice and spends nothing), and on the canonical prompt text actually differing | |
| # between base and head (byte-identical ⇒ nothing to replay), so no extra path filter is needed for | |
| # runs that only touched the backtest's watched paths. | |
| # | |
| # Advisory only (#8105): never a required check, never blocks merge, and every step after setup FAILS | |
| # OPEN (notice + green) — the review engine auto-closes contributor PRs on ANY red check, so this | |
| # workflow's own plumbing must never redden. Each half posts its OWN clearly-labeled PR comment, | |
| # update-in-place keyed on its marker. | |
| name: calibration-advisory | |
| on: | |
| pull_request: | |
| # Explicit list because the default (opened/synchronize/reopened) omits ready_for_review -- once the | |
| # draft guard below skips draft PRs, marking a PR ready must itself trigger a real run (#6670). | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| # The union of the two halves' watched surfaces. Backtest: exactly the paths whose changes can alter | |
| # linked_issue_scope_mismatch-adjacent detection logic (#8139's Design section). Replay: the judge- | |
| # prompt surface (REVIEW_PROMPT_VERSION + buildSystemPrompt/parseModelReview), which is | |
| # src/services/ai-review.ts — already in the backtest's list, so the union IS the backtest's list. | |
| paths: | |
| - "src/rules/**" | |
| - "src/review/content-lane/**" | |
| - "src/settings/agent-actions.ts" | |
| - "src/services/ai-review.ts" | |
| - "src/services/linked-issue-satisfaction.ts" | |
| # Least privilege: the checks only read the repo; pull-requests: write is for their advisory comments. | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| concurrency: | |
| # pull_request-only workflow, so one ref-scoped group suffices (no push/github.sha split like ci.yml | |
| # needs): a newer push cancels the superseded run — its comments would be overwritten anyway. | |
| group: calibration-advisory-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| advisory: | |
| name: calibration advisory (backtest + replay) | |
| # Skip draft PRs (#6670, anti-abuse). Fork PRs are excluded at the job level rather than per-step: | |
| # GitHub withholds repo secrets from fork-originated pull_request runs, so the D1 corpus reads below | |
| # are impossible there and the whole job (npm ci included) would be waste — the fork-notice job below | |
| # is this workflow's half of ci.yml's paired fork==true/!=true convention. | |
| if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork != true }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| # The PR's base commit, checked out INSIDE the head workspace: the dynamically imported base modules | |
| # resolve bare npm specifiers by walking up from their own directory into the head checkout's | |
| # node_modules, so one `npm ci` serves both sides of every comparison (#8139). | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| path: .advisory-base | |
| persist-credentials: false | |
| - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version-file: .nvmrc | |
| cache: "npm" | |
| - name: Install deps | |
| run: npm ci --ignore-scripts | |
| # Both halves' scripts import @loopover/engine, which resolves to its dist/ build output. | |
| - name: Build engine package | |
| run: npx turbo run build --filter=@loopover/engine | |
| # --------------------------------------------------------------------------------------------- | |
| # Every step from here down FAILS OPEN (notice + green, never a red check): the review engine | |
| # auto-closes a contributor PR on ANY failed check, required or not — so an advisory job that can | |
| # go red on an infra problem (an under-scoped CLOUDFLARE_API_TOKEN, a D1 outage, a comment-post | |
| # hiccup) would let OUR plumbing close an innocent contributor's PR. "Never blocks merge" (#8105) | |
| # has to hold against this job's own failures, not just its verdicts. | |
| # --------------------------------------------------------------------------------------------- | |
| # ============================ BACKTEST HALF (#8139) ============================ | |
| # Read-only corpus export (#8084's CLI, reused as-is — no new D1 read code). The wrangler secrets | |
| # are available here because the fork guard above already excluded fork-originated runs. | |
| - name: Export backtest corpus from D1 | |
| id: corpus | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| run: | | |
| if npx tsx scripts/backtest-corpus-export.ts --rule-id linked_issue_scope_mismatch --output backtest-corpus.json --remote; then | |
| echo "available=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "available=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Corpus export from D1 failed (missing or under-scoped CLOUDFLARE_API_TOKEN — it needs D1 read/write on the loopover database). Logic backtest skipped; advisory only, never fails the PR." | |
| fi | |
| - name: Run logic backtest | |
| id: backtest | |
| if: ${{ steps.corpus.outputs.available == 'true' }} | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| if npx tsx scripts/backtest-logic-check.ts \ | |
| --rule-id linked_issue_scope_mismatch \ | |
| --corpus backtest-corpus.json \ | |
| --head-root . \ | |
| --base-root .advisory-base \ | |
| --output backtest-comment.md \ | |
| --head-sha "$HEAD_SHA" \ | |
| --base-sha "$BASE_SHA" \ | |
| --persist --remote --db loopover \ | |
| --repo "$GITHUB_REPOSITORY" \ | |
| --pr "$PR_NUMBER"; then | |
| echo "ready=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "ready=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Logic backtest run failed — no comparison produced. Advisory only, never fails the PR." | |
| fi | |
| # Update-in-place keyed on the comment marker so a re-run edits the existing comment instead of | |
| # stacking a new one per push. | |
| - name: Post or update the backtest PR comment | |
| if: ${{ steps.backtest.outputs.ready == 'true' }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| post_comment() { | |
| marker="<!-- loopover-logic-backtest -->" | |
| # --paginate runs the --jq filter once per page, so pin to the first emitted id. | |
| comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ | |
| --jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1) | |
| if [ -n "$comment_id" ]; then | |
| gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@backtest-comment.md | |
| else | |
| gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@backtest-comment.md | |
| fi | |
| } | |
| if ! post_comment; then | |
| echo "::notice::PR comment post failed — backtest result computed and persisted but not posted. Advisory only, never fails the PR." | |
| fi | |
| # ============================ REPLAY HALF (#8222) ============================ | |
| # SPEND GUARD (#8222's requirement): the replay costs real model inference, so it is gated on the | |
| # deployment explicitly configuring a budget — the COUNTERFACTUAL_REPLAY_BUDGET repo variable | |
| # (neuron units, see COUNTERFACTUAL_DEFAULT_NEURON_BUDGET) plus the provider endpoint/model. | |
| # #8279: the default provider is Workers AI (the CLOUDFLARE_* secrets above double as its | |
| # credentials — the token needs Workers AI run perms); an explicit COUNTERFACTUAL_OLLAMA_URL | |
| # secret switches to a reachable ollama instead. | |
| - name: Check replay budget configuration | |
| id: budget | |
| env: | |
| REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} | |
| REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} | |
| REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} | |
| run: | | |
| if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ]; then | |
| echo "configured=true" >> "$GITHUB_OUTPUT" | |
| if [ -n "$REPLAY_OLLAMA_URL" ]; then echo "provider=ollama" >> "$GITHUB_OUTPUT"; else echo "provider=workers-ai" >> "$GITHUB_OUTPUT"; fi | |
| else | |
| echo "configured=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables; Workers AI runs on the existing CLOUDFLARE_* secrets, or set COUNTERFACTUAL_OLLAMA_URL to use a reachable ollama). Advisory only, never fails the PR." | |
| fi | |
| # Cheap self-gate replacing the old workflow's ai-review.ts-only path filter: extract the canonical | |
| # prompt from both checkouts and replay only when the text actually differs — a PR that matched this | |
| # workflow's paths without touching the judge prompt costs two script invocations and stops here. | |
| - name: Extract base/head canonical prompts | |
| if: ${{ steps.budget.outputs.configured == 'true' }} | |
| id: prompts | |
| run: | | |
| if npx tsx scripts/print-review-prompt.ts --root . > head-prompt.txt \ | |
| && npx tsx scripts/print-review-prompt.ts --root .advisory-base > base-prompt.txt \ | |
| && npx tsx scripts/print-review-prompt.ts --root . --version-only > head-version.txt \ | |
| && npx tsx scripts/print-review-prompt.ts --root .advisory-base --version-only > base-version.txt; then | |
| if cmp -s head-prompt.txt base-prompt.txt; then | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Canonical judge prompt text is byte-identical across base/head — nothing to replay." | |
| else | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Prompt extraction failed — replay skipped. Advisory only, never fails the PR." | |
| fi | |
| - name: Export replay corpus from D1 | |
| if: ${{ steps.budget.outputs.configured == 'true' && steps.prompts.outputs.changed == 'true' }} | |
| id: replaycorpus | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| run: | | |
| if npx tsx scripts/backtest-corpus-export.ts --rule-id ai_consensus_defect --output replay-corpus.json --remote; then | |
| echo "available=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "available=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Corpus export from D1 failed — replay skipped. Advisory only, never fails the PR." | |
| fi | |
| # Two runs over the IDENTICAL deterministic sample (seed = head SHA, per the design contract's | |
| # per-PR determinism requirement): base prompt first (the baseline artifact), then head with | |
| # --baseline (comparison + comment + persisted run event). | |
| - name: Replay base and head prompts | |
| if: ${{ steps.replaycorpus.outputs.available == 'true' }} | |
| id: replay | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} | |
| REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} | |
| REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} | |
| REPLAY_PROVIDER: ${{ steps.budget.outputs.provider }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha }} | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| BASE_VERSION=$(cat base-version.txt) | |
| HEAD_VERSION=$(cat head-version.txt) | |
| if npx tsx scripts/counterfactual-replay.ts \ | |
| --fixtures replay-corpus.json \ | |
| --variant "${BASE_VERSION}@${REPLAY_MODEL}" \ | |
| --prompt-file base-prompt.txt \ | |
| --budget "$REPLAY_BUDGET" \ | |
| --seed-suffix "$HEAD_SHA" \ | |
| --provider "$REPLAY_PROVIDER" \ | |
| --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ | |
| --out base-scores.json \ | |
| && npx tsx scripts/counterfactual-replay.ts \ | |
| --fixtures replay-corpus.json \ | |
| --variant "${HEAD_VERSION}@${REPLAY_MODEL}" \ | |
| --prompt-file head-prompt.txt \ | |
| --budget "$REPLAY_BUDGET" \ | |
| --seed-suffix "$HEAD_SHA" \ | |
| --provider "$REPLAY_PROVIDER" \ | |
| --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ | |
| --baseline base-scores.json \ | |
| --base-variant-label "$BASE_VERSION" \ | |
| --comment-out replay-comment.md \ | |
| --head-sha "$HEAD_SHA" --base-sha "$BASE_SHA" \ | |
| --persist --remote --db loopover \ | |
| --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER"; then | |
| echo "ready=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "ready=false" >> "$GITHUB_OUTPUT" | |
| echo "::notice::Counterfactual replay failed — no comparison produced. Advisory only, never fails the PR." | |
| fi | |
| - name: Post or update the replay PR comment | |
| if: ${{ steps.replay.outputs.ready == 'true' }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| post_comment() { | |
| marker="<!-- loopover-counterfactual-backtest -->" | |
| comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ | |
| --jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1) | |
| if [ -n "$comment_id" ]; then | |
| gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@replay-comment.md | |
| else | |
| gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@replay-comment.md | |
| fi | |
| } | |
| if ! post_comment; then | |
| echo "::notice::PR comment post failed — replay computed and persisted but not posted. Advisory only, never fails the PR." | |
| fi | |
| # The fork half of ci.yml's paired fork==true/!=true convention: fork-originated pull_request runs get | |
| # no repo secrets, so the D1-backed halves cannot run — say so visibly instead of failing or going | |
| # silent. Advisory only either way; a skipped run never blocks anything (#8105). | |
| fork-notice: | |
| name: calibration advisory (skipped for fork PRs) | |
| if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork == true }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Explain the skip | |
| run: echo "::notice::Fork PR — repo secrets are withheld, so the D1-backed calibration advisory checks (logic backtest + counterfactual replay) are skipped. Advisory only; nothing blocks." |