koth-gate.yml

# koth ladder gate - scores kit-only PRs against the reigning kit and
# enables auto-merge on a dethrone. the ditto mining loop, rebuilt on
# github primitives: the bench is the validator, branch protection is the
# chain, auto-merge is the emission.
#
# security model (do not weaken):
#   - pull_request_target: this definition and every line of executed code
#     come from the BASE branch. PR code is never checked out or executed.
#   - the only thing read from the PR is competition/kits/current/kit.yaml,
#     fetched over the api as data and schema-validated (closed allowlist)
#     before it is passed to the bench as extra_config.
#   - a PR qualifies for the ladder only if the kit file is the ONLY file
#     it touches. anything else is a normal PR: the gate passes without
#     scoring and a human reviews as usual.
#   - untrusted strings (branch names, repo names, titles) reach scripts
#     via env, never by interpolation into run bodies.
name: koth-gate

on:
  pull_request_target: # zizmor: ignore[dangerous-triggers] no PR code ever executes: the kit is fetched as data, validated against a closed-world allowlist, and scored by base-branch code with a read-only checkout
    types: [opened, synchronize, reopened, ready_for_review]

concurrency:
  group: koth-${{ github.event.pull_request.number }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  gate:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: write        # auto-merge (squash) needs it
      pull-requests: write   # enable auto-merge + scorecard comment
    steps:
      - name: checkout base branch (trusted code only)
        uses: actions/checkout@v4
        # pull_request_target defaults to the DEFAULT branch, not the PR
        # base - pin the base ref explicitly. still trusted code: a branch
        # of this repo, never the PR head. the base carries the scripts,
        # the engine, and the reigning kit.
        with:
          ref: ${{ github.event.pull_request.base.ref }}

      - name: classify the PR
        id: classify
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \
            --paginate --jq '.[].filename' > /tmp/changed.txt
          count=$(wc -l < /tmp/changed.txt)
          if [ "$count" = "1" ] && \
             grep -qx 'competition/kits/current/kit.yaml' /tmp/changed.txt; then
            echo "mode=ladder" >> "$GITHUB_OUTPUT"
          else
            echo "mode=normal" >> "$GITHUB_OUTPUT"
          fi

      - name: pass through (not a ladder PR)
        if: steps.classify.outputs.mode == 'normal'
        run: echo "not a kit-only PR - koth gate does not apply, humans review."

      - name: fetch challenger kit from the PR (data only)
        if: steps.classify.outputs.mode == 'ladder'
        env:
          GH_TOKEN: ${{ github.token }}
          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          KIT_PATH: competition/kits/current/kit.yaml
        run: |
          # the contents api inlines base64 only for files under its size
          # limit (~1MB); a larger file returns empty content and a download
          # url instead. read size + content together and refuse anything
          # that is not a small inlined blob, so an oversized kit can never
          # decode to "" and be scored as champion defaults.
          gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \
            > /tmp/kit-meta.json
          size=$(jq -r '.size // 0' /tmp/kit-meta.json)
          encoding=$(jq -r '.encoding // ""' /tmp/kit-meta.json)
          if [ "$encoding" != "base64" ]; then
            echo "kit not returned as an inlined base64 blob (encoding=$encoding, size=$size)" >&2
            exit 1
          fi
          jq -r '.content' /tmp/kit-meta.json | base64 -d > /tmp/kit-challenger.yaml
          if [ ! -s /tmp/kit-challenger.yaml ]; then
            echo "fetched kit is empty" >&2
            exit 1
          fi

      - name: set up python
        if: steps.classify.outputs.mode == 'ladder'
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: install vouch (base branch code)
        if: steps.classify.outputs.mode == 'ladder'
        run: python -m pip install -e .

      - name: validate challenger kit against the allowlist
        if: steps.classify.outputs.mode == 'ladder'
        run: python .github/scripts/validate_kit.py /tmp/kit-challenger.yaml

      - name: paired scoring - challenger vs reigning kit
        if: steps.classify.outputs.mode == 'ladder'
        id: score
        run: |
          set +e
          # seed identity = the tree actually scored (the checked-out base
          # tip), not github.sha, which is the default branch under
          # pull_request_target
          BASE_SHA="$(git rev-parse HEAD)"
          python .github/scripts/koth_score.py \
            --champion competition/kits/current/kit.yaml \
            --challenger /tmp/kit-challenger.yaml \
            --base-sha "$BASE_SHA" \
            --out /tmp/koth-report.json
          code=$?
          set -e
          if [ "$code" = "0" ]; then
            echo "verdict=dethroned" >> "$GITHUB_OUTPUT"
          elif [ "$code" = "3" ]; then
            echo "verdict=held" >> "$GITHUB_OUTPUT"
          else
            exit "$code"
          fi

      - name: post the scorecard
        if: steps.classify.outputs.mode == 'ladder'
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          VERDICT: ${{ steps.score.outputs.verdict }}
          BASE_REF: ${{ github.event.pull_request.base.ref }}
          LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }}
        run: |
          if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then
            gate_note="auto-merge on dethrone (provisional ladder branch)"
          else
            gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer promotes proven champions to shipped defaults by hand"
          fi
          # the posted markdown carries literal backticks
          # shellcheck disable=SC2016
          {
            echo "koth ladder - ${VERDICT}"
            echo
            echo '```json'
            cat /tmp/koth-report.json
            echo '```'
            echo
            echo "${gate_note}."
            echo
            echo "seeds derive from base sha + utc date; rerun locally with"
            echo '`python .github/scripts/koth_score.py --date <utc-date>` to reproduce.'
            echo "the daily result is provisional and overfittable - payout rank is"
            echo "settled monthly on sealed commit-reveal seeds (docs/vouchbench-seasons.md)."
          } > /tmp/comment.md
          gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \
            --body-file /tmp/comment.md

      # auto-merge fires ONLY on a dedicated, non-shipped ladder branch
      # (repo variable KOTH_LADDER_BASE). the trunk is never auto-written:
      # a kit-only PR against main is scored and commented, then a human
      # merges. this keeps the review gate load-bearing for everything that
      # ships - a beatable benchmark must never be the sole writer to code
      # users install. when KOTH_LADDER_BASE is unset, nothing auto-merges.
      - name: enable auto-merge on dethrone (ladder branch only)
        if: >-
          steps.classify.outputs.mode == 'ladder' &&
          steps.score.outputs.verdict == 'dethroned' &&
          vars.KOTH_LADDER_BASE != '' &&
          github.event.pull_request.base.ref == vars.KOTH_LADDER_BASE
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \
            --auto --squash

      - name: hold the throne (challenger did not clear the band)
        if: >-
          steps.classify.outputs.mode == 'ladder' &&
          steps.score.outputs.verdict == 'held'
        run: |
          echo "champion holds - challenger did not clear max(0.007, 1.96 x paired SE)."
          exit 1