koth-engine-gate.yml

# koth engine gate - scores strategy (ranking-code) submissions and, on a
# dethrone against the ladder branch, arms auto-merge: the benchmark decides
# what enters the quarantined contrib lane, with no human in the loop.
#
# what keeps that sane - the containment contract, all three parts:
#   1. merged strategies are QUARANTINED code: everything that executes a
#      contrib strategy (both gates, the local bench loop) runs it through
#      the sandbox child (vouch.strategy.run_sandboxed). nothing imports
#      contrib code in-process, and the ledger/ratchet scripts treat the
#      winner as text (promote_champion.py copies bytes, imports nothing).
#   2. the scoring job never holds a write token. it executes the untrusted
#      challenger, so it gets contents: read + the comment scope only; the
#      arm job below holds the write scopes, checks out nothing, and runs
#      no challenger code - it turns the verdict into a merge, that is all.
#      accepted residual: a sandbox escape during scoring can forge the
#      verdict and land its own PR in contrib/ - inside the quarantine.
#   3. shipped defaults stay human. auto-merge fires only when the PR base
#      is the ladder branch (KOTH_LADDER_BASE); promotion of a champion
#      into src/vouch as trusted, importable default code remains a
#      human-reviewed PR. the benchmark is never the sole gate to code
#      users install.
#
# security model of the scoring job itself (unchanged):
#   - pull_request_target: the workflow, the grader (score_strategy.py), and
#     the champion strategy all come from the BASE branch. only the challenger
#     .py is read from the PR, and it is executed ONLY inside the sandbox
#     child (vouch.strategy.run_sandboxed: rlimits + an audit hook blocking
#     network/subprocess/writes).
name: koth-engine-gate

on:
  pull_request_target: # zizmor: ignore[dangerous-triggers] workflow + grader + champion all come from the base branch; the PR's .py runs only inside the sandbox child, and the job token is read-only with no secrets
    types: [opened, synchronize, reopened, ready_for_review]

concurrency:
  group: koth-engine-${{ github.event.pull_request.number }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  gate:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
      pull-requests: write   # scorecard comment only - the merge lives in `arm`
    outputs:
      mode: ${{ steps.classify.outputs.mode }}
      verdict: ${{ steps.score.outputs.verdict }}
    steps:
      - name: checkout base branch (trusted code only)
        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        # pull_request_target defaults to the DEFAULT branch, not the PR
        # base - pin the base ref explicitly (same fix the kit gate got).
        # still trusted code: a branch of this repo, never the PR head.
        with:
          ref: ${{ github.event.pull_request.base.ref }}

      - name: classify the PR
        id: classify
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \
            --paginate --jq '.[].filename' > /tmp/changed.txt
          # a delete-only pr matches the filename shape but has nothing to
          # fetch at the head sha - classify it normal, not engine.
          gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \
            --paginate --jq '.[] | select(.status != "removed") | .filename' \
            > /tmp/present.txt
          count=$(wc -l < /tmp/changed.txt)
          only=$(head -1 /tmp/present.txt)
          case "$only" in
            contrib/strategies/baseline.py|contrib/strategies/README.md) only="" ;;
          esac
          if [ "$count" = "1" ] && \
             printf '%s' "$only" | grep -qE '^contrib/strategies/[A-Za-z0-9_]+\.py$'; then
            echo "mode=engine" >> "$GITHUB_OUTPUT"
            echo "path=$only" >> "$GITHUB_OUTPUT"
          else
            echo "mode=normal" >> "$GITHUB_OUTPUT"
          fi

      - name: pass through (not a strategy PR)
        if: steps.classify.outputs.mode == 'normal'
        run: echo "not a single-strategy PR - engine gate does not apply."

      - name: fetch challenger strategy from the PR (as data)
        if: steps.classify.outputs.mode == 'engine'
        env:
          GH_TOKEN: ${{ github.token }}
          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          KIT_PATH: ${{ steps.classify.outputs.path }}
        run: |
          gh api "repos/${HEAD_REPO}/contents/${KIT_PATH}?ref=${HEAD_SHA}" \
            > /tmp/strat-meta.json
          encoding=$(jq -r '.encoding // ""' /tmp/strat-meta.json)
          if [ "$encoding" != "base64" ]; then
            echo "strategy not returned as an inlined blob (encoding=$encoding)" >&2
            exit 1
          fi
          jq -r '.content' /tmp/strat-meta.json | base64 -d > /tmp/challenger.py
          if [ ! -s /tmp/challenger.py ]; then
            echo "fetched strategy is empty" >&2
            exit 1
          fi

      - name: set up python
        if: steps.classify.outputs.mode == 'engine'
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: '3.12'

      - name: install vouch (base branch code)
        if: steps.classify.outputs.mode == 'engine'
        run: python -m pip install -e .

      - name: paired scoring - challenger vs baseline champion (sandboxed)
        if: steps.classify.outputs.mode == 'engine'
        id: score
        run: |
          set +e
          # seed identity = the tree actually scored (the checked-out base
          # tip), not github.sha, which is the default branch under
          # pull_request_target
          BASE_SHA="$(git rev-parse HEAD)"
          python .github/scripts/score_strategy.py \
            --champion contrib/strategies/baseline.py \
            --challenger /tmp/challenger.py \
            --base-sha "$BASE_SHA" \
            --out /tmp/engine-report.json
          code=$?
          set -e
          if [ "$code" = "0" ]; then
            echo "verdict=dethroned" >> "$GITHUB_OUTPUT"
          elif [ "$code" = "3" ]; then
            echo "verdict=held" >> "$GITHUB_OUTPUT"
          else
            exit "$code"
          fi

      - name: post the scorecard
        if: steps.classify.outputs.mode == 'engine'
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          VERDICT: ${{ steps.score.outputs.verdict }}
          BASE_REF: ${{ github.event.pull_request.base.ref }}
          LADDER_BASE: ${{ vars.KOTH_LADDER_BASE }}
        run: |
          if [ "$BASE_REF" = "$LADDER_BASE" ] && [ -n "$LADDER_BASE" ]; then
            gate_note="auto-merge on dethrone (quarantined contrib lane: merged strategies only ever run inside the sandbox; shipped defaults still require a human PR)"
          else
            gate_note="scored only - auto-merge is disabled for base '${BASE_REF}'; a maintainer reviews and merges winners here"
          fi
          {
            echo "koth engine lane - ${VERDICT}"
            echo
            echo '```json'
            cat /tmp/engine-report.json
            echo '```'
            echo
            echo "${gate_note}."
            echo
            echo "the daily result is provisional (public seeds) - payout rank"
            echo "is settled by the monthly sealed run."
          } > /tmp/comment.md
          gh pr comment "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \
            --body-file /tmp/comment.md

      - name: report the verdict as the check result
        if: steps.classify.outputs.mode == 'engine'
        env:
          VERDICT: ${{ steps.score.outputs.verdict }}
        run: |
          echo "verdict: ${VERDICT}"
          # a held challenger is a green, informational result, and the arm
          # job merges nothing for it. a scoring error already failed the
          # job above.
          exit 0

  # the write-capable half, deliberately separated from the job that
  # executes the untrusted challenger: this job checks out nothing, runs no
  # challenger code, and touches no file from the PR - it reads the gate's
  # verdict and arms native auto-merge, exactly like the kit lane. the
  # merge itself is performed by github once required checks are green.
  # auto-merge fires ONLY when the PR base is the dedicated ladder branch
  # (repo variable KOTH_LADDER_BASE): the trunk keeps human review, and a
  # champion reaches shipped defaults only through a human PR.
  arm:
    needs: gate
    if: >-
      needs.gate.outputs.mode == 'engine' &&
      needs.gate.outputs.verdict == 'dethroned' &&
      vars.KOTH_LADDER_BASE != '' &&
      github.event.pull_request.base.ref == vars.KOTH_LADDER_BASE
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      contents: write        # auto-merge (squash) needs it
      pull-requests: write   # enable auto-merge
    steps:
      - name: enable auto-merge on dethrone (ladder branch only)
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          gh pr merge "$PR_NUMBER" --repo "$GITHUB_REPOSITORY" \
            --auto --squash