diff --git a/GOLD-STANDARD.html b/GOLD-STANDARD.html index 82cf089..ea3d22a 100644 --- a/GOLD-STANDARD.html +++ b/GOLD-STANDARD.html @@ -1237,6 +1237,7 @@

6. The harness standard

│ ├── deploy-dev.yml # merge to main -> client dev environment (automatic, unattended) │ ├── deploy-promote.yml # dev -> test -> prod; manual only, named approver every time │ ├── dependency-scan.yml # weekly advisory scan; raises an issue, never blocks +│ ├── rails-telemetry.yml # weekly gate-outcome report, committed for the fleet view │ └── eval-*.yml # eval-regression + eval-suite (§11 agentic work) ├── scripts/rails/ # diff-anchors.sh, apply-branch-protection.sh ├── eval-datasets/ + prompts/ # golden-set template + versioned judge prompts (§11 work only) @@ -1527,6 +1528,8 @@

10. The kit (what's in this repo)

│ │ # manual only, human go/no-go every time — §7) │ │ # + dependency-scan.yml (weekly standing-stock advisory scan; │ │ # the blocking half is ci.yml's dependency-gate job — §7) +│ │ # + rails-telemetry.yml (weekly gate-outcome report, committed; +│ │ # read across repos by scripts/collect_rails_telemetry.py — §9) │ │ # (+ eval-regression.yml, eval-suite.yml for agentic specs — §11) │ ├── packs/ # composable additions: stacks/dotnet, cicd/github, cicd/azure-devops, │ │ # frontend/generic, frontend/react, tools/gitnexus diff --git a/GOLD-STANDARD.md b/GOLD-STANDARD.md index 5ea74a6..1f57cb6 100644 --- a/GOLD-STANDARD.md +++ b/GOLD-STANDARD.md @@ -561,6 +561,34 @@ flat. Client-facing scorecard: their success metric, the DORA stability pair, accepted-as-is trend, and the demo. That's it. +### Watching the gates across engagements + +Every gate outcome and every override is recorded in the repo it happened in — the accepted-risk +labels, the two ledgers, the PR timeline. That is enough to answer a question about one repo and +useless for answering one about a portfolio. So each installed repo writes a weekly +`rails-telemetry.json` and commits it: which gates ran and what they concluded, every override by +name with the change it was applied to, and — the part that needs a machine — **which checks +branch protection actually requires, against which gate jobs actually exist**. + +That last comparison is the reason the file exists. A gate has two halves: the check, and the rule +requiring it to pass. Remove the rule and the check still runs, still reports, and looks entirely +normal on the pull request; a red run simply merges anyway. From outside that repo, a gate someone +disarmed and a gate that never caught anything produce identical evidence. No amount of counting +separates them. + +Two constraints on this, both non-negotiable. **It stays inside the client's tenancy** — the +workflow reads the repo's own history through the platform's own API and writes into the same +repo; nothing is transmitted anywhere, which is what makes it something a client security team can +approve. And **it counts gates, never people**: an override count is reported against merged +changes so it reads as a rate rather than a bare number, and there is no per-author breakdown +anywhere in the file. The rule from earlier in this section holds here too — we measure the rails, +not the humans. + +`scripts/collect_rails_telemetry.py` (operator tooling, not part of the kit and never installed) +reads those files across every reachable repo and reports worst-first. A repo that is not +reporting is listed as **unknown, not clean** — a fleet view that quietly counts silence as health +is the same failure it was built to catch. + --- ## 10. The kit (what's in this repo) @@ -597,6 +625,8 @@ intent-driven-development/ # cloned locally as delivery-standard/ on some ma │ │ # manual only, human go/no-go every time — §7) │ │ # + dependency-scan.yml (weekly standing-stock advisory scan; │ │ # the blocking half is ci.yml's dependency-gate job — §7) +│ │ # + rails-telemetry.yml (weekly gate-outcome report, committed; +│ │ # read across repos by scripts/collect_rails_telemetry.py — §9) │ │ # (+ eval-regression.yml, eval-suite.yml for agentic specs — §11) │ ├── packs/ # composable additions: stacks/dotnet, cicd/github, cicd/azure-devops, │ │ # frontend/generic, frontend/react, tools/gitnexus @@ -604,6 +634,7 @@ intent-driven-development/ # cloned locally as delivery-standard/ on some ma │ ├── prompts/ # versioned judge prompts (§11) │ ├── infra/ # Bicep starters │ └── profile/ # CODEOWNERS, rubrics, branch-protection ruleset, rails scripts +│ # + rails-telemetry.schema.json (the report's shape, fixed at v1) │ # + eval-bypasses.md and dependency-exceptions.md (the two │ # accepted-risk ledgers — each entry named, dated, and expiring) │ # (customer profiles — starter, microsoft-enterprise, … — live in diff --git a/docs/the-rails.md b/docs/the-rails.md index 97e8f99..5356e2f 100644 --- a/docs/the-rails.md +++ b/docs/the-rails.md @@ -526,6 +526,24 @@ inflate every one of those, and the published research is blunt about it — mea doubled PR volume while actual delivery stayed flat. The rails are healthy when changes flow and fail rarely, not when the agents are busy. +**Watch that the gates are still armed, not just that they are green.** Proving a rail once, at +Foundation, proves it was wired that week. Branch protection is edited later — during an +incident, in a repo reorganisation, by someone with admin who meant to change one thing. The +moment a check stops being *required*, it keeps running and keeps reporting, and a red run +merges anyway. Nothing on the pull request looks different. From outside the repo, a disarmed +gate and a gate that never caught anything are the same picture. + +So each repo writes a weekly `rails-telemetry.json` — what ran, every override by name, and the +comparison of what branch protection *requires* against what the workflows *declare*. It reads +the repo's own history through the platform's own API and commits into the same repo; nothing +leaves the client's tenancy. `scripts/collect_rails_telemetry.py` reads those files across the +fleet and puts disarmed gates at the top, with repos that are not reporting listed as **unknown +rather than clean** — counting silence as health is the failure this exists to catch. + +Where the file cannot read live branch protection it falls back to the committed ruleset and +says so, because that reading describes *intent* rather than what the platform is enforcing. +Presenting the two as equivalent would be the same silent-green problem one layer up. + **Log everything with provenance.** Every recommendation an agent made, every artifact that got applied, every policy-gate outcome — logged centrally, with co-authorship on the commits, so any change is traceable to the identity that produced it. After a poisoned tool-return steers an agent @@ -557,6 +575,11 @@ the provenance trail is what makes the rails auditable rather than merely automa - **The skipped environment.** A green build promoted from dev directly to prod because the operator picked the wrong target and nothing checked. "The same build that passed test" is only true if something enforces that it actually passed test. +- **The gate that was quietly unrequired.** Someone removes a check from branch protection to + unblock an urgent fix and never puts it back. The workflow still runs, still posts its + verdict, still looks exactly as it did — and stops mattering. This is not caught by watching + pull requests, because nothing about them changes; only comparing what is required against + what exists finds it. - **The dependency scan that reports nothing.** A misconfigured scan, a private feed with no vulnerability data, or a broken output parser all produce the same clean green as a genuinely clean repo. "No findings" and "not looking" are indistinguishable from the outside, which is diff --git a/kit/README.md b/kit/README.md index 7ab2495..ef956bc 100644 --- a/kit/README.md +++ b/kit/README.md @@ -65,6 +65,8 @@ so install here unless you also repoint the references. | `workflows/RAILS.md` | `./.github/RAILS.md` | Operator's guide + shakedown drills. | | `profile/rubrics/*` | `./.github/profile/rubrics/` | Workflows read these by this path. | | `workflows/dependency-scan.yml` | `./.github/workflows/` | Weekly scan of the **standing stock** of third-party advisories. Raises (and self-closes) one issue; never blocks — the blocking, diff-scoped half is `ci.yml`'s `dependency-gate` job. On by default: a security scan you have to remember to switch on is not running. | +| `workflows/rails-telemetry.yml` | `./.github/workflows/` | Weekly gate-outcome report, committed as `.github/rails-telemetry.json`. Records what ran, every override by name, and **which checks branch protection actually requires vs which gate jobs exist** — the comparison that catches a gate someone disarmed. No external calls: it reads this repo's own history and writes into this repo. | +| `profile/rails-telemetry.schema.json` | `./.github/rails-telemetry.schema.json` | The report's shape, fixed at version 1 before the install wave so it is not retrofitted across live repos. The fleet collector refuses a version it does not know rather than misreading it. | | `profile/eval-bypasses.md` | `./.github/eval-bypasses.md` | Override/bypass ledger. | | `profile/dependency-exceptions.md` | `./.github/dependency-exceptions.md` | Accepted-risk ledger for `dependency-gate`. The `accepted-risk:dependency` label clears one PR; this records why, who decided, whether the vulnerable path is reachable, and when the acceptance expires. Swept at Setup review. | | `profile/CODEOWNERS` | `./.github/CODEOWNERS` | | diff --git a/kit/packs/cicd/azure-devops/azure-pipelines/rails-telemetry.yml b/kit/packs/cicd/azure-devops/azure-pipelines/rails-telemetry.yml new file mode 100644 index 0000000..8ffe068 --- /dev/null +++ b/kit/packs/cicd/azure-devops/azure-pipelines/rails-telemetry.yml @@ -0,0 +1,230 @@ +# ============================================================================= +# One week of THIS repo's gate outcomes, written down where a human can read it. +# ============================================================================= +# Overrides and gate outcomes were already recorded per repo — the eval-bypasses ledger, the +# accepted-risk PR labels, the PR timeline. Nothing aggregated them, so across a portfolio +# nobody could answer three questions: are the gates firing, are they being routinely waved +# through, and has one been quietly switched off. +# +# This pipeline answers them for one repo and commits `.github/rails-telemetry.json`. The +# fleet collector (operator-side, not installed here) reads that file across every reachable +# repo. SAME SCHEMA as the github pack's version — different APIs, one output shape, so one +# collector reads a mixed GitHub/Azure fleet without caring which platform wrote a file. +# +# ── THE ONE THAT MATTERS: A GATE HAS TWO HALVES ────────────────────────────────────────── +# There is the build validation, and there is the branch POLICY requiring it. Delete the +# policy — deliberately, or in a reorganisation nobody reviewed — and the pipeline still runs, +# still reports, and looks entirely normal on the PR. But a red run completes anyway. From the +# outside, a disarmed gate is indistinguishable from a gate that never caught anything. No +# amount of counting finds that; only comparing what IS REQUIRED against what EXISTS does. +# +# ── CLIENT DATA BOUNDARY ────────────────────────────────────────────────────────────────── +# NO EXTERNAL CALLS. Everything queries this project's own history through the Azure DevOps +# REST API with the pipeline's own OAuth token, and writes the result into this repo. Nothing +# leaves the client's tenancy — a deliberate constraint, and what lets a client security team +# say yes to it. +# +# ── NO ACTIVITY METRICS ─────────────────────────────────────────────────────────────────── +# Counts are per GATE, never per author. `pull_requests_merged` exists only as a denominator: +# two overrides against three PRs is a very different week from two against ninety. It is not +# a productivity number and must never be reported as one. +# +# ── PLACEHOLDERS ────────────────────────────────────────────────────────────────────────── +# <> the protected branch this reports on (reference: main) +# ============================================================================= + +trigger: none +pr: none + +schedules: + # Monday 06:00 UTC — ahead of dependency-scan.yml's 07:00 so the two do not contend. + - cron: '0 6 * * 1' + displayName: 'Weekly rails telemetry' + branches: + include: + - main # <> + always: true # the gates' behaviour changes even when the code does not + +pool: + vmImage: 'ubuntu-latest' + +jobs: + - job: rails_telemetry + displayName: 'rails-telemetry' + condition: or(eq(variables['Build.Reason'], 'Manual'), ne(variables['RAILS_TELEMETRY_ENABLED'], 'false')) + steps: + - checkout: self + persistCredentials: true # the report is committed back + + - bash: | + set -euo pipefail + OUT=.github/rails-telemetry.json + mkdir -p .github + T="$(Agent.TempDirectory)" + TO="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + FROM="$(date -u -d "-${WINDOW_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)" + API="${SYSTEM_COLLECTIONURI%/}/${SYSTEM_TEAMPROJECT}/_apis" + AUTH="Authorization: Bearer ${SYSTEM_ACCESSTOKEN}" + + # ── activity: build results per pipeline definition, over the window ──────────── + curl -sf -H "$AUTH" "${API}/build/builds?minTime=${FROM}&api-version=7.1&\$top=1000" \ + > "${T}/builds.json" || echo '{"value":[]}' > "${T}/builds.json" + jq -c '.value[]? | {name: .definition.name, conclusion: (.result // "unknown")}' \ + "${T}/builds.json" > "${T}/runs.ndjson" || : > "${T}/runs.ndjson" + + # ── merged PRs and their labels ───────────────────────────────────────────────── + # Labels are not on the PR list payload, so they are fetched per PR — the same + # extra call ci.yml's spec-gate makes to honour no-spec:chore. + curl -sf -H "$AUTH" "${API}/git/repositories/${BUILD_REPOSITORY_ID}/pullrequests?searchCriteria.status=completed&searchCriteria.targetRefName=refs/heads/${DEFAULT_BRANCH}&api-version=7.1&\$top=200" \ + > "${T}/prs_raw.json" || echo '{"value":[]}' > "${T}/prs_raw.json" + : > "${T}/prs.ndjson" + for ID in $(jq -r --arg from "$FROM" '.value[]? | select(.closedDate >= $from) | .pullRequestId' "${T}/prs_raw.json"); do + LBL="$(curl -sf -H "$AUTH" "${API}/git/repositories/${BUILD_REPOSITORY_ID}/pullRequests/${ID}/labels?api-version=7.1" \ + | jq -c '[(.value // .)[]?.name]' || echo '[]')" + jq -c --argjson id "$ID" --argjson labels "$LBL" \ + '.value[] | select(.pullRequestId == $id) | {number: .pullRequestId, title: .title, merged_at: .closedDate, labels: $labels}' \ + "${T}/prs_raw.json" >> "${T}/prs.ndjson" || true + done + + # ── enforcement: what the branch POLICY actually requires ────────────────────── + # Prefer live policy — the only reading that catches someone changing it in the UI. + # Fall back to the committed desired-state file, recording WHICH, because reporting + # intent as though it were reality is precisely the silent-green failure this file + # exists to expose. + ENF_SOURCE=unavailable + : > "${T}/required.txt" + if curl -sf -H "$AUTH" "${API}/policy/configurations?api-version=7.1" > "${T}/policy.json" 2>/dev/null; then + jq -r '.value[]? | select(.isEnabled and .isBlocking) + | select(.type.displayName == "Build") | .settings.displayName // empty' \ + "${T}/policy.json" | sort -u > "${T}/required.txt" || true + if [ -s "${T}/required.txt" ]; then ENF_SOURCE=live; fi + fi + if [ "$ENF_SOURCE" != "live" ] && [ -f .azuredevops/rails/branch-policies.json ]; then + jq -r '.build_validation[]? | select(.isBlocking) | .displayName' \ + .azuredevops/rails/branch-policies.json | sort -u > "${T}/required.txt" || true + if [ -s "${T}/required.txt" ]; then ENF_SOURCE=committed; fi + fi + + # Pipelines/stages that CALL THEMSELVES merge gates. + grep -rhoE '^\s+displayName:\s*.(build-and-test|spec-gate|eval-gate|dependency-gate|grader|correctness-review|security-review|eval-regression).' \ + .azuredevops/pipelines/*.yml 2>/dev/null \ + | grep -oE '(build-and-test|spec-gate|eval-gate|dependency-gate|grader|correctness-review|security-review|eval-regression)' \ + | sort -u > "${T}/gatejobs.txt" || : > "${T}/gatejobs.txt" + + # ── ledgers ──────────────────────────────────────────────────────────────────── + TODAY="$(date -u +%Y-%m-%d)" + ledger_json() { + local path="$1" + if [ ! -f "$path" ]; then + jq -n --arg p "$path" '{path:$p, present:false, active_entries:0, expired_entries:0}' + return + fi + local total expired + total="$(grep -cE '^[[:space:]]*expires:' "$path" || true)" + expired="$(grep -oE '^[[:space:]]*expires:[[:space:]]*[0-9]{4}-[0-9]{2}-[0-9]{2}' "$path" \ + | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' \ + | awk -v today="$TODAY" '$0 < today' | wc -l | tr -d ' ')" + jq -n --arg p "$path" --argjson t "${total:-0}" --argjson e "${expired:-0}" \ + '{path:$p, present:true, active_entries:($t - $e), expired_entries:$e}' + } + ledger_json .azuredevops/rails/eval-bypasses.md > "${T}/ledgers.ndjson" + ledger_json .azuredevops/rails/dependency-exceptions.md >> "${T}/ledgers.ndjson" + + # ── assemble — IDENTICAL shape to the github pack's, by contract ─────────────── + jq -n \ + --arg repo "${SYSTEM_TEAMPROJECT}/${BUILD_REPOSITORY_NAME}" \ + --arg gen "$TO" --arg from "$FROM" --arg to "$TO" \ + --argjson days "${WINDOW_DAYS}" --arg enf_source "$ENF_SOURCE" \ + --slurpfile ledgers "${T}/ledgers.ndjson" \ + --rawfile runs_raw "${T}/runs.ndjson" \ + --rawfile prs_raw "${T}/prs.ndjson" \ + --rawfile required_raw "${T}/required.txt" \ + --rawfile gatejobs_raw "${T}/gatejobs.txt" ' + def lines(s): (s | split("\n") | map(select(length > 0))); + def objs(s): (s | split("\n") | map(select(length > 0) | fromjson)); + + (objs($runs_raw)) as $runs | + (objs($prs_raw)) as $prs | + (lines($required_raw)) as $required | + (lines($gatejobs_raw)) as $gatejobs | + + ([$runs[] | .name] | unique) as $names | + ($names | map(. as $n | ($runs | map(select(.name == $n))) as $r | { + context: $n, + runs: ($r | length), + passed: ($r | map(select(.conclusion == "succeeded")) | length), + failed: ($r | map(select(.conclusion == "failed")) | length), + skipped: ($r | map(select(.conclusion == "canceled")) | length) + })) as $gates | + + (["accepted-risk:correctness","accepted-risk:dependency","gate-exception","risk:high"] + | map(. as $lbl | ($prs | map(select(.labels | index($lbl)))) as $hit | { + label: $lbl, + count: ($hit | length), + uses: ($hit | map({pr: .number, title: .title, merged_at: .merged_at})) + })) as $labels | + + ([ ($gatejobs - $required | map({ + kind: "gate_not_required", severity: "high", context: ., + detail: ("The pipeline stage (" + . + ") declares itself a required build validation, but no blocking branch policy requires it. It runs, it reports, and a red run completes anyway — a disarmed gate.") + })), + ($required - $gatejobs | map({ + kind: "required_context_has_no_job", severity: "high", context: ., + detail: ("A blocking branch policy requires (" + . + "), but no pipeline produces it. Every pull request waits on a validation that will never report.") + })), + ($required | map(select(. as $c | ($gates | map(.context) | index($c)) == null)) | map({ + kind: "required_context_never_reported", severity: "medium", context: ., + detail: ("Required validation (" + . + ") produced no builds in this window. Either nothing triggered it, or it is quietly broken — from here those look the same.") + })), + (if $enf_source == "unavailable" then [{ + kind: "telemetry_incomplete", severity: "high", + detail: "Branch policy could not be read, live or committed, so no enforcement check was possible. Treat this repo gate status as UNKNOWN, never as clean." + }] elif $enf_source == "committed" then [{ + kind: "telemetry_incomplete", severity: "low", + detail: "Branch policy was read from the committed desired-state file, not the live API, so this reports intent rather than what Azure DevOps is actually enforcing." + }] else [] end), + ($ledgers | map(select(.expired_entries > 0)) | map({ + kind: "ledger_entry_expired", severity: "medium", + detail: (.path + " has " + (.expired_entries|tostring) + " acceptance(s) past their review date. An acceptance that outlives its expiry has quietly become permanent.") + })) + ] | flatten) as $findings | + + { + schema_version: 1, + repo: $repo, + generated_at: $gen, + window: { days: $days, from: $from, to: $to }, + activity: { gates: $gates, pull_requests_merged: ($prs | length) }, + overrides: { labels: $labels, ledgers: $ledgers }, + enforcement: { + source: $enf_source, + required_contexts: $required, + gate_jobs_present: $gatejobs, + findings: $findings + } + }' > "$OUT" + + echo "Wrote $OUT" + jq '{repo, gates: (.activity.gates|length), findings: (.enforcement.findings|length), source: .enforcement.source}' "$OUT" + displayName: 'Build the telemetry report' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + WINDOW_DAYS: '7' + DEFAULT_BRANCH: 'main' # <> + + - bash: | + set -euo pipefail + git config user.name "Azure Pipelines" + git config user.email "azuredevops@microsoft.com" + git add .github/rails-telemetry.json + # Nothing changed is the common case on a quiet week, and an empty commit every + # Monday is noise in the history everyone learns to scroll past. + if git diff --cached --quiet; then + echo "No change since the last report." + exit 0 + fi + git commit -m "chore: rails telemetry $(date -u +%Y-%m-%d)" + git push origin "HEAD:${DEFAULT_BRANCH}" + displayName: 'Commit the report' + env: + DEFAULT_BRANCH: 'main' # <> diff --git a/kit/packs/cicd/azure-devops/pack.yaml b/kit/packs/cicd/azure-devops/pack.yaml index b04366b..9fe5388 100644 --- a/kit/packs/cicd/azure-devops/pack.yaml +++ b/kit/packs/cicd/azure-devops/pack.yaml @@ -22,6 +22,7 @@ provides: - azure-pipelines/deploy-promote.yml # dev→test→prod — manual only; the target Environment's # approval check IS the human go/no-go (STARTER) - azure-pipelines/dependency-scan.yml # weekly standing-stock scan — raises a work item, never blocks + - azure-pipelines/rails-telemetry.yml # weekly gate-outcome report — same schema as the github pack's - azure-pipelines/eval-regression.yml # per-PR eval-regression gate — BLOCKS - azure-pipelines/eval-suite.yml # full benchmark — manual + scheduled, ADVISORY templates: @@ -101,6 +102,7 @@ overlays: - { src: azure-pipelines/deploy-dev.yml, dest: .azuredevops/pipelines/deploy-dev.yml } - { src: azure-pipelines/deploy-promote.yml, dest: .azuredevops/pipelines/deploy-promote.yml } - { src: azure-pipelines/dependency-scan.yml, dest: .azuredevops/pipelines/dependency-scan.yml } + - { src: azure-pipelines/rails-telemetry.yml, dest: .azuredevops/pipelines/rails-telemetry.yml } - { src: azure-pipelines/eval-regression.yml, dest: .azuredevops/pipelines/eval-regression.yml } - { src: azure-pipelines/eval-suite.yml, dest: .azuredevops/pipelines/eval-suite.yml } - { src: azure-pipelines/templates/setup-toolchain.yml, dest: .azuredevops/pipelines/templates/setup-toolchain.yml } diff --git a/kit/packs/cicd/github/pack.yaml b/kit/packs/cicd/github/pack.yaml index d7e0502..616dd72 100644 --- a/kit/packs/cicd/github/pack.yaml +++ b/kit/packs/cicd/github/pack.yaml @@ -19,6 +19,7 @@ provides: - workflows/deploy-dev.yml # promote-never-rebuild; rolls back on failure (starter) - workflows/deploy-promote.yml # dev→test→prod; manual only, human go/no-go via Environments - workflows/dependency-scan.yml # weekly standing-stock scan; raises an issue, never blocks + - workflows/rails-telemetry.yml # weekly gate-outcome report; committed for the fleet collector - workflows/eval-regression.yml # §11 per-PR blocking eval-regression gate - workflows/eval-suite.yml # §11 periodic advisory full benchmark rails_guide: RAILS.md # operator's guide + shakedown drills @@ -76,6 +77,7 @@ overlays: - { src: workflows/deploy-dev.yml, dest: .github/workflows/deploy-dev.yml } - { src: workflows/deploy-promote.yml, dest: .github/workflows/deploy-promote.yml } - { src: workflows/dependency-scan.yml, dest: .github/workflows/dependency-scan.yml } + - { src: workflows/rails-telemetry.yml, dest: .github/workflows/rails-telemetry.yml } - { src: dependabot.yml, dest: .github/dependabot.yml } - { src: workflows/eval-regression.yml, dest: .github/workflows/eval-regression.yml } - { src: workflows/eval-suite.yml, dest: .github/workflows/eval-suite.yml } diff --git a/kit/packs/cicd/github/workflows/rails-telemetry.yml b/kit/packs/cicd/github/workflows/rails-telemetry.yml new file mode 100644 index 0000000..fcba414 --- /dev/null +++ b/kit/packs/cicd/github/workflows/rails-telemetry.yml @@ -0,0 +1,251 @@ +name: Rails Telemetry + +# ============================================================================= +# One week of THIS repo's gate outcomes, written down where a human can read it. +# ============================================================================= +# Overrides and gate outcomes were already recorded per repo — the eval-bypasses +# ledger, the accepted-risk labels, the PR timeline. Nothing aggregated them, so +# across a portfolio nobody could answer three questions: are the gates firing, +# are they being routinely waved through, and has one been quietly switched off. +# +# This workflow answers them for one repo and commits the answer as +# `.github/rails-telemetry.json`. The fleet collector (operator-side, not +# installed here) reads that file across every reachable repo. +# +# ── THE ONE THAT MATTERS: A GATE HAS TWO HALVES ────────────────────────────── +# There is the check, and there is the rule requiring it to pass. Delete the +# second — deliberately, or by a repo reorganisation nobody reviewed — and the +# check still runs, still reports, and looks completely normal on the PR page. +# But a red run merges anyway. From the outside, a disarmed gate is +# indistinguishable from a gate that simply never caught anything. +# +# No amount of counting finds that. Only comparing what IS REQUIRED against what +# EXISTS finds it, which is the `enforcement` section below and the reason this +# file is worth committing at all. +# +# ── CLIENT DATA BOUNDARY ───────────────────────────────────────────────────── +# NO EXTERNAL CALLS. Everything here queries this repo's own history through its +# own platform's API, using the token the platform already issued, and writes the +# result into this repo. Nothing leaves the client's tenancy. That is a deliberate +# constraint, not an implementation detail — it is what lets a client security +# team say yes to it, and anything that phones home would have to be argued for +# from scratch. +# +# ── NO ACTIVITY METRICS ────────────────────────────────────────────────────── +# Counts are per GATE, never per author. The standard forbids activity metrics in +# client materials and does not track them internally either (§9): agents inflate +# every one of them. `pull_requests_merged` is here solely as a denominator, so +# two overrides can be read against three PRs or against ninety — those are very +# different weeks. It is not a productivity number and must never be reported as +# one. +# +# ── PLACEHOLDER ─────────────────────────────────────────────────────────────── +# <> the protected branch this reports on (reference: main) +# ============================================================================= +# +# ── PACK NOTE (github CI/CD pack) ──────────────────────────────── +# NO stack seam. This workflow reads gate outcomes and branch protection, not build output, +# so no ci-profile command is consumed. Its only placeholder is <>. +# The Azure DevOps pack ships the same report from different APIs — same schema, so one +# collector reads a mixed fleet. + +on: + schedule: + # Monday 06:00 UTC — before the working week, so the fleet report is ready when + # someone looks. Deliberately ahead of dependency-scan.yml's 07:00 so the two + # do not contend for the same runner minute. + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: write # it commits the report + actions: read # workflow run conclusions + pull-requests: read # merged PRs and their labels + checks: read + +concurrency: + group: rails-telemetry + cancel-in-progress: true + +jobs: + rails-telemetry: + name: rails-telemetry + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || vars.RAILS_TELEMETRY_ENABLED != 'false' + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build the telemetry report + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + WINDOW_DAYS: '7' + DEFAULT_BRANCH: 'main' # <> + run: | + set -euo pipefail + OUT=.github/rails-telemetry.json + mkdir -p .github + TO="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + FROM="$(date -u -d "-${WINDOW_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)" + T="${RUNNER_TEMP}" + + # ── activity ──────────────────────────────────────────────────────── + # Workflow-run conclusions per job name, over the window. `gh api --paginate` + # so a busy week is not silently truncated at page one — a report that + # quietly drops half its data is worse than no report. + gh api --paginate "repos/${REPO}/actions/runs?created=>${FROM}&per_page=100" \ + --jq '.workflow_runs[]? | {id, name, conclusion}' > "${T}/runs.ndjson" || echo '' > "${T}/runs.ndjson" + + gh api --paginate "repos/${REPO}/pulls?state=closed&base=${DEFAULT_BRANCH}&per_page=100" \ + --jq '.[] | select(.merged_at != null) | {number, title, merged_at, labels: [.labels[].name]}' \ + > "${T}/prs_all.ndjson" || echo '' > "${T}/prs_all.ndjson" + jq -c --arg from "$FROM" 'select(.merged_at >= $from)' "${T}/prs_all.ndjson" > "${T}/prs.ndjson" || : > "${T}/prs.ndjson" + + # ── enforcement: what is ACTUALLY required ────────────────────────── + # Prefer the live ruleset: it is the only reading that catches someone + # editing branch protection in the UI. Fall back to the committed + # desired-state file when the API is unreadable — worth having, but it + # describes INTENT rather than reality, so the source is recorded and the + # collector is told which it got. Reporting intent as though it were + # reality is exactly the silent-green failure this file exists to expose. + ENF_SOURCE=unavailable + : > "${T}/required.txt" + if gh api "repos/${REPO}/rules/branches/${DEFAULT_BRANCH}" > "${T}/rules.json" 2>/dev/null; then + jq -r '.[]? | select(.type == "required_status_checks") + | .parameters.required_status_checks[]?.context' "${T}/rules.json" \ + | sort -u > "${T}/required.txt" || true + if [ -s "${T}/required.txt" ]; then ENF_SOURCE=live; fi + fi + if [ "$ENF_SOURCE" != "live" ] && [ -f .github/rulesets/branch-protection.json ]; then + jq -r '.rules[]? | select(.type == "required_status_checks") + | .parameters.required_status_checks[]?.context' .github/rulesets/branch-protection.json \ + | sort -u > "${T}/required.txt" || true + if [ -s "${T}/required.txt" ]; then ENF_SOURCE=committed; fi + fi + + # Jobs that CALL THEMSELVES merge gates. The marker comment is the same one + # check_standard.py keys on in the standard's own repo, so the two agree on + # what "declares itself a gate" means. + grep -rhoE '^\s{4}name:\s*\S+\s*#\s*<--\s*required status-check context' \ + .github/workflows/*.yml 2>/dev/null \ + | sed -E 's/^[[:space:]]*name:[[:space:]]*//; s/[[:space:]]*#.*$//' \ + | sort -u > "${T}/gatejobs.txt" || : > "${T}/gatejobs.txt" + + # ── ledgers ───────────────────────────────────────────────────────── + TODAY="$(date -u +%Y-%m-%d)" + ledger_json() { + local path="$1" + if [ ! -f "$path" ]; then + jq -n --arg p "$path" '{path:$p, present:false, active_entries:0, expired_entries:0}' + return + fi + # `expires: YYYY-MM-DD` lines, per the ledger templates' documented format. + local total expired + total="$(grep -cE '^[[:space:]]*expires:' "$path" || true)" + expired="$(grep -oE '^[[:space:]]*expires:[[:space:]]*[0-9]{4}-[0-9]{2}-[0-9]{2}' "$path" \ + | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' \ + | awk -v today="$TODAY" '$0 < today' | wc -l | tr -d ' ')" + jq -n --arg p "$path" --argjson t "${total:-0}" --argjson e "${expired:-0}" \ + '{path:$p, present:true, active_entries:($t - $e), expired_entries:$e}' + } + # Concatenated into ONE file rather than passed via process substitution: + # `--slurpfile` reads every JSON value in a file into an array, so this hands + # jq a flat array of ledgers directly. Process substitution would work on the + # runner but not everywhere the drill might be rehearsed by hand, and a report + # that only builds on one machine is a report nobody verifies. + ledger_json .github/eval-bypasses.md > "${T}/ledgers.ndjson" + ledger_json .github/dependency-exceptions.md >> "${T}/ledgers.ndjson" + + # ── assemble ──────────────────────────────────────────────────────── + jq -n \ + --arg repo "$REPO" --arg gen "$TO" --arg from "$FROM" --arg to "$TO" \ + --argjson days "${WINDOW_DAYS}" --arg enf_source "$ENF_SOURCE" \ + --slurpfile ledgers "${T}/ledgers.ndjson" \ + --rawfile runs_raw "${T}/runs.ndjson" \ + --rawfile prs_raw "${T}/prs.ndjson" \ + --rawfile required_raw "${T}/required.txt" \ + --rawfile gatejobs_raw "${T}/gatejobs.txt" ' + def lines(s): (s | split("\n") | map(select(length > 0))); + def objs(s): (s | split("\n") | map(select(length > 0) | fromjson)); + + (objs($runs_raw)) as $runs | + (objs($prs_raw)) as $prs | + (lines($required_raw)) as $required | + (lines($gatejobs_raw)) as $gatejobs | + + ([$runs[] | .name] | unique) as $names | + ($names | map(. as $n | ($runs | map(select(.name == $n))) as $r | { + context: $n, + runs: ($r | length), + passed: ($r | map(select(.conclusion == "success")) | length), + failed: ($r | map(select(.conclusion == "failure")) | length), + skipped: ($r | map(select(.conclusion == "skipped")) | length) + })) as $gates | + + (["accepted-risk:correctness","accepted-risk:dependency","gate-exception","risk:high"] + | map(. as $lbl | ($prs | map(select(.labels | index($lbl)))) as $hit | { + label: $lbl, + count: ($hit | length), + uses: ($hit | map({pr: .number, title: .title, merged_at: .merged_at})) + })) as $labels | + + # findings: the whole point. Structured so a fleet can be ranked without + # parsing prose. + ([ ($gatejobs - $required | map({ + kind: "gate_not_required", severity: "high", context: ., + detail: ("The job '" + . + "' declares itself a required status check, but branch protection does not require it. It runs, it reports, and a red run merges anyway — a disarmed gate.") + })), + ($required - $gatejobs | map({ + kind: "required_context_has_no_job", severity: "high", context: ., + detail: ("Branch protection requires '" + . + "', but no workflow job produces it. Every pull request waits on a check that will never report.") + })), + ($required | map(select(. as $c | ($gates | map(.context) | index($c)) == null)) | map({ + kind: "required_context_never_reported", severity: "medium", context: ., + detail: ("Required check '" + . + "' produced no runs in this window. Either nothing triggered it, or it is quietly broken — from here those look the same.") + })), + (if $enf_source == "unavailable" then [{ + kind: "telemetry_incomplete", severity: "high", + detail: "Branch protection could not be read, live or committed, so no enforcement check was possible. Treat this repo's gate status as UNKNOWN, never as clean." + }] elif $enf_source == "committed" then [{ + kind: "telemetry_incomplete", severity: "low", + detail: "Branch protection was read from the committed ruleset, not the live API, so this reports intent rather than what the platform is actually enforcing. A change made in the UI would not show up here." + }] else [] end), + ($ledgers | map(select(.expired_entries > 0)) | map({ + kind: "ledger_entry_expired", severity: "medium", + detail: (.path + " has " + (.expired_entries|tostring) + " acceptance(s) past their review date. An acceptance that outlives its expiry has quietly become permanent.") + })) + ] | flatten) as $findings | + + { + schema_version: 1, + repo: $repo, + generated_at: $gen, + window: { days: $days, from: $from, to: $to }, + activity: { gates: $gates, pull_requests_merged: ($prs | length) }, + overrides: { labels: $labels, ledgers: $ledgers }, + enforcement: { + source: $enf_source, + required_contexts: $required, + gate_jobs_present: $gatejobs, + findings: $findings + } + }' > "$OUT" + + echo "Wrote $OUT" + jq '{repo, window: .window.days, gates: (.activity.gates|length), findings: (.enforcement.findings|length), source: .enforcement.source}' "$OUT" + + - name: Commit the report + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add .github/rails-telemetry.json + # Nothing changed is the common case on a quiet week, and an empty commit + # every Monday is noise in the history everyone learns to scroll past. + if git diff --cached --quiet; then + echo "No change since the last report." + exit 0 + fi + git commit -m "chore: rails telemetry $(date -u +%Y-%m-%d)" + git push diff --git a/kit/profile/rails-telemetry.schema.json b/kit/profile/rails-telemetry.schema.json new file mode 100644 index 0000000..ab7c8dc --- /dev/null +++ b/kit/profile/rails-telemetry.schema.json @@ -0,0 +1,162 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://mckruz.dev/delivery-standard/rails-telemetry.schema.json", + "title": "Rails telemetry", + "description": "One week of gate outcomes for ONE repo, written by the rails-telemetry workflow into .github/rails-telemetry.json and committed. The fleet collector reads these files across every reachable repo. Client-owned data: the workflow queries only the repo's own history through its own platform's API and makes no external calls, which is what makes it acceptable to a client security team. Fixed at version 1 BEFORE the install wave, deliberately — a schema retrofitted across installed repos is a migration nobody schedules.", + "type": "object", + "required": ["schema_version", "repo", "generated_at", "window", "activity", "overrides", "enforcement"], + "additionalProperties": false, + "properties": { + "schema_version": { + "const": 1, + "description": "Bump only for a breaking shape change. The collector refuses a version it does not know rather than silently misreading it." + }, + "repo": { + "type": "string", + "description": "owner/name (GitHub) or project/repo (Azure DevOps). The fleet key." + }, + "generated_at": { "type": "string", "format": "date-time" }, + "window": { + "type": "object", + "required": ["days", "from", "to"], + "additionalProperties": false, + "description": "The period covered. Explicit rather than implied, so a collector reading files written on different days never compares mismatched spans.", + "properties": { + "days": { "type": "integer", "minimum": 1 }, + "from": { "type": "string", "format": "date-time" }, + "to": { "type": "string", "format": "date-time" } + } + }, + + "activity": { + "type": "object", + "required": ["gates", "pull_requests_merged"], + "additionalProperties": false, + "description": "Did the gates run, and what did they conclude. Counts only — never per-author attribution. The standard forbids activity metrics in client materials and does not track them internally either (GOLD-STANDARD section 9); this measures the RAILS, not the people.", + "properties": { + "gates": { + "type": "array", + "items": { + "type": "object", + "required": ["context", "runs", "passed", "failed"], + "additionalProperties": false, + "properties": { + "context": { "type": "string", "description": "The status-check context / job name." }, + "runs": { "type": "integer", "minimum": 0 }, + "passed": { "type": "integer", "minimum": 0 }, + "failed": { "type": "integer", "minimum": 0 }, + "skipped": { "type": "integer", "minimum": 0 } + } + } + }, + "pull_requests_merged": { + "type": "integer", + "minimum": 0, + "description": "Denominator only. Present so an override count can be read as a rate rather than a bare number; two overrides against three PRs is a different story from two against ninety. NOT a productivity measure and never reported as one." + } + } + }, + + "overrides": { + "type": "object", + "required": ["labels", "ledgers"], + "additionalProperties": false, + "description": "Where a human consciously stepped past a gate. Every one of these is legitimate by design — the point is not to eliminate them but to make them visible and countable, because an override nobody can see is indistinguishable from a gate that never fired.", + "properties": { + "labels": { + "type": "array", + "items": { + "type": "object", + "required": ["label", "count", "uses"], + "additionalProperties": false, + "properties": { + "label": { + "type": "string", + "description": "accepted-risk:correctness | accepted-risk:dependency | gate-exception | risk:high" + }, + "count": { "type": "integer", "minimum": 0 }, + "uses": { + "type": "array", + "items": { + "type": "object", + "required": ["pr"], + "additionalProperties": false, + "properties": { + "pr": { "type": ["integer", "string"] }, + "title": { "type": "string" }, + "merged_at": { "type": "string", "format": "date-time" } + } + } + } + } + } + }, + "ledgers": { + "type": "array", + "description": "The committed accepted-risk files. An acceptance is meant to expire; an expired entry still sitting here is a decision that quietly became permanent.", + "items": { + "type": "object", + "required": ["path", "present", "active_entries", "expired_entries"], + "additionalProperties": false, + "properties": { + "path": { "type": "string" }, + "present": { "type": "boolean" }, + "active_entries": { "type": "integer", "minimum": 0 }, + "expired_entries": { "type": "integer", "minimum": 0 } + } + } + } + } + }, + + "enforcement": { + "type": "object", + "required": ["source", "required_contexts", "gate_jobs_present", "findings"], + "additionalProperties": false, + "description": "THE SECTION THAT JUSTIFIES THIS FILE. A gate has two halves: the check, and the rule requiring it to pass. Remove the second and the check still runs, still reports, and looks entirely normal — but a red run merges anyway. From the outside, a disarmed gate is indistinguishable from a gate that never caught anything. Counts cannot tell them apart; only this comparison can.", + "properties": { + "source": { + "type": "string", + "enum": ["live", "committed", "unavailable"], + "description": "Where required_contexts came from. 'live' is the platform's actual branch protection — the only reading that catches someone editing it in the UI. 'committed' is the repo's desired-state ruleset file, used when the API is unreadable: still worth having, but it reports intent rather than reality, and the collector says so rather than implying a check it did not make." + }, + "required_contexts": { + "type": "array", + "items": { "type": "string" }, + "description": "Status checks branch protection actually requires." + }, + "gate_jobs_present": { + "type": "array", + "items": { "type": "string" }, + "description": "Jobs in the repo's workflows that declare themselves merge gates." + }, + "findings": { + "type": "array", + "description": "Structured, not prose, so the collector can rank a fleet by severity without parsing English.", + "items": { + "type": "object", + "required": ["kind", "severity", "detail"], + "additionalProperties": false, + "properties": { + "kind": { + "type": "string", + "enum": [ + "gate_not_required", + "required_context_has_no_job", + "required_context_never_reported", + "override_without_ledger_entry", + "ledger_entry_expired", + "telemetry_incomplete" + ], + "description": "gate_not_required — a job calls itself a gate but nothing requires it. THE DISARMED CASE, and the reason this file exists. required_context_has_no_job — required but nothing produces it, so every PR waits forever. required_context_never_reported — required, produced, but silent all window: either nothing triggered it or it is quietly broken, and the two look the same from here. override_without_ledger_entry — a label used with no written justification. ledger_entry_expired — an acceptance past its review date. telemetry_incomplete — this file could not be fully built; treat its absences as unknown, never as clean." + }, + "severity": { "type": "string", "enum": ["high", "medium", "low"] }, + "context": { "type": "string" }, + "detail": { "type": "string" } + } + } + } + } + } + } +} diff --git a/kit/workflows/rails-telemetry.yml b/kit/workflows/rails-telemetry.yml new file mode 100644 index 0000000..fa26908 --- /dev/null +++ b/kit/workflows/rails-telemetry.yml @@ -0,0 +1,245 @@ +name: Rails Telemetry + +# ============================================================================= +# One week of THIS repo's gate outcomes, written down where a human can read it. +# ============================================================================= +# Overrides and gate outcomes were already recorded per repo — the eval-bypasses +# ledger, the accepted-risk labels, the PR timeline. Nothing aggregated them, so +# across a portfolio nobody could answer three questions: are the gates firing, +# are they being routinely waved through, and has one been quietly switched off. +# +# This workflow answers them for one repo and commits the answer as +# `.github/rails-telemetry.json`. The fleet collector (operator-side, not +# installed here) reads that file across every reachable repo. +# +# ── THE ONE THAT MATTERS: A GATE HAS TWO HALVES ────────────────────────────── +# There is the check, and there is the rule requiring it to pass. Delete the +# second — deliberately, or by a repo reorganisation nobody reviewed — and the +# check still runs, still reports, and looks completely normal on the PR page. +# But a red run merges anyway. From the outside, a disarmed gate is +# indistinguishable from a gate that simply never caught anything. +# +# No amount of counting finds that. Only comparing what IS REQUIRED against what +# EXISTS finds it, which is the `enforcement` section below and the reason this +# file is worth committing at all. +# +# ── CLIENT DATA BOUNDARY ───────────────────────────────────────────────────── +# NO EXTERNAL CALLS. Everything here queries this repo's own history through its +# own platform's API, using the token the platform already issued, and writes the +# result into this repo. Nothing leaves the client's tenancy. That is a deliberate +# constraint, not an implementation detail — it is what lets a client security +# team say yes to it, and anything that phones home would have to be argued for +# from scratch. +# +# ── NO ACTIVITY METRICS ────────────────────────────────────────────────────── +# Counts are per GATE, never per author. The standard forbids activity metrics in +# client materials and does not track them internally either (§9): agents inflate +# every one of them. `pull_requests_merged` is here solely as a denominator, so +# two overrides can be read against three PRs or against ninety — those are very +# different weeks. It is not a productivity number and must never be reported as +# one. +# +# ── PLACEHOLDER ─────────────────────────────────────────────────────────────── +# <> the protected branch this reports on (reference: main) +# ============================================================================= + +on: + schedule: + # Monday 06:00 UTC — before the working week, so the fleet report is ready when + # someone looks. Deliberately ahead of dependency-scan.yml's 07:00 so the two + # do not contend for the same runner minute. + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: write # it commits the report + actions: read # workflow run conclusions + pull-requests: read # merged PRs and their labels + checks: read + +concurrency: + group: rails-telemetry + cancel-in-progress: true + +jobs: + rails-telemetry: + name: rails-telemetry + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || vars.RAILS_TELEMETRY_ENABLED != 'false' + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build the telemetry report + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + WINDOW_DAYS: '7' + DEFAULT_BRANCH: 'main' # <> + run: | + set -euo pipefail + OUT=.github/rails-telemetry.json + mkdir -p .github + TO="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + FROM="$(date -u -d "-${WINDOW_DAYS} days" +%Y-%m-%dT%H:%M:%SZ)" + T="${RUNNER_TEMP}" + + # ── activity ──────────────────────────────────────────────────────── + # Workflow-run conclusions per job name, over the window. `gh api --paginate` + # so a busy week is not silently truncated at page one — a report that + # quietly drops half its data is worse than no report. + gh api --paginate "repos/${REPO}/actions/runs?created=>${FROM}&per_page=100" \ + --jq '.workflow_runs[]? | {id, name, conclusion}' > "${T}/runs.ndjson" || echo '' > "${T}/runs.ndjson" + + gh api --paginate "repos/${REPO}/pulls?state=closed&base=${DEFAULT_BRANCH}&per_page=100" \ + --jq '.[] | select(.merged_at != null) | {number, title, merged_at, labels: [.labels[].name]}' \ + > "${T}/prs_all.ndjson" || echo '' > "${T}/prs_all.ndjson" + jq -c --arg from "$FROM" 'select(.merged_at >= $from)' "${T}/prs_all.ndjson" > "${T}/prs.ndjson" || : > "${T}/prs.ndjson" + + # ── enforcement: what is ACTUALLY required ────────────────────────── + # Prefer the live ruleset: it is the only reading that catches someone + # editing branch protection in the UI. Fall back to the committed + # desired-state file when the API is unreadable — worth having, but it + # describes INTENT rather than reality, so the source is recorded and the + # collector is told which it got. Reporting intent as though it were + # reality is exactly the silent-green failure this file exists to expose. + ENF_SOURCE=unavailable + : > "${T}/required.txt" + if gh api "repos/${REPO}/rules/branches/${DEFAULT_BRANCH}" > "${T}/rules.json" 2>/dev/null; then + jq -r '.[]? | select(.type == "required_status_checks") + | .parameters.required_status_checks[]?.context' "${T}/rules.json" \ + | sort -u > "${T}/required.txt" || true + if [ -s "${T}/required.txt" ]; then ENF_SOURCE=live; fi + fi + if [ "$ENF_SOURCE" != "live" ] && [ -f .github/rulesets/branch-protection.json ]; then + jq -r '.rules[]? | select(.type == "required_status_checks") + | .parameters.required_status_checks[]?.context' .github/rulesets/branch-protection.json \ + | sort -u > "${T}/required.txt" || true + if [ -s "${T}/required.txt" ]; then ENF_SOURCE=committed; fi + fi + + # Jobs that CALL THEMSELVES merge gates. The marker comment is the same one + # check_standard.py keys on in the standard's own repo, so the two agree on + # what "declares itself a gate" means. + grep -rhoE '^\s{4}name:\s*\S+\s*#\s*<--\s*required status-check context' \ + .github/workflows/*.yml 2>/dev/null \ + | sed -E 's/^[[:space:]]*name:[[:space:]]*//; s/[[:space:]]*#.*$//' \ + | sort -u > "${T}/gatejobs.txt" || : > "${T}/gatejobs.txt" + + # ── ledgers ───────────────────────────────────────────────────────── + TODAY="$(date -u +%Y-%m-%d)" + ledger_json() { + local path="$1" + if [ ! -f "$path" ]; then + jq -n --arg p "$path" '{path:$p, present:false, active_entries:0, expired_entries:0}' + return + fi + # `expires: YYYY-MM-DD` lines, per the ledger templates' documented format. + local total expired + total="$(grep -cE '^[[:space:]]*expires:' "$path" || true)" + expired="$(grep -oE '^[[:space:]]*expires:[[:space:]]*[0-9]{4}-[0-9]{2}-[0-9]{2}' "$path" \ + | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' \ + | awk -v today="$TODAY" '$0 < today' | wc -l | tr -d ' ')" + jq -n --arg p "$path" --argjson t "${total:-0}" --argjson e "${expired:-0}" \ + '{path:$p, present:true, active_entries:($t - $e), expired_entries:$e}' + } + # Concatenated into ONE file rather than passed via process substitution: + # `--slurpfile` reads every JSON value in a file into an array, so this hands + # jq a flat array of ledgers directly. Process substitution would work on the + # runner but not everywhere the drill might be rehearsed by hand, and a report + # that only builds on one machine is a report nobody verifies. + ledger_json .github/eval-bypasses.md > "${T}/ledgers.ndjson" + ledger_json .github/dependency-exceptions.md >> "${T}/ledgers.ndjson" + + # ── assemble ──────────────────────────────────────────────────────── + jq -n \ + --arg repo "$REPO" --arg gen "$TO" --arg from "$FROM" --arg to "$TO" \ + --argjson days "${WINDOW_DAYS}" --arg enf_source "$ENF_SOURCE" \ + --slurpfile ledgers "${T}/ledgers.ndjson" \ + --rawfile runs_raw "${T}/runs.ndjson" \ + --rawfile prs_raw "${T}/prs.ndjson" \ + --rawfile required_raw "${T}/required.txt" \ + --rawfile gatejobs_raw "${T}/gatejobs.txt" ' + def lines(s): (s | split("\n") | map(select(length > 0))); + def objs(s): (s | split("\n") | map(select(length > 0) | fromjson)); + + (objs($runs_raw)) as $runs | + (objs($prs_raw)) as $prs | + (lines($required_raw)) as $required | + (lines($gatejobs_raw)) as $gatejobs | + + ([$runs[] | .name] | unique) as $names | + ($names | map(. as $n | ($runs | map(select(.name == $n))) as $r | { + context: $n, + runs: ($r | length), + passed: ($r | map(select(.conclusion == "success")) | length), + failed: ($r | map(select(.conclusion == "failure")) | length), + skipped: ($r | map(select(.conclusion == "skipped")) | length) + })) as $gates | + + (["accepted-risk:correctness","accepted-risk:dependency","gate-exception","risk:high"] + | map(. as $lbl | ($prs | map(select(.labels | index($lbl)))) as $hit | { + label: $lbl, + count: ($hit | length), + uses: ($hit | map({pr: .number, title: .title, merged_at: .merged_at})) + })) as $labels | + + # findings: the whole point. Structured so a fleet can be ranked without + # parsing prose. + ([ ($gatejobs - $required | map({ + kind: "gate_not_required", severity: "high", context: ., + detail: ("The job '" + . + "' declares itself a required status check, but branch protection does not require it. It runs, it reports, and a red run merges anyway — a disarmed gate.") + })), + ($required - $gatejobs | map({ + kind: "required_context_has_no_job", severity: "high", context: ., + detail: ("Branch protection requires '" + . + "', but no workflow job produces it. Every pull request waits on a check that will never report.") + })), + ($required | map(select(. as $c | ($gates | map(.context) | index($c)) == null)) | map({ + kind: "required_context_never_reported", severity: "medium", context: ., + detail: ("Required check '" + . + "' produced no runs in this window. Either nothing triggered it, or it is quietly broken — from here those look the same.") + })), + (if $enf_source == "unavailable" then [{ + kind: "telemetry_incomplete", severity: "high", + detail: "Branch protection could not be read, live or committed, so no enforcement check was possible. Treat this repo's gate status as UNKNOWN, never as clean." + }] elif $enf_source == "committed" then [{ + kind: "telemetry_incomplete", severity: "low", + detail: "Branch protection was read from the committed ruleset, not the live API, so this reports intent rather than what the platform is actually enforcing. A change made in the UI would not show up here." + }] else [] end), + ($ledgers | map(select(.expired_entries > 0)) | map({ + kind: "ledger_entry_expired", severity: "medium", + detail: (.path + " has " + (.expired_entries|tostring) + " acceptance(s) past their review date. An acceptance that outlives its expiry has quietly become permanent.") + })) + ] | flatten) as $findings | + + { + schema_version: 1, + repo: $repo, + generated_at: $gen, + window: { days: $days, from: $from, to: $to }, + activity: { gates: $gates, pull_requests_merged: ($prs | length) }, + overrides: { labels: $labels, ledgers: $ledgers }, + enforcement: { + source: $enf_source, + required_contexts: $required, + gate_jobs_present: $gatejobs, + findings: $findings + } + }' > "$OUT" + + echo "Wrote $OUT" + jq '{repo, window: .window.days, gates: (.activity.gates|length), findings: (.enforcement.findings|length), source: .enforcement.source}' "$OUT" + + - name: Commit the report + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add .github/rails-telemetry.json + # Nothing changed is the common case on a quiet week, and an empty commit + # every Monday is noise in the history everyone learns to scroll past. + if git diff --cached --quiet; then + echo "No change since the last report." + exit 0 + fi + git commit -m "chore: rails telemetry $(date -u +%Y-%m-%d)" + git push diff --git a/scripts/collect_rails_telemetry.py b/scripts/collect_rails_telemetry.py new file mode 100644 index 0000000..2b78a7a --- /dev/null +++ b/scripts/collect_rails_telemetry.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Read every repo's committed rails-telemetry.json and report the fleet. + +Overrides and gate outcomes were already recorded per repo. Nothing aggregated them, so across +a portfolio nobody could answer three questions: are the gates firing, are they being routinely +waved through, and has one been quietly switched off. + +The third is the one that needs a tool. A gate has two halves — the check, and the rule that +requires it to pass. Delete the rule and the check still runs, still reports, and looks entirely +normal; a red run just merges anyway. From outside that repo, a disarmed gate and a gate that +never caught anything produce identical evidence. `check_standard.py` catches this in OUR repo; +nothing caught it in a client's, which is what the telemetry file and this script exist for. + +This is OPERATOR TOOLING and is deliberately NOT part of `kit/` — it is never installed into a +client repo. It reads what the installed `rails-telemetry` workflow committed. + + python scripts/collect_rails_telemetry.py [...] # report; exit 1 on a high finding + python scripts/collect_rails_telemetry.py --root ../clients # every repo under a directory + python scripts/collect_rails_telemetry.py --root ../clients --json + +Accepts a repo root (the file is found at .github/rails-telemetry.json) or the JSON file itself. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path + +SCHEMA_VERSION = 1 +REPORT_REL = Path(".github") / "rails-telemetry.json" + +# Ranked worst-first. `gate_not_required` leads because it is the failure the whole pipeline of +# work exists to surface: the repo looks green precisely because nothing is being enforced. +SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2} + +# A report older than this is describing a week nobody has looked at since. Not an error — repos +# go quiet legitimately — but reporting it as current would be the same lie the telemetry exists +# to prevent. +STALE_AFTER_DAYS = 21 + + +@dataclass +class RepoReport: + """One repo's telemetry, or the reason it could not be read.""" + + source: Path + repo: str = "?" + findings: list[dict] = field(default_factory=list) + overrides: int = 0 + merged: int = 0 + gates: int = 0 + enforcement_source: str = "unavailable" + generated_at: str | None = None + error: str | None = None + + @property + def worst(self) -> int: + return min((SEVERITY_ORDER.get(f.get("severity", "low"), 2) for f in self.findings), + default=3) + + @property + def high_findings(self) -> list[dict]: + return [f for f in self.findings if f.get("severity") == "high"] + + +def _resolve(target: str) -> Path: + """A repo root, or the report file itself. Both are natural things to type.""" + p = Path(target) + return p if p.is_file() else p / REPORT_REL + + +def load(target: str) -> RepoReport: + """Parse one report. An unreadable report is a RESULT, never an exception. + + A collector that dies on the first malformed file tells you nothing about the other + forty repos, and the one that failed is usually the one worth looking at. + """ + path = _resolve(target) + rep = RepoReport(source=path) + + if not path.is_file(): + rep.error = "no rails-telemetry.json — the workflow is not installed, has never run, or is disabled" + return rep + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + rep.error = f"unreadable: {exc}" + return rep + if not isinstance(data, dict): + rep.error = "not a JSON object" + return rep + + version = data.get("schema_version") + if version != SCHEMA_VERSION: + # Refuse rather than guess. Reading a v2 file with v1 assumptions would produce numbers + # that look fine and mean something else, which is worse than an obvious gap. + rep.error = f"schema_version {version!r}, expected {SCHEMA_VERSION} — not read" + return rep + + rep.repo = data.get("repo", "?") + rep.generated_at = data.get("generated_at") + enforcement = data.get("enforcement") or {} + rep.enforcement_source = enforcement.get("source", "unavailable") + rep.findings = list(enforcement.get("findings") or []) + + activity = data.get("activity") or {} + rep.gates = len(activity.get("gates") or []) + rep.merged = activity.get("pull_requests_merged", 0) + + overrides = data.get("overrides") or {} + rep.overrides = sum(entry.get("count", 0) for entry in (overrides.get("labels") or [])) + + if (stale := _staleness_days(rep.generated_at)) is not None and stale > STALE_AFTER_DAYS: + rep.findings.append({ + "kind": "telemetry_incomplete", + "severity": "medium", + "detail": f"last report is {stale} days old — this repo's gate status is not being watched", + }) + return rep + + +def _staleness_days(generated_at: str | None) -> int | None: + if not generated_at: + return None + try: + when = datetime.fromisoformat(generated_at.replace("Z", "+00:00")) + except ValueError: + return None + return (datetime.now(timezone.utc) - when).days + + +def collect(targets: list[str]) -> list[RepoReport]: + """Worst first — the point of a fleet view is that you read the top and stop.""" + return sorted((load(t) for t in targets), + key=lambda r: (0 if r.error else 1, r.worst, r.repo)) + + +def discover(root: str) -> list[str]: + """Every git repo directly under `root`, reporting or not. + + Deliberately keyed on `.git` rather than on the report file. Discovering by report would + only ever find repos that are ALREADY reporting, so a repo where the workflow was never + installed — or was installed and has never run — would be silently absent from the fleet + view. That is precisely the failure this tool exists to prevent, reproduced one level up: + a clean-looking report whose cleanliness comes from not having looked. + + Sorted so output is stable run to run. + """ + return sorted(str(p.parent) for p in Path(root).glob("*/.git")) + + +def render(reports: list[RepoReport]) -> str: + if not reports: + return "No repos given. Pass repo paths, or --root to scan for them." + + lines: list[str] = ["Rails telemetry — fleet view", ""] + + unreadable = [r for r in reports if r.error] + disarmed = [r for r in reports if not r.error and r.high_findings] + + # The headline is a claim about coverage, and it is deliberately careful: a repo we could + # not read is UNKNOWN, not healthy. Folding those into "clean" is how a fleet report starts + # lying by omission. + lines.append( + f" {len(reports)} repo(s): {len(disarmed)} with high-severity findings, " + f"{len(unreadable)} not reporting, " + f"{len(reports) - len(disarmed) - len(unreadable)} clean" + ) + lines.append("") + + for rep in reports: + if rep.error: + lines.append(f" [NOT REPORTING] {rep.source.parent.parent.name or rep.source}") + lines.append(f" {rep.error}") + lines.append("") + continue + + mark = "HIGH" if rep.high_findings else ("...." if not rep.findings else "note") + lines.append(f" [{mark}] {rep.repo}") + detail = (f" gates seen {rep.gates} · merged PRs {rep.merged} · " + f"overrides {rep.overrides} · enforcement read from {rep.enforcement_source}") + lines.append(detail) + for finding in sorted(rep.findings, + key=lambda f: SEVERITY_ORDER.get(f.get("severity", "low"), 2)): + ctx = finding.get("context") + head = f" - [{finding.get('severity')}] {finding.get('kind')}" + lines.append(f"{head} ({ctx})" if ctx else head) + lines.append(f" {finding.get('detail', '')}") + lines.append("") + + if disarmed or unreadable: + lines.append(" A gate that is present but not required reports normally and blocks nothing.") + lines.append(" A repo that is not reporting is UNKNOWN, not clean.") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("targets", nargs="*", help="repo roots, or rails-telemetry.json paths") + parser.add_argument("--root", help="scan this directory for repos that have a report") + parser.add_argument("--json", action="store_true", help="machine-readable output") + args = parser.parse_args(argv) + + targets = list(args.targets) + if args.root: + found = discover(args.root) + if not found and not targets: + # Deliberately distinct from the no-arguments error. "Nothing there" and "you + # forgot the flag" have different fixes, and sending someone to re-type a flag + # they already typed is how a tool gets a reputation for being broken. + print(f"No rails-telemetry.json found under {args.root!r}.\n" + f"Expected each repo at //{REPORT_REL.as_posix()}.\n" + f"Either the workflow is not installed in those repos, or it has not run yet — " + f"which is itself worth knowing: an unreported repo is unknown, not clean.", + file=sys.stderr) + return 1 + targets.extend(found) + if not targets: + parser.error("give at least one repo path, or --root ") + + reports = collect(targets) + + if args.json: + print(json.dumps([{ + "repo": r.repo, "source": str(r.source), "error": r.error, + "enforcement_source": r.enforcement_source, + "overrides": r.overrides, "merged": r.merged, + "findings": r.findings, + } for r in reports], indent=2)) + else: + print(render(reports)) + + # Non-zero when something needs a person: a high finding, or a repo we could not read. + # "Could not read" counts deliberately — silence is not the same as clean, and a collector + # that exits 0 on a fleet it failed to inspect is a green light nobody earned. + return 1 if any(r.error or r.high_findings for r in reports) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_collect_rails_telemetry.py b/scripts/tests/test_collect_rails_telemetry.py new file mode 100644 index 0000000..9e43ccd --- /dev/null +++ b/scripts/tests/test_collect_rails_telemetry.py @@ -0,0 +1,228 @@ +"""Tests for the fleet telemetry collector. + +Same discipline as test_check_standard.py: inject real drift and assert it is caught. A +collector that has silently stopped noticing disarmed gates is worse than no collector — it +reports a clean fleet forever and everyone stops looking. These tests are what make its green +mean anything. +""" + +from __future__ import annotations + +import json +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import collect_rails_telemetry as ct # noqa: E402 + +SCHEMA = json.loads( + (Path(__file__).resolve().parents[2] / "kit/profile/rails-telemetry.schema.json") + .read_text(encoding="utf-8") +) + + +def report(**overrides) -> dict: + """A well-formed, clean report. Tests mutate one thing so the failure is unambiguous.""" + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + base = { + "schema_version": 1, + "repo": "acme/claims", + "generated_at": now, + "window": {"days": 7, "from": now, "to": now}, + "activity": { + "gates": [{"context": "build-and-test", "runs": 3, "passed": 3, "failed": 0}], + "pull_requests_merged": 4, + }, + "overrides": { + "labels": [{"label": "accepted-risk:correctness", "count": 0, "uses": []}], + "ledgers": [{"path": ".github/eval-bypasses.md", "present": True, + "active_entries": 0, "expired_entries": 0}], + }, + "enforcement": { + "source": "live", + "required_contexts": ["build-and-test"], + "gate_jobs_present": ["build-and-test"], + "findings": [], + }, + } + base.update(overrides) + return base + + +def write(tmp_path: Path, name: str, data: dict | str) -> Path: + repo = tmp_path / name + (repo / ".github").mkdir(parents=True) + path = repo / ".github" / "rails-telemetry.json" + path.write_text(data if isinstance(data, str) else json.dumps(data), encoding="utf-8") + return repo + + +# --------------------------------------------------------------------------- the schema itself + +def test_schema_matches_the_shape_the_workflows_emit(): + """The schema and the fixture must agree, or the schema documents a file nobody writes.""" + props = SCHEMA["properties"] + assert set(SCHEMA["required"]) <= set(report()) + assert props["schema_version"]["const"] == ct.SCHEMA_VERSION + for section in ("activity", "overrides", "enforcement"): + assert set(props[section]["required"]) <= set(report()[section]) + + +def test_every_finding_kind_the_collector_ranks_exists_in_the_schema(): + """A kind the workflows can emit but the schema does not name would be silently untyped.""" + kinds = set(SCHEMA["properties"]["enforcement"]["properties"]["findings"] + ["items"]["properties"]["kind"]["enum"]) + assert "gate_not_required" in kinds # the disarmed case + assert "telemetry_incomplete" in kinds # the collector appends this one itself + assert set(SCHEMA["properties"]["enforcement"]["properties"]["source"]["enum"]) == { + "live", "committed", "unavailable"} + + +# --------------------------------------------------------------------------- reading + +def test_clean_repo_reports_clean(tmp_path): + repo = write(tmp_path, "clean", report()) + assert ct.main([str(repo)]) == 0 + + +def test_disarmed_gate_is_high_and_fails_the_run(tmp_path): + """THE case this whole mechanism exists for: the gate runs, reports, and enforces nothing.""" + data = report() + data["enforcement"]["required_contexts"] = [] # nobody requires it any more + data["enforcement"]["findings"] = [{ + "kind": "gate_not_required", "severity": "high", "context": "build-and-test", + "detail": "declares itself required; branch protection does not require it", + }] + repo = write(tmp_path, "disarmed", data) + + rep = ct.load(str(repo)) + assert rep.high_findings + assert ct.main([str(repo)]) == 1 + + +def test_missing_report_is_unknown_not_clean(tmp_path): + """Silence must never be read as health — the whole point of the coverage line.""" + empty = tmp_path / "silent" + empty.mkdir() + rep = ct.load(str(empty)) + assert rep.error is not None + assert ct.main([str(empty)]) == 1 + + +def test_unknown_schema_version_is_refused_not_guessed(tmp_path): + """A v2 file read with v1 assumptions yields numbers that look fine and mean something else.""" + repo = write(tmp_path, "future", report(schema_version=2)) + rep = ct.load(str(repo)) + assert rep.error is not None and "schema_version" in rep.error + assert not rep.findings + + +def test_malformed_json_does_not_abort_the_fleet(tmp_path): + """One broken file must not hide the other repos — usually it is the interesting one.""" + bad = write(tmp_path, "broken", "{not json") + good = write(tmp_path, "fine", report(repo="acme/other")) + reports = ct.collect([str(bad), str(good)]) + assert len(reports) == 2 + assert any(r.error for r in reports) + assert any(r.repo == "acme/other" for r in reports) + + +def test_stale_report_is_flagged(tmp_path): + old = (datetime.now(timezone.utc) - timedelta(days=ct.STALE_AFTER_DAYS + 5)) + repo = write(tmp_path, "stale", report(generated_at=old.strftime("%Y-%m-%dT%H:%M:%SZ"))) + rep = ct.load(str(repo)) + assert any(f["kind"] == "telemetry_incomplete" for f in rep.findings) + + +def test_committed_enforcement_source_is_surfaced(tmp_path): + """Intent read from a file is not proof of what the platform enforces; say which it was.""" + data = report() + data["enforcement"]["source"] = "committed" + repo = write(tmp_path, "intent", data) + assert ct.load(str(repo)).enforcement_source == "committed" + + +# --------------------------------------------------------------------------- fleet behaviour + +def test_worst_repo_sorts_first(tmp_path): + clean = write(tmp_path, "a-clean", report(repo="acme/a")) + data = report(repo="acme/z") + data["enforcement"]["findings"] = [ + {"kind": "gate_not_required", "severity": "high", "context": "grader", "detail": "x"}] + bad = write(tmp_path, "z-bad", data) + + ordered = ct.collect([str(clean), str(bad)]) + # Unreadable first, then by severity — a fleet view is read from the top. + assert ordered[0].repo == "acme/z" + + +def test_empty_root_is_distinguished_from_a_missing_argument(tmp_path, capsys): + """A root with no reports is a FINDING, not a usage error — and must not exit 0.""" + (tmp_path / "some-repo").mkdir() + assert ct.main(["--root", str(tmp_path)]) == 1 + assert "not clean" in capsys.readouterr().err + + +def test_discover_finds_repos_not_merely_reports(tmp_path): + """A repo that never ran the workflow must still appear — absent is the finding.""" + for name in ("one", "two"): + write(tmp_path, name, report(repo=f"acme/{name}")) + (tmp_path / name / ".git").mkdir() + silent = tmp_path / "never-ran" # a real repo, no report + (silent / ".git").mkdir(parents=True) + (tmp_path / "not-a-repo").mkdir() # not a repo at all — correctly ignored + + found = ct.discover(str(tmp_path)) + assert len(found) == 3 + assert any("never-ran" in f for f in found) + + reports = ct.collect(found) + assert any(r.error and "never-ran" in str(r.source) for r in reports) + + +def test_silent_repo_makes_the_fleet_view_fail(tmp_path): + """The whole promise: silence is reported as unknown and exits non-zero.""" + write(tmp_path, "good", report()) + (tmp_path / "good" / ".git").mkdir() + (tmp_path / "silent" / ".git").mkdir(parents=True) + assert ct.main(["--root", str(tmp_path)]) == 1 + + +def test_overrides_are_totalled_across_labels(tmp_path): + data = report() + data["overrides"]["labels"] = [ + {"label": "accepted-risk:correctness", "count": 2, "uses": []}, + {"label": "gate-exception", "count": 1, "uses": []}, + ] + repo = write(tmp_path, "overridden", data) + assert ct.load(str(repo)).overrides == 3 + + +def test_render_names_the_disarmed_repo(tmp_path): + data = report(repo="acme/quiet") + data["enforcement"]["findings"] = [ + {"kind": "gate_not_required", "severity": "high", "context": "security-review", + "detail": "not required"}] + repo = write(tmp_path, "quiet", data) + out = ct.render(ct.collect([str(repo)])) + assert "acme/quiet" in out and "gate_not_required" in out and "security-review" in out + + +def test_report_file_path_accepted_directly(tmp_path): + repo = write(tmp_path, "direct", report()) + assert ct.load(str(repo / ".github" / "rails-telemetry.json")).error is None + + +@pytest.mark.parametrize("severity,expected_exit", [("high", 1), ("medium", 0), ("low", 0)]) +def test_only_high_findings_fail_the_run(tmp_path, severity, expected_exit): + """Medium and low are for reading, not for stopping a person's morning.""" + data = report() + data["enforcement"]["findings"] = [ + {"kind": "required_context_never_reported", "severity": severity, + "context": "grader", "detail": "quiet"}] + repo = write(tmp_path, f"sev-{severity}", data) + assert ct.main([str(repo)]) == expected_exit