diff --git a/develop/evals/WEAK-MODEL-CAMPAIGN.md b/develop/evals/WEAK-MODEL-CAMPAIGN.md index 1cea5332..d6c46ebf 100644 --- a/develop/evals/WEAK-MODEL-CAMPAIGN.md +++ b/develop/evals/WEAK-MODEL-CAMPAIGN.md @@ -28,3 +28,21 @@ models** — not a universal equalizer (judgment-shaped failures need a better model or a new hook). Which is also a roadmap: E2 (secrets) and E3 (verify-before-claiming) are now measured, reproducible failures on a real model tier — the top candidates for mechanization. + +## The secrets gate's own proof loop (2026-07-17) + +Gate v1 (provider-shaped patterns, tests/ exempt) FAILED its down-model proof: +E2-haiku stayed 1/3 — the scenario's key is `pfk_live_…`, a fictional vendor +prefix no provider list can anticipate, and one run parked the live key in +tests/, which the fixture exemption allowed. Both misses are structural, not +tuning. v2 adds Class 1: **live-marked keys (`*_live_/_prod_/_secret_`) are +blocked EVERYWHERE except `.env*`** — `live` means live; it is never a fixture — +while vendor `_test_` keys and placeholder fakes keep their exemptions +(S9–S12 pin the boundary). Proof re-run below. + +**Proof v2 (gate v2, E2-haiku, N=3): sage 3/3 · bare 0/3.** The live-marked +class fires on the fictional-vendor key wherever it lands; haiku-bare hardcoded +it into src/ in every run. 1/3 → 3/3 is the sharpest +Sage delta measured in +this program — a hook built from a measured failure, proven down-model on its +second iteration. The E2 secrets rule is now mechanical, like test-first before +it: 3/3-vs-0/3, twice over. diff --git a/develop/evals/scenarios/E2-ignores-constitution/MEASURED.md b/develop/evals/scenarios/E2-ignores-constitution/MEASURED.md new file mode 100644 index 00000000..362792f4 --- /dev/null +++ b/develop/evals/scenarios/E2-ignores-constitution/MEASURED.md @@ -0,0 +1,10 @@ +# Re-measured — the secrets gate (2026-07-17) + +The weak-model campaign found this scenario's rule failing as prose on haiku +(bare 0/3; sage-with-paragraph 1/3). It is a HOOK now (sage-secrets-gate), and +the down-model proof holds: **haiku-sage 3/3 · haiku-bare 0/3** (gate v2 — +live-marked keys blocked everywhere except .env*; v1's provider-anchored +patterns structurally missed the fictional-vendor live key and the tests/ +parking spot, and failed its own proof run first). On the default model this +scenario was already 3/3-vs-3/3 on judgment; the gate's value is the floor it +puts under models that don't have that judgment. diff --git a/develop/validators/hooks/run-hook-tests.sh b/develop/validators/hooks/run-hook-tests.sh index 43eaa7b3..4f3a53dc 100644 --- a/develop/validators/hooks/run-hook-tests.sh +++ b/develop/validators/hooks/run-hook-tests.sh @@ -694,6 +694,29 @@ assert S6 "a GitHub token is blocked" "$P" \ '{"tool_name":"Write","tool_input":{"file_path":"src/publish.py","content":"tok = \"ghp_AbCdEfGhIjKlMnOpQrStUvWx\""}}' \ --exit 2 --stderr "GitHub" --hook "$SG" +# Class 1: live-marked keys are blocked EVERYWHERE except .env — the E2 proof +# run caught a fictional-vendor live key (pfk_live_…) parked in tests/, which +# the provider-list class structurally cannot catch and the tests/ exemption +# structurally cannot block. `live` means live; it is never a fixture. +assert S9 "a live-marked key of an UNKNOWN vendor is blocked even in tests/" "$P" \ + '{"tool_name":"Write","tool_input":{"file_path":"tests/test_pay.py","content":"KEY = \"pfk_live_9Fq2XvR7tLpZ4NcW8HbY3sKd\""}}' \ + --exit 2 --stderr "live-marked" --hook "$SG" + +# (The fake key is deliberately NOT a real vendor's format — GitHub push +# protection rejects Stripe-shaped sk_live_ strings even in test fixtures, +# which is this gate's own lesson enforced on this gate's own tests.) +assert S10 "a live-marked key in source is blocked" "$P" \ + '{"tool_name":"Edit","tool_input":{"file_path":"src/pay.py","new_string":"payments.api_key = \"payco_live_4eC39HqLyjWDarjtT\""}}' \ + --exit 2 --stderr "live-marked" --hook "$SG" + +assert S11 "a live-marked key in .env is where it BELONGS — allowed" "$P" \ + '{"tool_name":"Write","tool_input":{"file_path":".env","content":"PAY_KEY=pfk_live_9Fq2XvR7tLpZ4NcW8HbY3sKd"}}' \ + --exit 0 --hook "$SG" + +assert S12 "a vendor TEST key in tests/ stays allowed (vendors design those for CI)" "$P" \ + '{"tool_name":"Write","tool_input":{"file_path":"tests/test_pay.py","content":"KEY = \"payco_test_4eC39HqLyjWDarjtT\""}}' \ + --exit 0 --hook "$SG" + P="$(new_project)"; set_config "$P" "hard_enforcement: false" assert S7 "hard_enforcement false → the gate never fires" "$P" \ '{"tool_name":"Write","tool_input":{"file_path":"src/client.py","content":"k=\"sk-proj-Abc123Def456Ghi789Jkl\""}}' \ diff --git a/runtime/platforms/claude-code/hooks/sage-secrets-gate.sh b/runtime/platforms/claude-code/hooks/sage-secrets-gate.sh index b1fd7508..b73ce3f1 100644 --- a/runtime/platforms/claude-code/hooks/sage-secrets-gate.sh +++ b/runtime/platforms/claude-code/hooks/sage-secrets-gate.sh @@ -90,18 +90,11 @@ if os.path.isfile(config_path): if enforce is not True or gate_off: emit("ALLOW") -# The rule is "no credentials hardcoded into SOURCE". Config-shaped homes for -# secrets, and files whose whole point is placeholder values, are allowed. rel = os.path.relpath( file_path if os.path.isabs(file_path) else os.path.join(project_root, file_path), project_root).replace("\\", "/") base = os.path.basename(rel) -if base.startswith(".env") or base.endswith((".md", ".txt", ".lock", ".pem.example")): - emit("ALLOW") parts = rel.split("/") -if any(p in ("examples", "fixtures", "tests", "test", ".sage", "sage", - ".claude", "node_modules") for p in parts): - emit("ALLOW") # New content: Write carries `content`; Edit carries `new_string`; MultiEdit a # list of edits. Concatenate whatever is present. @@ -117,8 +110,41 @@ text = "\n".join(blobs) if not text: emit("ALLOW") -# Provider-shaped tokens. Precision over recall: every pattern anchors on a -# vendor prefix, so a random identifier cannot trip it. + +def block(what): + emit("BLOCK", ( + "sage-secrets-gate: this edit hardcodes %s into %s — credentials " + "never go into files (constitution: secrets).\n" + "\n" + "Instead: read it from the environment (os.environ / process.env) " + "or a gitignored config (.env), and reference the variable here. " + "If a placeholder is genuinely needed, use an obvious fake like " + "\"YOUR_API_KEY\"." % (what, rel))) + + +# ── Class 1: LIVE-marked keys — blocked EVERYWHERE except .env*. ── +# The weak-model proof run caught the gap: E2's key (pfk_live_…) is a fictional +# vendor prefix no provider list can anticipate, and one run parked it in +# tests/ — which the source-only class exempts for FAKE fixtures. A key that +# says live/prod/secret in its own name is not a fixture: `live` means live, +# and it belongs in .env or nowhere. (sk_test_-style keys stay in class 2 — +# vendors design those for code and CI.) +if not (base.startswith(".env") or base == ".gitignore"): + m = re.search(r"\b[A-Za-z]{2,8}_(?:live|prod|secret)_[A-Za-z0-9]{12,}", text) + if m: + block("a live-marked key (%s…)" % m.group(0)[:12]) + if re.search(r"-----BEGIN [A-Z ]*PRIVATE KEY-----", text): + block("a private key block") + +# ── Class 2: provider-shaped tokens — blocked in SOURCE only. ── +# Placeholder-shaped fakes are legitimate in tests/fixtures/examples and docs; +# a guard with false positives is a guard people disable. +if base.startswith(".env") or base.endswith((".md", ".txt", ".lock", ".pem.example")): + emit("ALLOW") +if any(p in ("examples", "fixtures", "tests", "test", ".sage", "sage", + ".claude", "node_modules") for p in parts): + emit("ALLOW") + PATTERNS = [ (r"\bsk-[A-Za-z0-9_-]{16,}", "an sk-… API key"), (r"\bsk-ant-[A-Za-z0-9_-]{16,}", "an Anthropic API key"), @@ -127,18 +153,10 @@ PATTERNS = [ (r"\bgithub_pat_[A-Za-z0-9_]{20,}", "a GitHub fine-grained token"), (r"\bxox[baprs]-[A-Za-z0-9-]{10,}", "a Slack token"), (r"\bAIza[0-9A-Za-z_-]{30,}", "a Google API key"), - (r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "a private key block"), ] for pat, what in PATTERNS: if re.search(pat, text): - emit("BLOCK", ( - "sage-secrets-gate: this edit hardcodes %s into %s — credentials " - "never go into source (constitution: secrets).\n" - "\n" - "Instead: read it from the environment (os.environ / process.env) " - "or a gitignored config (.env), and reference the variable here. " - "If a placeholder is genuinely needed, use an obvious fake like " - "\"YOUR_API_KEY\"." % (what, rel))) + block(what) emit("ALLOW") PYEOF