diff --git a/README.md b/README.md index 8b79b96f..157a67ac 100644 --- a/README.md +++ b/README.md @@ -189,7 +189,7 @@ Cold start 50–200 ms, warm start under 5 ms. Nothing stays resident. Full flow --- -## 86 ready-made cuecards +## 87 ready-made cuecards cue ships with pre-built profiles for common stacks and workflows. A taste: @@ -208,7 +208,7 @@ cue ships with pre-built profiles for common stacks and workflows. A taste: | 🏢 **agency** | 63 delegatable subagents — design, sales, product, PM, QA | ```bash -cue list # see all 86 +cue list # see all 87 cue auto-detect # suggest the right one for the current directory cue use # pin it ``` diff --git a/llms.txt b/llms.txt index 5d15c9dc..9da097a4 100644 --- a/llms.txt +++ b/llms.txt @@ -7,7 +7,7 @@ Key facts an LLM should know up front: - Install: `npm install -g cue-ai`. Pin a profile to a repo: `echo > .cue.profile`. Then type `claude` or `codex` — the shim launches the real binary with the scoped runtime. - Architecture: resolve → materialize → exec. Hash-cached: warm start <5 ms. No daemon, no background process. -- Profiles inherit from a `core` baseline. 86 profiles ship by default (backend, frontend, marketing, cybersecurity, medusa-dev, …). +- Profiles inherit from a `core` baseline. 87 profiles ship by default (backend, frontend, marketing, cybersecurity, medusa-dev, …). - Ten agents supported via `cue materialize `: claude, codex, cursor, cline, gemini, copilot, windsurf, roo, amp, aider. - License: MIT. Repo: https://github.com/opencue/cuecards. Package: https://www.npmjs.com/package/cue-ai. diff --git a/resources/hooks/liedetector-tag-density.sh b/resources/hooks/liedetector-tag-density.sh index 41fa5517..58a99705 100755 --- a/resources/hooks/liedetector-tag-density.sh +++ b/resources/hooks/liedetector-tag-density.sh @@ -59,14 +59,18 @@ TAGS = ("VERIFIED", "KNOWN", "INFERRED", "ASSUMED", "GUESSED", "STALE", "UNKNOWN", "CORRECTION") TAG_RE = re.compile(r"\[(?:%s)[^\]]*\]" % "|".join(TAGS)) -# The protocol requires a ~N% on every yellow and orange tag, drawn from that -# tier's ladder. Yellow spans ~50-85%, orange ~20-45%, so the ladders don't -# overlap each other or green (>=90%). +# The protocol requires a ~N% on every yellow and orange tag, snapped to a +# 5-point raster. Yellow spans ~50-85%, orange ~20-45%, so the tiers don't +# overlap each other or green (>=90%). The raster is deliberately coarser than +# the model's apparent precision: self-reported LLM confidence is miscalibrated +# in absolute terms, so ~67% would read as a measurement where none happened. +# 14 steps is enough to ORDER claims against each other, which is all the number +# is for. LADDER = { - "INFERRED": {"50", "60", "70", "80"}, - "ASSUMED": {"50", "60", "70", "80"}, - "GUESSED": {"20", "30", "40"}, - "STALE": {"20", "30", "40"}, + "INFERRED": {"50", "55", "60", "65", "70", "75", "80", "85"}, + "ASSUMED": {"50", "55", "60", "65", "70", "75", "80", "85"}, + "GUESSED": {"20", "25", "30", "35", "40", "45"}, + "STALE": {"20", "25", "30", "35", "40", "45"}, } CAL_RE = re.compile(r"\[(%s)([^\]]*)\]" % "|".join(LADDER)) PCT_RE = re.compile(r"~\s*(\d+)\s*%") @@ -142,8 +146,8 @@ if missing or offladder: if offladder: parts.append("%d off-ladder (%s)" % (len(offladder), ", ".join(sorted(set(offladder))))) - print("liedetector: %s. Yellow ([INFERRED]/[ASSUMED]) takes ~50/60/70/80%%, " - "orange ([GUESSED]/[STALE]) takes ~20/30/40%% — nothing else. " + print("liedetector: %s. Yellow ([INFERRED]/[ASSUMED]) takes ~50-85%%, " + "orange ([GUESSED]/[STALE]) takes ~20-45%%, both in 5-point steps. " "Skip with [skip-tag-density]." % "; ".join(parts)) sys.exit(0) diff --git a/resources/personas/integrity-protocol-compact.md b/resources/personas/integrity-protocol-compact.md index 28494fe6..39f1d113 100644 --- a/resources/personas/integrity-protocol-compact.md +++ b/resources/personas/integrity-protocol-compact.md @@ -14,7 +14,7 @@ Applies to every response. Flag uncertainty *before* the claim, never bury hedge Pick the *most specific* tag and **downgrade when in doubt** — false confidence hurts more than false hedging. -**Every yellow and orange tag carries a `~N%`** drawn from its tier's ladder — yellow `~50/60/70/80%`, orange `~20/30/40%`, nothing else. A bare `[INFERRED]` or `[GUESSED]` is a protocol violation; so is `~67%` (false precision), `~90%` on yellow (green's range), or `~50%` on orange (yellow's). Skip the % on green and red — the tier already says it. Can't pick a value? You're in the wrong tier: downgrade. The number orders claims *within one response*; it is not a calibrated absolute probability. +**Every yellow and orange tag carries a `~N%`** on a 5-point raster — yellow `~50%` to `~85%`, orange `~20%` to `~45%`, nothing between the steps. A bare `[INFERRED]` or `[GUESSED]` is a protocol violation; so is `~67%` (false precision), `~90%` on yellow (green's range), or `~50%` on orange (yellow's). Skip the % on green and red — the tier already says it. Can't pick a value? You're in the wrong tier: downgrade. The number orders claims *within one response*; it is not a calibrated absolute probability, so don't reach past the raster for digits you didn't measure. **Confidence audit** when a response has 2+ yellow-or-worse claims, recommends an action, or summarizes external evidence: end with Evidence quality (Strong/Moderate/Weak/Insufficient), the biggest confidence limiter, and one thing to verify externally. diff --git a/resources/personas/integrity-protocol.md b/resources/personas/integrity-protocol.md index 72ae0561..63af819c 100644 --- a/resources/personas/integrity-protocol.md +++ b/resources/personas/integrity-protocol.md @@ -25,9 +25,10 @@ Rewritten by Claude (Opus 4.7) from your hallucination-reduction draft. Applies - 🔴 `[UNKNOWN]` — outside my reliable knowledge. I'm saying so instead of fabricating an answer. Hand off to a search or to the user. **Required percentage calibration on yellow/orange tags.** Every yellow and orange tag carries a `~N%` drawn from its tier's ladder, with a tilde to signal it's a rough self-calibration rather than a true probability: `🟡 [INFERRED ~80%]`, `🟠 [GUESSED ~30%]`. Rules: - - Yellow (`[INFERRED]`, `[ASSUMED]`) → one of `~50%` `~60%` `~70%` `~80%` - - Orange (`[GUESSED]`, `[STALE]`) → one of `~20%` `~30%` `~40%` - - Nothing else on the ladder. Never `~67%` or `~73%` (false precision), never `~90%` on yellow (that's green's range) or `~50%` on orange (that's yellow's) + - Yellow (`[INFERRED]`, `[ASSUMED]`) → `~50%` to `~85%` in 5-point steps: `~50%` `~55%` `~60%` `~65%` `~70%` `~75%` `~80%` `~85%` + - Orange (`[GUESSED]`, `[STALE]`) → `~20%` to `~45%` in 5-point steps: `~20%` `~25%` `~30%` `~35%` `~40%` `~45%` + - Nothing between the steps. Never `~67%` or `~73%` (false precision), never `~90%` on yellow (that's green's range) or `~50%` on orange (that's yellow's) + - The raster is coarser than your apparent precision on purpose. Self-reported confidence is miscalibrated in absolute terms, so a digit you didn't measure reads as a measurement. 14 steps is enough to *order* claims, which is all the number does - A bare `[INFERRED]` / `[ASSUMED]` / `[GUESSED]` / `[STALE]` is a protocol violation - Always prefix `~` so the reader knows it's an estimate - Skip the % on green and red — the tier already says it diff --git a/src/commands/summon.test.ts b/src/commands/summon.test.ts index 13e69e19..9eb91286 100644 --- a/src/commands/summon.test.ts +++ b/src/commands/summon.test.ts @@ -4,7 +4,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { summon, detectActiveProfile, REEXEC_CMD } from "./summon"; -import { existsSync } from "node:fs"; +import { loadProfile } from "../lib/profile-loader"; +import { getSkillDependencies } from "../lib/skill-dependencies"; +import { existsSync, readdirSync } from "node:fs"; // `resources/skills` is a git submodule. The mcp_status assertion below reads a // skill's `requires_mcps` from disk; without it checked out (`git submodule @@ -12,6 +14,44 @@ import { existsSync } from "node:fs"; // rather than fail spuriously. const SKILLS_PRESENT = existsSync(join(import.meta.dir, "../../resources/skills/skills")); +/** + * Find a live (profile with an MCP-gated skill, other profile supplying that + * skill's MCPs) pairing. Returns null when no such pairing exists. + */ +async function findMcpGatedPair(): Promise< + { summonProfile: string; skillId: string; deps: string[]; provider: string } | null +> { + const names = readdirSync(join(import.meta.dir, "../../profiles"), { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + .sort(); + + const loaded = new Map>>(); + for (const n of names) { + try { + loaded.set(n, await loadProfile(n)); + } catch { + // Unloadable profile — not this test's problem; profile-loader has its own. + } + } + + for (const [name, profile] of loaded) { + for (const s of profile.skills.local) { + // Keep the raw ids: summon reports `missing:` with original casing. + const deps = [...new Set(getSkillDependencies(s.id).map((d) => d.mcpId))]; + if (deps.length === 0) continue; + for (const [providerName, provider] of loaded) { + if (providerName === name) continue; + const ids = new Set(provider.mcps.map((m) => m.id.toLowerCase())); + if (deps.every((d) => ids.has(d.toLowerCase()))) { + return { summonProfile: name, skillId: s.id, deps, provider: providerName }; + } + } + } + } + return null; +} + let dir: string; beforeEach(async () => { dir = await mkdtemp(join(tmpdir(), "cue-summon-")); }); afterEach(async () => { await rm(dir, { recursive: true, force: true }); }); @@ -110,14 +150,27 @@ describe("summon", () => { }); test.skipIf(!SKILLS_PRESENT)("mcp_status reflects the active session's loaded MCPs", async () => { - // browser/lightpanda needs the `lightpanda` MCP; core loads it. - const lp = (skills: { id: string; mcp_status: string }[]) => - skills.find((s) => s.id === "browser/lightpanda"); - - const noActive = await summon({ cwd: dir, profile: "vercel", active: null, noPin: true }); - const withCore = await summon({ cwd: dir, profile: "vercel", active: "core", noPin: true }); - - expect(lp(noActive.skills)?.mcp_status).toBe("missing:lightpanda"); - expect(lp(withCore.skills)?.mcp_status).toBe("ok"); + // The pairing is derived from live profile data, not hardcoded. This test + // used to pin browser/lightpanda + core; 18570880 (#121) dropped the + // lightpanda MCP from every profile that pinned it, so the assertion named + // a pairing that no longer existed and failed for a reason unrelated to + // what it tests. The behaviour under test is mcp_status resolution, so + // derive any still-valid pairing and assert against that. + const pair = await findMcpGatedPair(); + // Loud rather than a silent skip: if nothing is satisfiable, that is itself + // worth a human look, not a vacuous green. + expect(pair).not.toBeNull(); + const { summonProfile, skillId, deps, provider } = pair!; + + const find = (skills: { id: string; mcp_status: string }[]) => + skills.find((s) => s.id === skillId); + const missing = `missing:${deps.join(",")}`; + const opts = { cwd: dir, profile: summonProfile, noPin: true }; + + const noActive = await summon({ ...opts, active: null }); + const withProvider = await summon({ ...opts, active: provider }); + + expect(find(noActive.skills)?.mcp_status).toBe(missing); + expect(find(withProvider.skills)?.mcp_status).toBe("ok"); }); }); diff --git a/src/lib/integrity-ladder.test.ts b/src/lib/integrity-ladder.test.ts new file mode 100644 index 00000000..98d6a8de --- /dev/null +++ b/src/lib/integrity-ladder.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, test } from "bun:test"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +/** + * Drift guard for the integrity protocol's ~N% ladder. + * + * The ladder (yellow [INFERRED]/[ASSUMED] → ~50/60/70/80%, orange + * [GUESSED]/[STALE] → ~20/30/40%) is defined in two places that CANNOT import + * from each other: + * + * - resources/hooks/liedetector-tag-density.sh — the Stop hook, lives in the + * cue repo. + * - resources/skills/skills/meta/liedetector/scripts/run-evals.sh — the eval + * grader, lives in the resources/skills SUBMODULE and ships standalone via + * `npx agent-liedetector-skill` to agents that have no cue tree at all. + * + * Duplication is therefore the correct design, but silent divergence is not: + * the whole point of the ladder is that one rule governs every surface. This + * test reads both definitions and asserts they agree. It also asserts the + * prose sources still state the same values, so a doc edit can't drift from + * the code either. + */ + +const REPO = join(import.meta.dir, "../.."); +const HOOK = join(REPO, "resources/hooks/liedetector-tag-density.sh"); +const GRADER = join( + REPO, + "resources/skills/skills/meta/liedetector/scripts/run-evals.sh", +); + +type Ladder = Record; + +/** Parse the `LADDER = { "INFERRED": {"50", ...}, ... }` python literal. */ +function parseLadder(source: string, file: string): Ladder { + const block = source.match(/LADDER\s*=\s*\{([\s\S]*?)\n\}/); + if (!block) throw new Error(`no LADDER definition found in ${file}`); + + const ladder: Ladder = {}; + const entry = /"(\w+)"\s*:\s*\{([^}]*)\}/g; + for (const m of block[1].matchAll(entry)) { + const values = [...m[2].matchAll(/"(\d+)"/g)].map((v) => v[1]); + ladder[m[1]] = values.sort(); + } + if (Object.keys(ladder).length === 0) { + throw new Error(`LADDER in ${file} parsed to zero entries`); + } + return ladder; +} + +/** + * The one true ladder: a 5-point raster, yellow ~50-85%, orange ~20-45%. + * Changing it here means changing it in both scripts and all four prose files. + */ +const YELLOW = ["50", "55", "60", "65", "70", "75", "80", "85"]; +const ORANGE = ["20", "25", "30", "35", "40", "45"]; +const EXPECTED: Ladder = { + INFERRED: YELLOW, + ASSUMED: YELLOW, + GUESSED: ORANGE, + STALE: ORANGE, +}; + +describe("integrity protocol ~N% ladder", () => { + test("the hook and the eval grader define the same ladder", async () => { + const hook = parseLadder(await readFile(HOOK, "utf8"), HOOK); + const grader = parseLadder(await readFile(GRADER, "utf8"), GRADER); + expect(hook).toEqual(grader); + }); + + test("both match the canonical ladder", async () => { + expect(parseLadder(await readFile(HOOK, "utf8"), HOOK)).toEqual(EXPECTED); + expect(parseLadder(await readFile(GRADER, "utf8"), GRADER)).toEqual(EXPECTED); + }); + + test("tiers do not overlap each other or green", () => { + const yellow = EXPECTED.INFERRED.map(Number); + const orange = EXPECTED.GUESSED.map(Number); + // Yellow spans ~50-85%, orange ~20-45%, green starts at 90%. + expect(Math.max(...orange)).toBeLessThan(Math.min(...yellow)); + expect(Math.max(...yellow)).toBeLessThan(90); + expect(EXPECTED.ASSUMED).toEqual(EXPECTED.INFERRED); + expect(EXPECTED.STALE).toEqual(EXPECTED.GUESSED); + }); + + // The prose sources are what the model actually reads. If a doc says ~90% is + // legal on yellow while the hook flags it, the model gets nudged for obeying + // its own instructions — the exact failure this ladder was introduced to fix. + const PROSE = [ + "resources/personas/integrity-protocol.md", + "resources/personas/integrity-protocol-compact.md", + "resources/skills/skills/meta/liedetector/SKILL.md", + "resources/skills/skills/meta/integrity-tags/SKILL.md", + ]; + + // The compact persona states the raster as a range ("~50% to ~85% in 5-point + // steps") to stay short; the long-form docs also spell every step out. So + // assert the RULE — both tier boundaries plus the step size, which together + // pin all 14 values — rather than demanding 14 literals in every file. + test.each(PROSE)("%s states the 5-point raster, no stale values", async (rel) => { + const text = await readFile(join(REPO, rel), "utf8"); + + for (const v of ["50", "85"]) expect(text).toContain(`~${v}%`); // yellow bounds + for (const v of ["20", "45"]) expect(text).toContain(`~${v}%`); // orange bounds + expect(text).toMatch(/5-point/); // the step size fills in between + + // Both retired ladders are gone: the original decile list, and the 4-value + // yellow ladder that briefly replaced it. + expect(text).not.toContain("20 / 30 / 40 / 60 / 80 / 90"); + expect(text).not.toMatch(/~50%`?[,\s]*`?~60%`?[,\s]*`?~70%`?[,\s]*`?~80%/); + }); + + test("calibration is stated as required, not optional", async () => { + for (const rel of PROSE) { + const text = await readFile(join(REPO, rel), "utf8"); + expect(text).not.toMatch(/Optional\s+(decile\s+|percentage\s+)?calibration/i); + expect(text).not.toMatch(/##\s*Optional\s+`?~N%`?/i); + } + }); +}); diff --git a/src/lib/liedetector-hooks.test.ts b/src/lib/liedetector-hooks.test.ts new file mode 100644 index 00000000..a8f0a841 --- /dev/null +++ b/src/lib/liedetector-hooks.test.ts @@ -0,0 +1,302 @@ +import { describe, expect, test, beforeAll, beforeEach, afterEach } from "bun:test"; +import { mkdtemp, mkdir, writeFile, symlink, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** + * Behavior tests for the two liedetector Stop hooks. + * + * Both are gated shell scripts that parse a Claude Code transcript, so the only + * honest way to test them is to feed a real-shaped transcript through stdin and + * read what they emit. Nothing under resources/hooks/ had coverage before this. + * + * Transcript records MUST be compact JSON — tag-audit.sh matches the literal + * `"type":"user"` to find the turn boundary, and Claude Code writes compact + * JSONL. A pretty-printed fixture silently produces zero output. + */ + +const REPO = join(import.meta.dir, "../.."); +const TAG_AUDIT = join(REPO, "resources/hooks/tag-audit.sh"); +const DENSITY = join(REPO, "resources/hooks/liedetector-tag-density.sh"); + +type Tool = { name: string; command?: string }; + +let dir: string; +let seq = 0; + +/** + * Both hooks are gated on a state file under $HOME, so the tests must run with + * HOME pointed at a temp dir — otherwise they'd read the developer's real + * config and pass or fail for the wrong reason. + * + * That breaks `python3` on setups where it resolves to a wrapper script that + * execs `$HOME/.nix-profile/bin/python3` (nix, pyenv, mise, asdf all do shapes + * of this). The hook then fails open and emits nothing — which silently turns + * every "expects no output" assertion into a vacuous pass. So: resolve the real + * interpreter under the ambient HOME once, and front-load a PATH shim pointing + * straight at it. setup asserts the shim survives a foreign HOME, so a broken + * environment fails loudly instead of quietly greening the suite. + */ +let shimBin: string; +beforeAll(async () => { + shimBin = await mkdtemp(join(tmpdir(), "cue-hook-shim-")); + const resolved = Bun.spawnSync(["python3", "-c", "import sys; print(sys.executable)"]); + const real = resolved.stdout.toString().trim(); + if (!resolved.success || !real) { + throw new Error("cannot resolve a real python3 interpreter for hook tests"); + } + await symlink(real, join(shimBin, "python3")); + + const probe = Bun.spawnSync(["python3", "-c", "print('ok')"], { + env: { ...process.env, HOME: shimBin, PATH: `${shimBin}:${process.env.PATH}` }, + }); + if (probe.stdout.toString().trim() !== "ok") { + throw new Error("python3 shim does not survive a foreign HOME"); + } +}); + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "cue-liedetector-hooks-")); +}); +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +/** One user turn plus one assistant turn carrying `text` and `tools`. */ +async function transcript(text: string, tools: Tool[] = []): Promise { + const content: unknown[] = [{ type: "text", text }]; + for (const t of tools) { + content.push({ + type: "tool_use", + name: t.name, + input: t.command ? { command: t.command } : {}, + }); + } + const lines = [ + JSON.stringify({ type: "user", message: { content: "go" } }), + JSON.stringify({ type: "assistant", message: { content } }), + ]; + const path = join(dir, `t${seq++}.jsonl`); + await writeFile(path, lines.join("\n") + "\n"); + return path; +} + +/** Run a hook with the Stop payload on stdin; return everything it emitted. */ +async function runHook( + hook: string, + transcriptPath: string, + env: Record = {}, +): Promise<{ out: string; code: number }> { + const proc = Bun.spawn(["bash", hook], { + stdin: new TextEncoder().encode( + JSON.stringify({ + transcript_path: transcriptPath, + session_id: `test-${seq}`, + }), + ), + stdout: "pipe", + stderr: "pipe", + env: { + ...process.env, + HOME: dir, + PATH: `${shimBin}:${process.env.PATH}`, + ...env, + }, + }); + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { out: (stdout + stderr).trim(), code }; +} + +async function enableDensityGate() { + await mkdir(join(dir, ".config/cue"), { recursive: true }); + await writeFile(join(dir, ".config/cue/liedetector-tag-check"), ""); +} + +const PAD = " filler words to clear the length gate.".repeat(40); +const tag = (s: string) => s; // readability at call sites + +describe("tag-audit.sh", () => { + test("reports the tag mix on a turn with 3+ tags", async () => { + const t = await transcript( + tag( + "🟢 [VERIFIED] a. 🟢 [KNOWN] b. " + + "🟡 [INFERRED ~80%] c. 🟡 [INFERRED ~70%] d. 🟡 [ASSUMED ~60%] e. " + + "🟡 [ASSUMED ~50%] f. 🟡 [INFERRED ~60%] g. " + + "🟠 [GUESSED ~30%] h. 🟠 [GUESSED ~20%] i. 🟠 [STALE ~40%] j. " + + "🔴 [UNKNOWN] k.", + ), + [{ name: "Read" }, { name: "Grep" }], + ); + const { out, code } = await runHook(TAG_AUDIT, t); + expect(code).toBe(0); + expect(out).toContain("Tag mix (11 claims)"); + expect(out).toContain("🟢2 🟡5 🟠3 🔴1"); + expect(out).toContain("18% grounded"); + expect(out).toContain("36% guess-or-worse"); + // Evidence was present, so no violation warning. + expect(out).not.toContain("Tag audit:"); + }); + + test("counts [CORRECTION] separately in the mix line", async () => { + const t = await transcript( + "🟢 [VERIFIED] a. 🟡 [INFERRED ~80%] b. 🟠 [GUESSED ~30%] c. " + + "🟠 [CORRECTION] earlier I said d.", + [{ name: "Read" }], + ); + const { out } = await runHook(TAG_AUDIT, t); + expect(out).toContain("Tag mix (3 claims)"); + expect(out).toContain("1x [CORRECTION]"); + }); + + test("warns when [VERIFIED] appears with no verification action", async () => { + const t = await transcript( + "🟢 [VERIFIED] a. 🟢 [VERIFIED] b. 🟢 [VERIFIED] c. 🟡 [INFERRED ~80%] d.", + [{ name: "Write" }], + ); + const { out } = await runHook(TAG_AUDIT, t); + expect(out).toContain("zero observable verification action"); + expect(out).toContain("Tag mix (4 claims)"); + }); + + test("stays silent below the 3-tag threshold", async () => { + const t = await transcript("🟢 [VERIFIED] a. 🟡 [INFERRED ~80%] b.", [ + { name: "Read" }, + ]); + const { out } = await runHook(TAG_AUDIT, t); + expect(out).toBe(""); + }); + + test("CUE_TAG_MIX_OFF=1 suppresses the mix line", async () => { + const t = await transcript( + "🟢 [VERIFIED] a. 🟡 [INFERRED ~80%] b. 🟠 [GUESSED ~30%] c.", + [{ name: "Read" }], + ); + const { out } = await runHook(TAG_AUDIT, t, { CUE_TAG_MIX_OFF: "1" }); + expect(out).toBe(""); + }); + + test("[skip-tag-audit] suppresses the whole hook", async () => { + const t = await transcript( + "🟢 [VERIFIED] a. 🟢 [VERIFIED] b. 🟢 [VERIFIED] c. [skip-tag-audit]", + [{ name: "Write" }], + ); + const { out } = await runHook(TAG_AUDIT, t); + expect(out).toBe(""); + }); + + test("fails open on an unreadable transcript", async () => { + const { out, code } = await runHook(TAG_AUDIT, join(dir, "missing.jsonl")); + expect(code).toBe(0); + expect(out).toBe(""); + }); +}); + +describe("liedetector-tag-density.sh", () => { + test("flags a yellow tag with no ~N%", async () => { + await enableDensityGate(); + const t = await transcript( + "🟡 [INFERRED] the fix works. 🟠 [GUESSED ~30%] more bugs." + PAD, + ); + const { out, code } = await runHook(DENSITY, t); + expect(code).toBe(0); + expect(out).toContain("no ~N%"); + expect(out).toContain("INFERRED"); + }); + + test("flags off-ladder values on both tiers", async () => { + await enableDensityGate(); + const t = await transcript( + "🟡 [INFERRED ~90%] a. 🟠 [GUESSED ~50%] b." + PAD, + ); + const { out } = await runHook(DENSITY, t); + expect(out).toContain("off-ladder"); + expect(out).toContain("INFERRED ~90%"); + expect(out).toContain("GUESSED ~50%"); + }); + + test("flags false precision", async () => { + await enableDensityGate(); + const t = await transcript("🟡 [INFERRED ~67%] a." + PAD); + const { out } = await runHook(DENSITY, t); + expect(out).toContain("off-ladder"); + }); + + // One tag per transcript: packing all 28 into one response trips the + // tag-spam check instead, which is correct hook behavior but a different + // assertion. No PAD either — the zero-tag check can't fire when a tag is + // present, and 1 tag is well under the spam floor. + const YELLOW = ["50", "55", "60", "65", "70", "75", "80", "85"]; + const ORANGE = ["20", "25", "30", "35", "40", "45"]; + + test("accepts every step of the 5-point raster", async () => { + await enableDensityGate(); + const cases: Array<[string, string[]]> = [ + ["INFERRED", YELLOW], + ["ASSUMED", YELLOW], + ["GUESSED", ORANGE], + ["STALE", ORANGE], + ]; + for (const [name, ladder] of cases) { + for (const v of ladder) { + const t = await transcript(`[${name} ~${v}%] a claim.`); + const { out } = await runHook(DENSITY, t); + expect(`${name} ~${v}% -> ${out}`).toBe(`${name} ~${v}% -> `); + } + } + }); + + test("green and red carry no ~N% and stay legal", async () => { + await enableDensityGate(); + for (const name of ["VERIFIED", "KNOWN", "UNKNOWN"]) { + const t = await transcript(`[${name}] a claim.`); + const { out } = await runHook(DENSITY, t); + expect(out).toBe(""); + } + }); + + test("rejects values that fall between the 5-point steps", async () => { + await enableDensityGate(); + for (const bad of ["52", "63", "77", "22", "38"]) { + const t = await transcript(`🟡 [INFERRED ~${bad}%] a.` + PAD); + const { out } = await runHook(DENSITY, t); + expect(out).toContain("off-ladder"); + } + }); + + test("nudges a long response carrying zero tags", async () => { + await enableDensityGate(); + const t = await transcript("A long untagged answer." + PAD); + const { out } = await runHook(DENSITY, t); + expect(out).toContain("zero confidence tags"); + }); + + test("nudges tag-spam", async () => { + await enableDensityGate(); + const t = await transcript( + Array.from({ length: 12 }, (_, i) => `🟢 [KNOWN] w${i}.`).join(" "), + ); + const { out } = await runHook(DENSITY, t); + expect(out).toContain("Tag-spam"); + }); + + test("[skip-tag-density] suppresses the nudge", async () => { + await enableDensityGate(); + const t = await transcript( + "🟡 [INFERRED] no percent here. [skip-tag-density]" + PAD, + ); + const { out } = await runHook(DENSITY, t); + expect(out).toBe(""); + }); + + test("no-ops entirely when the opt-in gate is absent", async () => { + // gate deliberately not created + const t = await transcript("🟡 [INFERRED] no percent here." + PAD); + const { out, code } = await runHook(DENSITY, t); + expect(code).toBe(0); + expect(out).toBe(""); + }); +});