diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-48409bc6fd25f99c.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-48409bc6fd25f99c.json deleted file mode 100644 index 813df2e..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-48409bc6fd25f99c.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "48409bc6fd25f99c", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_rule_extraction", - "usage": { - "input_tokens": 1340, - "output_tokens": 533, - "cached_tokens": 1206, - "embed_tokens": 0, - "cost_usd": 0.000142318 - }, - "value": { - "rules_markdown": "- One responsibility per module; core types go in a module-local `types.rs`.\n- Each port in `src/ports/` is one trait in one file.\n- Declare every CLI subcommand even when its milestone has not landed.\n- Presets are data, not code: adding a preset is a new folder, never a new module.\n- Never edit the `vendor/tinyagents` submodule here; change it upstream and bump the pin.\n- Declare examples explicitly in `Cargo.toml` with `required-features`.\n- Keep the default build offline; put network needs behind features (`harness`, `github`, `serve`).\n- Use rustfmt output and Rust 2024 idioms; `snake_case` modules/files, `PascalCase` types.\n- Return `Result` using the crate error type from `src/error.rs`.\n- Open every file with a `//!` module doc describing its role and any feature gating.\n- Comments explain the decision, not the code; note load-bearing flags or ordering and why.\n- Public items carry doc comments; Clap fields use `///` so it becomes help text.\n- Keep tests in-crate as `#[cfg(test)] mod tests`; move to sibling `test.rs` when they grow; no `tests/` directory.\n- Provide every port an always-compiled offline mock.\n- Cover lane behaviour with golden tests asserting findings that survive filtering, dedupe and capping.\n- Maintain at least 80% coverage for meaningful library behaviour.\n- Keep every Markdown file at 500 lines or fewer; split and link when it grows.\n- The model never holds a write token; lanes run against a read-only checkout.\n- Write credentials are minted only in `src/apply/` after every model call has returned.\n- Contributor code is never executed; read the diff and tree only, no builds or scripts.\n- Treat PR bodies, comments and diffs as untrusted input; fence and label them as data in prompts.\n- A model verdict is advisory; only deterministic policy in `src/apply/` and `src/automerge/` may mutate GitHub.\n- Report found secrets by type and location only; the value never reaches comments, summaries or logs.\n- Use concise, imperative commit subjects; commit each coherent, validated slice on its own.\n- Pull requests state what changed, any behaviour change, and how it was verified." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-5d5f08dc06f14946.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-5d5f08dc06f14946.json deleted file mode 100644 index ccdf854..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-5d5f08dc06f14946.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "5d5f08dc06f14946", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_rule_extraction", - "usage": { - "input_tokens": 3106, - "output_tokens": 5561, - "cached_tokens": 1536, - "embed_tokens": 0, - "cost_usd": 0.003111033 - }, - "value": { - "rules_markdown": "- src/ is the single crate; one responsibility per module, core types in module-local types.rs, and every port in src/ports/ is one trait in one file.\n- src/bin/tinysweeper.rs is the CLI; every subcommand is declared even if its milestone has not landed.\n- src/server/ is the only production surface; there is no GitHub Actions distribution path.\n- presets/ are review policy as data, not code; adding a preset is a new folder, never a new module.\n- vendor/tinyagents is a git submodule; never edit it here, change upstream and bump the pin.\n- docs/modules//README.md is one document per src/ module.\n- examples/ are declared explicitly in Cargo.toml with required-features so credential-needing smoke tests never build in CI.\n- The default build is offline; anything that needs the network goes behind a feature (harness, github, serve).\n- Follow rustfmt output and Rust 2024 idioms; use snake_case for modules/files and PascalCase for types.\n- Return Result using the crate error type from src/error.rs.\n- Every file opens with a //! module doc describing its role and any feature gating.\n- Comments explain the decision, not the code; say why when a flag or ordering is load-bearing.\n- Public items carry doc comments; Clap fields use /// which becomes the help text.\n- Tests live in-crate: a #[cfg(test)] mod tests block at module bottom, moving to sibling test.rs or _test.rs when they grow; no tests/ directory.\n- Every port has an always-compiled offline mock; MockForge records what would have been written so tests can assert on exact check runs and comments.\n- Lane behaviour is covered by golden tests: fixture diff plus a canned structured model response, asserting findings that survive filtering, dedupe, and capping.\n- Maintain at least 80% coverage for meaningful library behaviour.\n- Keep every Markdown file at 500 lines or fewer; split into focused files and link from module README when over.\n- The model never holds a write token; lanes run against a read-only checkout; write credentials minted only in src/apply/ after all model calls returned; a lane leaving a change in checkout fails.\n- Contributor code is never executed; read the diff and tree, do not build, install dependencies, or run target repository's scripts.\n- Pull request bodies, comments, and diffs are untrusted input; fence and label them as data in prompts; model verdict advisory; only deterministic policy in src/apply/ and src/automerge/ may mutate GitHub.\n- Secrets found by scanners are reported by type and location only; the value never reaches a comment, check-run summary, or log.\n- Use concise, imperative commit subjects; keep commits small; commit each coherent, validated slice on its own.\n- Pull requests state what changed, any behaviour change, and how it was verified." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-68b7ad38f2397deb.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-68b7ad38f2397deb.json new file mode 100644 index 0000000..bef8074 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0001-68b7ad38f2397deb.json @@ -0,0 +1,16 @@ +{ + "key": "68b7ad38f2397deb", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_rule_extraction", + "usage": { + "input_tokens": 1531, + "output_tokens": 491, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00012278518 + }, + "value": { + "rules_markdown": "- `src/` — single crate, one responsibility per module, core types in `types.rs`, ports in `src/ports/` as one trait per file.\n- `src/bin/tinysweeper.rs` — CLI with every subcommand declared; `src/server/` is the only production surface.\n- `presets/` — review policy as data, not code; a preset is a folder with `preset.toml`, `README.md`, optional prompt overrides.\n- `vendor/tinyagents` — agent harness as git submodule; never edit here, change upstream and bump pin.\n- `docs/modules//README.md` — one document per `src/` module.\n- `examples/` — declared in `Cargo.toml` with `required-features`.\n- rustfmt output, Rust 2024 idioms; `snake_case` modules and files, `PascalCase` types.\n- Return `Result` using the crate error type from `src/error.rs`.\n- Every file opens with `//!` module doc describing role and feature gating.\n- Comments explain decision, not code; public items carry doc comments; Clap fields use `///`.\n- Tests live in-crate: `#[cfg(test)] mod tests` block, moving to sibling `test.rs` when large; no `tests/` directory.\n- Every port has an always-compiled offline mock; `MockForge` records writes for assertions.\n- Lane behaviour covered by golden tests: fixture diff plus canned structured model response.\n- Maintain at least 80% coverage for meaningful library behaviour.\n- Keep every Markdown file at 500 lines or fewer; split when a topic grows past that limit.\n- The model never holds a write token; write credentials are minted only in `src/apply/`.\n- Contributor code is never executed; we read the diff and tree, not build or run scripts.\n- Pull request bodies, comments, diffs are untrusted input; fence and label as data in prompts.\n- Secrets found by scanners are reported by type and location only, value never reaches comments or logs.\n- Use concise, imperative commit subjects; keep commits small and coherent." + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-17541401c04d839b.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-17541401c04d839b.json deleted file mode 100644 index c94ed20..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-17541401c04d839b.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "17541401c04d839b", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2845, - "output_tokens": 552, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.001717815 - }, - "value": { - "findings": [], - "summary": "The change adds a new public module `summarise` to `src/findings/mod.rs`. The declaration follows the repository's snake_case naming convention and does not introduce any apparent issues." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-86f2a405ae5e917d.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-86f2a405ae5e917d.json deleted file mode 100644 index 48e86ff..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-86f2a405ae5e917d.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "key": "86f2a405ae5e917d", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 1779, - "output_tokens": 1145, - "cached_tokens": 1583, - "embed_tokens": 0, - "cost_usd": 0.000286199 - }, - "value": { - "findings": [ - { - "body": "OpenRouter is a chat/completion routing gateway; it does not expose an OpenAI-compatible embeddings endpoint, so `provider = \"openrouter\"` paired with `openai/text-embedding-3-small` will most likely fail at request time. The comment explicitly states this goes \"through the same gateway the completion models use,\" which is the load-bearing assumption that is probably wrong: completions and embeddings are different API surfaces, and OpenRouter does not relay the latter. If so, no vectors ever get written regardless of `enabled = true`. I could not verify OpenRouter's current API surface from the diff, so treat this as a check-the-vendor claim rather than a certainty, but if it holds the section is inert.", - "confidence": 0.4, - "existing_code": "provider = \"openrouter\"\nmodel = \"openai/text-embedding-3-small\"", - "late": false, - "path": ".tinysweeper.toml", - "rule": "Return `Result` using the crate error type from `src/error.rs`.", - "severity": "medium", - "title": "Verify OpenRouter serves an embeddings endpoint before relying on it" - } - ], - "resolved": [], - "summary": "The added `[embeddings]` section is internally consistent and the partition-key comment is sound, but one load-bearing assumption needs a real check: OpenRouter is a chat/completion gateway and is not known to expose an embeddings endpoint, so `provider = \"openrouter\"` with `openai/text-embedding-3-small` may never produce vectors. Flag that for verification; otherwise the change looks fine." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-c312a6bd2f611374.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-c312a6bd2f611374.json new file mode 100644 index 0000000..d773ca9 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-c312a6bd2f611374.json @@ -0,0 +1,17 @@ +{ + "key": "c312a6bd2f611374", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 3283, + "output_tokens": 56, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.0001658797 + }, + "value": { + "summary": "The change fixes environment variable names to match what the binary actually reads, adds missing API keys for embedding providers, and mounts the configuration file so the server doesn't silently run with defaults. No bugs introduced.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-e90ec1d2def4d0bd.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-e90ec1d2def4d0bd.json deleted file mode 100644 index f8baffa..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0002-e90ec1d2def4d0bd.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "e90ec1d2def4d0bd", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2836, - "output_tokens": 3704, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.004014412 - }, - "value": { - "findings": [], - "summary": "The change adds an embeddings configuration block to `.tinysweeper.toml`; the TOML syntax looks valid and no correctness, security, or repository-policy issues are apparent from the diff." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-1c93c25d9aadd0a7.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-1c93c25d9aadd0a7.json deleted file mode 100644 index 4c0e13b..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-1c93c25d9aadd0a7.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "key": "1c93c25d9aadd0a7", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2964, - "output_tokens": 2819, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.003576222 - }, - "value": { - "findings": [ - { - "body": "The diff turns on embeddings retrieval via an external provider (`provider = \"openrouter\"`) without any feature gating. The repository rule says the default build is offline and anything that needs the network goes behind a feature (harness, github, serve). Enabling this in `.tinysweeper.toml` makes network calls part of the default configuration even when the relevant feature is off, breaking offline use. Either set `enabled = false` here or ensure the code only reads this section when a network feature is active.", - "confidence": 0.5, - "existing_code": "enabled = true", - "late": false, - "path": ".tinysweeper.toml", - "rule": "network-off-by-default", - "severity": "medium", - "suggestion": null, - "title": "Disable embeddings retrieval by default or gate it behind a feature" - } - ], - "summary": "The change adds embeddings configuration; as written it enables network retrieval by default, which appears to violate the offline-default policy. I'd resolve that before merging." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-908b8d5671bc4efe.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-908b8d5671bc4efe.json new file mode 100644 index 0000000..6237957 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-908b8d5671bc4efe.json @@ -0,0 +1,17 @@ +{ + "key": "908b8d5671bc4efe", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 2827, + "output_tokens": 70, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00014496762 + }, + "value": { + "summary": "This pull request adds an `[embeddings]` section to `.tinysweeper.toml` with a table-driven configuration for retrieval, using the same gateway as the completion models. No bugs or regressions are introduced by these three lines of configuration data.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-b31533cbc185afb5.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-b31533cbc185afb5.json deleted file mode 100644 index ab91047..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-b31533cbc185afb5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "b31533cbc185afb5", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 751, - "output_tokens": 122, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00007941000000000001 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-d79975eaecc88e58.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-d79975eaecc88e58.json deleted file mode 100644 index a6f3df5..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0003-d79975eaecc88e58.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "d79975eaecc88e58", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3536, - "output_tokens": 14776, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.013951552 - }, - "value": { - "findings": [], - "summary": "The shim correctly locates the image entrypoint, patches the kernel-compatibility function into an early-returning no-op while preserving the rest of the source, and executes the patched entrypoint with an appropriate module namespace. The change looks sound for its intended deployment context." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-01b577a5fa03b6a3.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-01b577a5fa03b6a3.json deleted file mode 100644 index 78f75cc..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-01b577a5fa03b6a3.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "01b577a5fa03b6a3", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3059, - "output_tokens": 10783, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.010270147 - }, - "value": { - "findings": [ - { - "body": "When Compose merges this overlay with the base service, it keeps the base service's `command` (or the image CMD `[\"mongod\"]`) and appends it to the new `entrypoint`. The list already ends in `\"mongod\"`, so the Python shim receives `mongod` twice. If the shim forwards its arguments to `mongod`, MongoDB will reject the second positional `mongod` and fail to start. Drop the third list item and let the existing CMD/command supply the executable argument, or set `command` explicitly if the shim requires it.", - "confidence": 0.7, - "existing_code": " entrypoint:\n [\"python3\", \"/usr/local/bin/kernel-bypass-entrypoint.py\", \"mongod\"]", - "path": "docker-compose.kernel-bypass.yml", - "rule": "duplicate-command-arg", - "severity": "high", - "suggestion": " entrypoint:\n [\"python3\", \"/usr/local/bin/kernel-bypass-entrypoint.py\"]", - "title": "Remove hard-coded mongod arg that duplicates image CMD" - } - ], - "summary": "The overlay adds a GLIBC tunable and a Python shim to work around MongoDB's kernel check, but the new entrypoint duplicates the existing command argument, so the shim will likely start mongod with an extra `mongod` argument and fail to bring up the database." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-3df0bb3a4c9249d9.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-3df0bb3a4c9249d9.json deleted file mode 100644 index 6afc0a3..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-3df0bb3a4c9249d9.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "3df0bb3a4c9249d9", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1013, - "output_tokens": 176, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.000593775 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-508455793c710ced.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-508455793c710ced.json deleted file mode 100644 index c5033e0..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-508455793c710ced.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "508455793c710ced", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2496, - "output_tokens": 3374, - "cached_tokens": 2121, - "embed_tokens": 0, - "cost_usd": 0.0007961029999999999 - }, - "value": { - "findings": [ - { - "body": "The module imports `runpy` but never calls it. The subsequent line accesses `runpy.run_path.__globals__` solely to create a reference to the `runpy` name so linters do not flag the unused import, then suppresses the resulting unused-variable warning with `noqa: F841`. The variable `globals_dict` is never read — the `exec` call two lines later passes a completely separate dict literal. This is dead code left over from an earlier approach, and the comment \"keep runpy imported\" gives no reason to keep it. Remove both `import runpy` and this line; the `compile` + `exec` path does not depend on `runpy`.", - "confidence": 0.85, - "existing_code": "globals_dict = runpy.run_path.__globals__ # noqa: F841 - keep runpy imported", - "path": "deploy/mongo/kernel-bypass-entrypoint.py", - "rule": "", - "severity": "low", - "title": "Remove dead runpy import and its linter-suppression shim" - } - ], - "resolved": [], - "summary": "The patching logic is sound: it fails loudly when the target function signature is missing, and the rename-and-replace strategy correctly neutralizes `enforce_kernel_compatibility` while keeping the original body in the file as dead code. The one issue is a leftover `runpy` import with a fake-usage line whose only purpose is to suppress the unused-import lint; it should be removed before merge. No functional bugs found." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-71cddd72c75d2722.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-71cddd72c75d2722.json new file mode 100644 index 0000000..4e0de1b --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0004-71cddd72c75d2722.json @@ -0,0 +1,17 @@ +{ + "key": "71cddd72c75d2722", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 3050, + "output_tokens": 77, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00016103304 + }, + "value": { + "summary": "This overlay file adds environment variables and an entrypoint shim to work around a kernel compatibility issue in the MongoDB container. The configuration is sound: the environment variable is set on both affected services, the entrypoint binding is correct, and the file-level documentation is thorough. No bugs or unsafe patterns were introduced.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-183c4d7693b78647.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-183c4d7693b78647.json deleted file mode 100644 index 06effef..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-183c4d7693b78647.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "key": "183c4d7693b78647", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3187, - "output_tokens": 4694, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.005470125 - }, - "value": { - "findings": [ - { - "body": "The MongoDB community-server image (especially the UBI9 variant) typically does not include Python 3. Overriding the entrypoint to invoke `python3` will cause the container to fail to start with 'exec: python3: not found' if the interpreter is absent. Use a shell-based entrypoint or install Python before running the script.", - "confidence": 0.8, - "existing_code": " [\"python3\", \"/usr/local/bin/kernel-bypass-entrypoint.py\", \"mongod\"]", - "path": "docker-compose.kernel-bypass.yml", - "rule": "missing-runtime-dependency", - "severity": "high", - "suggestion": null, - "title": "MongoDB container image may lack python3" - }, - { - "body": "The mongo-secrets service inherits the original image entrypoint, which the file's comments say hard-refuses to start `mongod` on affected kernels. Setting `GLIBC_TUNABLES` alone may not bypass that check because the check is separate from the glibc tunable. If the base service uses the same entrypoint, it will fail to start. Override its entrypoint as well or confirm it does not run the kernel check.", - "confidence": 0.7, - "existing_code": " mongo-secrets:\n environment:\n GLIBC_TUNABLES: glibc.pthread.rseq=0", - "path": "docker-compose.kernel-bypass.yml", - "rule": "inconsistent-kernel-bypass", - "severity": "high", - "suggestion": null, - "title": "mongo-secrets still uses the refusing entrypoint" - }, - { - "body": "The MongoDB image's original entrypoint performs important setup (e.g., file permissions, keyfile generation, user/group setup) before executing `mongod`. Replacing it entirely with a custom Python script risks skipping those steps, leading to startup failures or security misconfigurations. Verify that the script replicates necessary initialization or use a wrapper that calls the original entrypoint with a flag to bypass only the kernel check.", - "confidence": 0.3, - "existing_code": " mongod:\n entrypoint:\n [\"python3\", \"/usr/local/bin/kernel-bypass-entrypoint.py\", \"mongod\"]", - "path": "docker-compose.kernel-bypass.yml", - "rule": "entrypoint-replacement", - "severity": "medium", - "suggestion": null, - "title": "Overriding entrypoint drops original setup steps" - } - ], - "summary": "The overlay aims to bypass MongoDB's kernel restriction, but it relies on Python being present in the image and leaves `mongo-secrets` potentially failing, while discarding the original entrypoint's setup logic. These issues need to be addressed before merging." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-2269be87e5446f3d.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-2269be87e5446f3d.json deleted file mode 100644 index b58938a..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-2269be87e5446f3d.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "2269be87e5446f3d", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1331, - "output_tokens": 243, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.000790395 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-9dcf52aa6a97d3bd.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-9dcf52aa6a97d3bd.json deleted file mode 100644 index fb4e05f..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-9dcf52aa6a97d3bd.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "9dcf52aa6a97d3bd", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1466, - "output_tokens": 197, - "cached_tokens": 1077, - "embed_tokens": 0, - "cost_usd": 0.000084571 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-b867d8949a942d78.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-b867d8949a942d78.json new file mode 100644 index 0000000..0b561e1 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0005-b867d8949a942d78.json @@ -0,0 +1,17 @@ +{ + "key": "b867d8949a942d78", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 2708, + "output_tokens": 98, + "cached_tokens": 1024, + "embed_tokens": 0, + "cost_usd": 0.000101863328 + }, + "value": { + "summary": "Adds a `summarise` module declaration to the findings module. The change itself is trivial and safe; it exposes a name that must resolve to a file in the same directory. I cannot verify that `src/findings/summarise.rs` exists or that it compiles, but that is a completeness check outside the diff's scope, not a bug in this one-line addition.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-548c6658c51e5f85.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-548c6658c51e5f85.json deleted file mode 100644 index 99f4dce..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-548c6658c51e5f85.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "548c6658c51e5f85", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 1990, - "output_tokens": 3417, - "cached_tokens": 1691, - "embed_tokens": 0, - "cost_usd": 0.000794653 - }, - "value": { - "findings": [ - { - "body": "The comment states this overlay targets \"Linux kernels 6.19 and newer\" and describes the problem as affecting a \"current desktop kernel.\" Linux 6.19 has not been released as of the latest kernel versions I can account for; the rseq/tcmalloc issue that `glibc.pthread.rseq=0` mitigates has been reported on kernels in the 6.9–6.13 range. If 6.19 is a typo for 6.9 (or another version), users reading this guidance will apply — or skip — the overlay for the wrong kernels.", - "confidence": 0.35, - "existing_code": "# Opt-in overlay for Linux kernels 6.19 and newer.", - "path": "docker-compose.kernel-bypass.yml", - "rule": "Comments explain the decision, not the code; note load-bearing flags or ordering and why.", - "severity": "low", - "title": "Verify the kernel version cited in the header comment" - } - ], - "resolved": [], - "summary": "The overlay is syntactically valid Docker Compose and its structure (entrypoint override + GLIBC_TUNABLES for both services) is internally consistent. The one concern I can anchor to is that the header comment cites kernel 6.19, which does not appear to be a released kernel version; if this is a typo the guidance will mislead users about when to apply the overlay. I could not verify whether `python3` is present in the MongoDB UBI9 image, whether `deploy/mongo/kernel-bypass-entrypoint.py` exists, or whether `mongo-secrets` also needs the entrypoint shim — all of those depend on files outside this diff." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-78bfb662caf7b753.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-78bfb662caf7b753.json new file mode 100644 index 0000000..62a7cdd --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-78bfb662caf7b753.json @@ -0,0 +1,48 @@ +{ + "key": "78bfb662caf7b753", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 3527, + "output_tokens": 1052, + "cached_tokens": 1024, + "embed_tokens": 0, + "cost_usd": 0.000235104548 + }, + "value": { + "summary": "This shim patching a MongoDB entrypoint to bypass a kernel compatibility check is a clever approach, but contains a critical bug: it retrieves `runpy.run_path.__globals__` at import time, not at the point of use, which is a no-op that does not affect the patched module's globals. It also silently patches any occurrence of the function definition, not just the one at module top level, and does not handle the case where the image's entrypoint is not a Python file.", + "findings": [ + { + "path": "deploy/mongo/kernel-bypass-entrypoint.py", + "existing_code": " globals_dict = runpy.run_path.__globals__ # noqa: F841 - keep runpy imported\n compiled = compile(patched, ENTRYPOINT, \"exec\")\n exec(compiled, {\"__name__\": \"__main__\", \"__file__\": ENTRYPOINT}) # noqa: S102", + "rule": "unused-variable", + "title": "Remove the unused `globals_dict` variable or use it properly", + "body": "`runpy.run_path` is an imported function object; its `__globals__` attribute is the module's own namespace, not the namespace of the file being executed. The `globals_dict` variable is never passed to `exec`, meaning the patched code runs with a minimal dict containing only `__name__` and `__file__`, which may cause `NameError` for any symbols (e.g. `open`, `os.path`, etc.) the entrypoint script expects to find by default. Meanwhile the `# noqa: F841` comment suppresses a flake8 warning about an unused variable, but the variable is indeed unused and this is the real bug.", + "severity": "high", + "confidence": 1, + "suggestion": " compiled = compile(patched, ENTRYPOINT, \"exec\")\n exec(compiled, {\"__name__\": \"__main__\", \"__file__\": ENTRYPOINT, \"open\": open, \"sys\": sys, \"runpy\": runpy}) # noqa: S102" + }, + { + "path": "deploy/mongo/kernel-bypass-entrypoint.py", + "existing_code": " patched = source.replace(\n needle,\n \"def enforce_kernel_compatibility() -> None:\\n\"\n \" return # patched out by deploy/mongo/kernel-bypass-entrypoint.py\\n\"\n \"\\n\"\n \"def _enforce_kernel_compatibility_unused() -> None:\",\n 1,\n )", + "rule": "incorrect-replacement", + "title": "Replace only the function signature, not the definition header", + "body": "`source.replace(needle, replacement, 1)` replaces the *first occurrence* of the string `\"def enforce_kernel_compatibility() -> None:\"`. This replaces only the `def` line but leaves the body of the function as-is. The replacement string adds a new `def _enforce_kernel_compatibility_unused()` stub, but the original function's indented body (the `source` between the `def` line and the next `def` at the same level) remains in the output after the replacement, leading to a syntax error or silently defining two functions with the same name. The correct approach is to find the entire function body (using `ast` or line-based splicing) and replace the whole thing with just `return`.", + "severity": "high", + "confidence": 1, + "suggestion": "" + }, + { + "path": "deploy/mongo/kernel-bypass-entrypoint.py", + "existing_code": " if __name__ == \"__main__\":\n main()", + "rule": "missing-di-handling", + "title": "Guard the import of the image's entrypoint for non-Python images", + "body": "The shim hardcodes `ENTRYPOINT` as a Python file and assumes it exists and is readable. If the underlying MongoDB image changes its entrypoint to a compiled binary, shell script, or any non-Python format, `open(ENTRYPOINT, ...)` will either fail (if the file does not exist) or produce text that `compile()` cannot parse. The shim's exit path only handles the case where `enforce_kernel_compatibility` is missing from the source; it should also handle `FileNotFoundError` and `SyntaxError` gracefully, with a message that the bypass is no longer applicable.", + "severity": "medium", + "confidence": 0.7, + "suggestion": "" + } + ] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-7fd8433c8c3dcc5b.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-7fd8433c8c3dcc5b.json deleted file mode 100644 index 5ce8d84..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-7fd8433c8c3dcc5b.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "7fd8433c8c3dcc5b", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3292, - "output_tokens": 14717, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.013794082 - }, - "value": { - "findings": [], - "summary": "The change wires the correct provider API keys, fixes the GitHub App auth environment variable names, and mounts the server configuration file read-only. The compose syntax and substitutions shown look consistent and safe to merge." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-86efd049e0b944eb.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-86efd049e0b944eb.json deleted file mode 100644 index 678f4f7..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0006-86efd049e0b944eb.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "86efd049e0b944eb", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3663, - "output_tokens": 5259, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.006168735 - }, - "value": { - "findings": [ - { - "body": "When `rfind(' ')` returns `None`, the code appends an ellipsis to the full `limit`-character `head`, producing a result of `limit + 1` characters. This violates the documented contract (\"at most `limit` characters\") and may exceed downstream limits (e.g., GitHub check-run titles). The fix removes one character from `head` before appending the ellipsis, except when `limit` is 0, where an empty string should be returned.", - "confidence": 0.95, - "existing_code": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }", - "path": "src/findings/summarise.rs", - "rule": "off-by-one", - "severity": "medium", - "suggestion": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => {\n let mut head = head;\n if head.is_empty() {\n String::new()\n } else {\n head.pop();\n format!(\"{head}…\")\n }\n }\n }", - "title": "Respect the character limit when no space is found" - } - ], - "summary": "The change adds `shorten` and `first_sentence` utilities with tests, but `shorten` can return a string one character longer than the requested limit when the truncated prefix contains no space." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-46f4d1a79f482cab.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-46f4d1a79f482cab.json deleted file mode 100644 index 0cf7ea7..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-46f4d1a79f482cab.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "46f4d1a79f482cab", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 933, - "output_tokens": 252, - "cached_tokens": 802, - "embed_tokens": 0, - "cost_usd": 0.000075036 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-abbb7a6b9db04578.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-abbb7a6b9db04578.json deleted file mode 100644 index 2d0176d..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-abbb7a6b9db04578.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "abbb7a6b9db04578", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3420, - "output_tokens": 5319, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00611523 - }, - "value": { - "findings": [], - "summary": "The change renames GitHub App environment variables to match what the binary reads, adds API key forwarding, and mounts a read-only config file. No issues found." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-c5de1dac54c27232.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-c5de1dac54c27232.json new file mode 100644 index 0000000..02ad64b --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-c5de1dac54c27232.json @@ -0,0 +1,25 @@ +{ + "key": "c5de1dac54c27232", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_falsify", + "usage": { + "input_tokens": 2067, + "output_tokens": 263, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00012669398 + }, + "value": { + "incorrect": [ + { + "index": 2, + "reason": "The replacement string adds a new `def _enforce_kernel_compatibility_unused():` followed by the original function body (which becomes part of the second definition) — but the original function body was part of the replacement string, not left over. The diff shows the replacement string includes `def enforce_kernel_compatibility() -> None:\\n return # patched out...` followed by `def _enforce_kernel_compatibility_unused() -> None:`, so the original body is fully replaced. The finding incorrectly claims the body remains." + }, + { + "index": 1, + "reason": "The diff shows `globals_dict = runpy.run_path.__globals__ # noqa: F841 - keep runpy imported` followed by `exec(compiled, {'__name__': '__main__', '__file__': ENTRYPOINT})`. The # noqa comment explains the variable's purpose: to keep `runpy` imported (so its `__globals__` attribute remains accessible if needed elsewhere). The finding's claim that `globals_dict` is unused and a bug is disproved by the comment explicitly stating it's preserved for that reason." + } + ] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-fa1f7ac7ffabd355.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-fa1f7ac7ffabd355.json deleted file mode 100644 index c76601d..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0007-fa1f7ac7ffabd355.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "fa1f7ac7ffabd355", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2717, - "output_tokens": 3061, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.003403237 - }, - "value": { - "findings": [], - "summary": "The change adds a public `summarise` module to the findings module. The one-line declaration is syntactically valid and introduces no visible bug in this file; the module body itself is outside this diff." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-12ef78c1f6c205ef.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-12ef78c1f6c205ef.json new file mode 100644 index 0000000..bfe5f06 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-12ef78c1f6c205ef.json @@ -0,0 +1,29 @@ +{ + "key": "12ef78c1f6c205ef", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "z-ai/glm-5.2", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 3548, + "output_tokens": 8417, + "cached_tokens": 2504, + "embed_tokens": 0, + "cost_usd": 0.0093953856 + }, + "value": { + "summary": "The module adds two helpers for truncating model-authored text to fit length limits. `first_sentence` is sound, but `shorten` can return `limit + 1` characters when no word boundary is found in the first `limit` chars, contradicting its documented \"at most `limit` characters\" contract and potentially exceeding the hard ceilings it exists to respect.", + "resolved": [], + "findings": [ + { + "path": "src/findings/summarise.rs", + "existing_code": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }", + "rule": "off-by-one", + "title": "Reserve room for the ellipsis so output stays within limit", + "body": "`head` is exactly `limit` characters. When `rfind(' ')` returns `None`, the result is `head` + `…` = `limit + 1` characters, exceeding the `limit` the doc comment promises. The `Some` branch is fine because it drops at least the space, but the `None` branch is not. Since the function exists to stay under GitHub's check-run title ceiling, a caller that passes that ceiling as `limit` can still overflow it. Take `limit - 1` chars for `head` so the ellipsis fits within `limit`. (`limit == 0` with non-empty text remains an edge case that would still produce `\"…\"`; a guard for that is optional but worth considering.)", + "severity": "medium", + "confidence": 0.9, + "suggestion": " // Leave room for the trailing ellipsis so the result stays within `limit`\n // characters: `head` is at most `limit - 1` chars, plus one for `…`.\n let head: String = text.chars().take(limit.saturating_sub(1)).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }" + } + ] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-87d1384e18c9d053.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-87d1384e18c9d053.json deleted file mode 100644 index 5159d2f..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-87d1384e18c9d053.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "87d1384e18c9d053", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1417, - "output_tokens": 456, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.000847467 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-bbaf0cb9ee51f766.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-bbaf0cb9ee51f766.json deleted file mode 100644 index 0b7d16f..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-bbaf0cb9ee51f766.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "key": "bbaf0cb9ee51f766", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3535, - "output_tokens": 12863, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.012286807 - }, - "value": { - "findings": [ - { - "body": "`String::repeat(40)` creates a temporary `String`, and `.as_str()` stores a reference to it in the array literal. That temporary is dropped at the end of the array expression, leaving a dangling `&str`; the borrow checker rejects this with E0716, so the test module does not compile. Bind the repeated string to a local variable before the array and use `cafe.as_str()`.", - "confidence": 0.95, - "existing_code": " for text in [\n \"Guard the índex before dereferencing —— twice\",\n \"日本語のタイトルはとても長いのです\",\n \"café \".repeat(40).as_str(),\n ] {", - "path": "src/findings/summarise.rs", - "rule": "dangling-temporary", - "severity": "high", - "suggestion": " let cafe = \"café \".repeat(40);\n for text in [\n \"Guard the índex before dereferencing —— twice\",\n \"日本語のタイトルはとても長いのです\",\n cafe.as_str(),\n ] {", - "title": "Hoist the repeated string into a local binding" - }, - { - "body": "The doc says the result is at most `limit` characters, but when truncation occurs the function takes the first `limit` characters and then appends a trailing `…`, so the result can be `limit + 1` characters long. A caller passing a hard GitHub title limit would get a value just over the cap. Reserve one character for the ellipsis, or document that the result may be `limit + 1`.", - "confidence": 0.6, - "existing_code": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }", - "path": "src/findings/summarise.rs", - "rule": "off-by-one", - "severity": "low", - "suggestion": null, - "title": "Make shorten respect its character limit" - } - ], - "summary": "The module adds sensible truncation helpers, but the test suite will not compile because an array element borrows from a temporary `String`, and `shorten` can exceed its documented character limit by one." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-f8ee04ae981b9121.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-f8ee04ae981b9121.json deleted file mode 100644 index 9bf5055..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0008-f8ee04ae981b9121.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "f8ee04ae981b9121", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2219, - "output_tokens": 942, - "cached_tokens": 1974, - "embed_tokens": 0, - "cost_usd": 0.000250052 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change looks sound. It renames two environment variables that the comment says were never read by the binary (`TINYSWEEPER_GITHUB_*` → `TINYSWEEPER_APP_*`), makes them required (`:?`) so the container fails fast instead of starting unable to authenticate, adds the missing API keys the embeddings provider needs, and mounts the config file the runtime image does not carry. I cannot verify from this diff alone that `server::auth::AppAuth::from_env` reads `TINYSWEEPER_APP_ID` / `TINYSWEEPER_APP_PRIVATE_KEY` verbatim, nor that `/home/tinysweeper/.tinysweeper.toml` is the path the binary searches — those claims depend on code not shown — but nothing in the compose file itself introduces a bug." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-2310062d1ab2ba6a.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-2310062d1ab2ba6a.json new file mode 100644 index 0000000..f85ea74 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-2310062d1ab2ba6a.json @@ -0,0 +1,21 @@ +{ + "key": "2310062d1ab2ba6a", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_falsify", + "usage": { + "input_tokens": 1740, + "output_tokens": 205, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.000108059 + }, + "value": { + "incorrect": [ + { + "index": 1, + "reason": "On line 20, `head` is built from `text.chars().take(limit)`, so `head` has at most `limit` characters. Then on line 23, when no space is found, `None => format!(\"{head}…\")` produces output of at most `limit + 1` characters. The doc comment on line 8 says 'Shorten `text` to at most `limit` characters', which the `None` branch violates because it can yield `limit + 1` characters. However, the diff itself does not disprove that the function claims one thing but does another — the diff contains the code that does exactly what the finding describes. The finding is correct based on the visible code; the diff does not contain any countervailing evidence that would disprove it. Therefore this finding cannot be rejected." + } + ] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-2d53b24abad09d41.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-2d53b24abad09d41.json deleted file mode 100644 index 5998a37..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-2d53b24abad09d41.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "2d53b24abad09d41", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1703, - "output_tokens": 1110, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.001430425 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-6088af22011645cb.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-6088af22011645cb.json deleted file mode 100644 index 8c3b5f7..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-6088af22011645cb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "6088af22011645cb", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1874, - "output_tokens": 482, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00123453 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-976d996747c9e480.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-976d996747c9e480.json deleted file mode 100644 index 2977918..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0009-976d996747c9e480.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "976d996747c9e480", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 1659, - "output_tokens": 209, - "cached_tokens": 1443, - "embed_tokens": 0, - "cost_usd": 0.00007985900000000001 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change is a one-line addition declaring `pub mod summarise;` in `src/findings/mod.rs`. This is a straightforward module registration with no logic to introduce a bug. The contents of the new `summarise` module are not part of this diff and are out of scope for this file's review." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-0a7b31caad337548.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-0a7b31caad337548.json new file mode 100644 index 0000000..0ad168a --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-0a7b31caad337548.json @@ -0,0 +1,17 @@ +{ + "key": "0a7b31caad337548", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 2806, + "output_tokens": 60, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00014296436 + }, + "value": { + "summary": "Adds an `[embeddings]` configuration section for OpenRouter text embeddings. The change is safe: the API key is referenced by environment variable name only, not embedded, and the configuration is static data with no executable logic.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-3ef1e4ebacbe0381.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-3ef1e4ebacbe0381.json deleted file mode 100644 index 49586b7..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-3ef1e4ebacbe0381.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "3ef1e4ebacbe0381", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2519, - "output_tokens": 2466, - "cached_tokens": 2267, - "embed_tokens": 0, - "cost_usd": 0.000589631 - }, - "value": { - "findings": [ - { - "body": "The doc comment promises `Shorten text to at most limit characters`, but the truncation path can return `limit + 1` characters: `head` is built from `text.chars().take(limit)` (up to `limit` chars) and then an ellipsis character is appended. When there is no space in `head`, `format!(\"{head}…\")` yields exactly `limit + 1` chars; when there is a space the prefix is shorter, so it usually fits, but the no-space branch does not. The test at `a_long_string_stops_on_a_word_boundary` even asserts `out.len() <= 21` for `limit = 20`, so the +1 is baked in. Because the stated purpose is fitting under a hard GitHub ceiling, a caller passing that ceiling as `limit` can produce a title one character over. Either reserve one character for the ellipsis (`take(limit.saturating_sub(1))`) or fix the doc to say `limit + 1`.", - "confidence": 0.82, - "existing_code": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }", - "path": "src/findings/summarise.rs", - "rule": "Return Result using the crate error type", - "severity": "medium", - "title": "shorten can return limit+1 characters, breaking its own contract" - } - ], - "resolved": [], - "summary": "The multibyte-slicing fix in `shorten` is sound and `first_sentence` is safe, but `shorten` violates its own `at most limit characters` contract on the no-space branch by appending an ellipsis to a `limit`-character head, yielding `limit + 1` characters — a problem when `limit` is a real ceiling such as GitHub's check-run title cap. The test even encodes the +1. No other issues found in this file." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-51cbaf94131edeb2.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-51cbaf94131edeb2.json deleted file mode 100644 index 8a483f9..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-51cbaf94131edeb2.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "51cbaf94131edeb2", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3515, - "output_tokens": 6757, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.007407615 - }, - "value": { - "findings": [], - "summary": "This new file is a development compose-overlay entrypoint that patches and execs the MongoDB image's own docker-entrypoint.py with a hardcoded path; no untrusted input reaches the exec and no secrets, permissions, or network/subprocess behaviour are introduced in this file. The change looks safe to merge for its stated opt-in, non-production use." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-d6bf343291f4be03.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-d6bf343291f4be03.json deleted file mode 100644 index 5dda22f..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0010-d6bf343291f4be03.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "d6bf343291f4be03", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3664, - "output_tokens": 6495, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.007078842 - }, - "value": { - "findings": [ - { - "body": "The file handle returned by `open()` is never closed. While the process will eventually close it on exit, the handle remains open during the entire execution of the patched entrypoint, which may run indefinitely if the entrypoint starts a long-lived process. Use a context manager to ensure it is closed immediately after reading.", - "confidence": 0.8, - "existing_code": " source = open(ENTRYPOINT, encoding=\"utf-8\").read()", - "path": "deploy/mongo/kernel-bypass-entrypoint.py", - "rule": "resource-leak", - "severity": "low", - "suggestion": " with open(ENTRYPOINT, encoding=\"utf-8\") as f:\n source = f.read()", - "title": "Close the entrypoint file after reading" - } - ], - "summary": "The change adds a shim to bypass the MongoDB kernel compatibility check; it is generally sound, with one minor resource leak." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-324a651e69b6c33a.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-324a651e69b6c33a.json deleted file mode 100644 index 05276e5..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-324a651e69b6c33a.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "324a651e69b6c33a", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3271, - "output_tokens": 4594, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.005419665 - }, - "value": { - "findings": [], - "summary": "The change corrects the GitHub App environment variable names, forwards the additional provider keys, and adds a read-only bind mount for the server configuration. No security issues introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-6c1f6e3ffb9626df.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-6c1f6e3ffb9626df.json deleted file mode 100644 index 6f4c5c6..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-6c1f6e3ffb9626df.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "6c1f6e3ffb9626df", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1553, - "output_tokens": 110, - "cached_tokens": 1305, - "embed_tokens": 0, - "cost_usd": 0.000058525000000000006 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-7c0e6d892204ca20.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-7c0e6d892204ca20.json deleted file mode 100644 index bad8361..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-7c0e6d892204ca20.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "7c0e6d892204ca20", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1663, - "output_tokens": 299, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.000707455 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-f67467cb64fe2258.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-f67467cb64fe2258.json new file mode 100644 index 0000000..53435c7 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0011-f67467cb64fe2258.json @@ -0,0 +1,17 @@ +{ + "key": "f67467cb64fe2258", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 3505, + "output_tokens": 54, + "cached_tokens": 1024, + "embed_tokens": 0, + "cost_usd": 0.000136505068 + }, + "value": { + "summary": "This pull request introduces a new `shorten` and `first_sentence` utility for truncating model-authored text, along with tests. I found no security problems introduced by this change.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-324e39bcc10103cd.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-324e39bcc10103cd.json deleted file mode 100644 index 64da69f..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-324e39bcc10103cd.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "324e39bcc10103cd", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2475, - "output_tokens": 818, - "cached_tokens": 2178, - "embed_tokens": 0, - "cost_usd": 0.00022906400000000001 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change looks sound. The `exec` target is the MongoDB image's own bundled entrypoint at a hardcoded path (`/usr/local/bin/docker-entrypoint.py`), not untrusted or contributor-supplied code; the shim patches one function body in that known file and executes it as `__main__`, which is what the container would do anyway. No untrusted input reaches a dangerous sink, no path traversal, no secrets, no network or subprocess calls. The repository's coding rules are Rust-centric and do not apply to this Python deploy helper." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-b5380fa68e6677d3.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-b5380fa68e6677d3.json deleted file mode 100644 index 394ef5e..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-b5380fa68e6677d3.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "b5380fa68e6677d3", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3642, - "output_tokens": 506, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00202449 - }, - "value": { - "findings": [], - "summary": "The new string truncation and sentence extraction helpers are pure and safe; no security issues introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-d4663b5b7171ece8.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-d4663b5b7171ece8.json deleted file mode 100644 index 16ce4da..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-d4663b5b7171ece8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "d4663b5b7171ece8", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2815, - "output_tokens": 4451, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.004655167 - }, - "value": { - "findings": [], - "summary": "The change adds a configuration section enabling OpenRouter-based embeddings and names an environment variable for the API key; it introduces no secret values, permission changes, or security-sensitive behavior in this file." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-d60090c3108a8bc7.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-d60090c3108a8bc7.json new file mode 100644 index 0000000..adad5ac --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0012-d60090c3108a8bc7.json @@ -0,0 +1,17 @@ +{ + "key": "d60090c3108a8bc7", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 3262, + "output_tokens": 46, + "cached_tokens": 1024, + "embed_tokens": 0, + "cost_usd": 0.000123850328 + }, + "value": { + "summary": "The change fixes environment variable names so the container can authenticate as a GitHub App and adds a volume mount for runtime configuration. The fixes are correct and safe.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-3d5e8e65650775c9.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-3d5e8e65650775c9.json deleted file mode 100644 index 6188fd2..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-3d5e8e65650775c9.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "3d5e8e65650775c9", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2824, - "output_tokens": 508, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.0016704 - }, - "value": { - "findings": [], - "summary": "The change adds a new submodule `summarise` to the findings module. No security-sensitive code is introduced in this file; the change is safe to merge." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-6e98aa73235b777f.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-6e98aa73235b777f.json deleted file mode 100644 index 757b47b..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-6e98aa73235b777f.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "6e98aa73235b777f", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2198, - "output_tokens": 828, - "cached_tokens": 1934, - "embed_tokens": 0, - "cost_usd": 0.00022578199999999997 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The docker-compose.yml change looks sound. It tightens the auth environment variables from silently-defaulting `TINYSWEEPER_GITHUB_*` to required `TINYSWEEPER_APP_*` (`:?`), which is a security improvement. The new API-key variables default to empty and are forwarded from the host environment, not hardcoded. The read-only volume mount of the config file is an intended operational choice and carries no path-traversal or write risk. No untrusted input reaches a dangerous sink, and no secrets are committed to code." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-85f1803a0e941767.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-85f1803a0e941767.json new file mode 100644 index 0000000..e0820fd --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-85f1803a0e941767.json @@ -0,0 +1,17 @@ +{ + "key": "85f1803a0e941767", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 3506, + "output_tokens": 98, + "cached_tokens": 1024, + "embed_tokens": 0, + "cost_usd": 0.000140853608 + }, + "value": { + "summary": "This is a helper script to bypass a kernel compatibility check in a MongoDB Docker image entrypoint. The script reads the image's entrypoint, patches one function definition, and executes the patched code. It is an opt-in overlay for development use only, has a guard that fails hard if the target function is missing, and is not a default or production path. I see no security problems introduced by this change.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-8ac85a2f4f827918.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-8ac85a2f4f827918.json deleted file mode 100644 index 58b2cf2..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0013-8ac85a2f4f827918.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "8ac85a2f4f827918", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3038, - "output_tokens": 5196, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.005400322 - }, - "value": { - "findings": [ - { - "body": "Overriding `entrypoint` replaces the official MongoDB image's `docker-entrypoint.sh`, which normally drops privileges to the `mongodb` user with `gosu` before starting `mongod`. The Python shim is not shown here, and the file's comment says it only stops the image refusing to start, so unless the shim re-execs as `mongodb`, `mongod` will run as root in the container. Keep the privilege drop in the shim, or re-use the official entrypoint after setting the environment override.", - "confidence": 0.5, - "existing_code": " entrypoint:\n [\"python3\", \"/usr/local/bin/kernel-bypass-entrypoint.py\", \"mongod\"]", - "path": "docker-compose.kernel-bypass.yml", - "rule": "privilege-drop", - "severity": "medium", - "suggestion": null, - "title": "Preserve the image's privilege drop when overriding entrypoint" - } - ], - "summary": "The overlay adds an opt-in kernel workaround for local MongoDB by mounting and running a Python shim as the entrypoint. It should not be merged as-is because overriding the official entrypoint likely loses the image's privilege drop to the `mongodb` user, causing `mongod` to run as root." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-13bed2303116481d.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-13bed2303116481d.json deleted file mode 100644 index 847ddc3..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-13bed2303116481d.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "13bed2303116481d", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 1758, - "output_tokens": 154, - "cached_tokens": 1529, - "embed_tokens": 0, - "cost_usd": 0.000069787 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change adds an `[embeddings]` config block that names a provider, model, dimensions, and an environment variable holding the API key. No secret value is committed; only the env var name appears. No new subprocess, network call, deserialization, auth change, or widened permissions are introduced in this file. Nothing actionable." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-41ed18e32c1b3bd3.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-41ed18e32c1b3bd3.json deleted file mode 100644 index 55dedf4..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-41ed18e32c1b3bd3.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "41ed18e32c1b3bd3", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2696, - "output_tokens": 525, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.001187782 - }, - "value": { - "findings": [], - "summary": "The change adds a public `summarise` module to the findings module. It introduces no untrusted input paths, authentication or authorization changes, network/subprocess calls, deserialization, path traversal, secret handling, or dependency changes." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-877af0253e3927ce.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-877af0253e3927ce.json new file mode 100644 index 0000000..40a8dad --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-877af0253e3927ce.json @@ -0,0 +1,17 @@ +{ + "key": "877af0253e3927ce", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "z-ai/glm-5.2", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 2667, + "output_tokens": 200, + "cached_tokens": 2295, + "embed_tokens": 0, + "cost_usd": 0.0004794 + }, + "value": { + "summary": "The change adds a new `summarise` submodule declaration to `src/findings/mod.rs`. It is a one-line, low-risk addition with no security-relevant surface.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-fb11783005e5e9fd.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-fb11783005e5e9fd.json deleted file mode 100644 index f0c87fe..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0014-fb11783005e5e9fd.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "fb11783005e5e9fd", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3399, - "output_tokens": 648, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.002042325 - }, - "value": { - "findings": [], - "summary": "The change updates the compose environment to require the GitHub App id/private key and to forward only the relevant API keys, and mounts a read-only config file; no security issues found." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-0fdeb27f432cc6dd.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-0fdeb27f432cc6dd.json deleted file mode 100644 index a1c867f..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-0fdeb27f432cc6dd.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "0fdeb27f432cc6dd", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3514, - "output_tokens": 1410, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.002313562 - }, - "value": { - "findings": [], - "summary": "This adds `shorten` and `first_sentence` helpers for truncating model-authored text at character boundaries, with unit tests. The code handles multibyte input safely and introduces no security issues." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-3651492c72f6983f.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-3651492c72f6983f.json deleted file mode 100644 index 633ee78..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-3651492c72f6983f.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "3651492c72f6983f", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3166, - "output_tokens": 744, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00202449 - }, - "value": { - "findings": [], - "summary": "The overlay adds a workaround for running MongoDB on newer Linux kernels by setting GLIBC_TUNABLES and mounting a custom entrypoint script. The script itself is not part of this diff and cannot be reviewed here. The change is explicitly scoped to local development and introduces no new security issues that are visible from the diff." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-7d0a235486943283.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-7d0a235486943283.json deleted file mode 100644 index 3f3d6dd..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-7d0a235486943283.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "7d0a235486943283", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 1969, - "output_tokens": 706, - "cached_tokens": 1752, - "embed_tokens": 0, - "cost_usd": 0.000193286 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The new docker-compose overlay is sound. It mounts a local Python script read-only into the container, sets a documented `GLIBC_TUNABLES` mitigation, and is clearly scoped to local development. No secrets, network calls, widened permissions, or untrusted-input sinks are introduced by this file. The referenced `deploy/mongo/kernel-bypass-entrypoint.py` is not part of this diff and could not be reviewed here." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-b136475a5e9b393f.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-b136475a5e9b393f.json new file mode 100644 index 0000000..4cbebe0 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0015-b136475a5e9b393f.json @@ -0,0 +1,17 @@ +{ + "key": "b136475a5e9b393f", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "z-ai/glm-5.2", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 2998, + "output_tokens": 519, + "cached_tokens": 2363, + "embed_tokens": 0, + "cost_usd": 0.0009088752 + }, + "value": { + "summary": "This new compose overlay adds a glibc tunable and a read-only bind-mounted Python shim to work around a MongoDB startup failure on newer Linux kernels, scoped to local development. No secrets, dangerous sinks, or widened permissions are introduced; the change looks safe to merge.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-24edd5723968eefd.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-24edd5723968eefd.json deleted file mode 100644 index 0e5bf29..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-24edd5723968eefd.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "24edd5723968eefd", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 1638, - "output_tokens": 153, - "cached_tokens": 1425, - "embed_tokens": 0, - "cost_usd": 0.000067095 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change adds a `pub mod summarise;` declaration to `src/findings/mod.rs`. This introduces no untrusted input handling, dangerous sinks, authentication/authorization changes, network or subprocess calls, deserialization, path traversal, secret exposure, dependency changes, or CI permission widening. No security concerns." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-89683e319847d619.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-89683e319847d619.json deleted file mode 100644 index ff23b44..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-89683e319847d619.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "key": "89683e319847d619", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_tests", - "usage": { - "input_tokens": 5685, - "output_tokens": 13473, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.014194485 - }, - "value": { - "findings": [ - { - "body": "`shorten` promises at most `limit` characters, but when the first `limit` characters contain no ASCII space the `None` branch appends an ellipsis to the full `limit`-character `head`, returning `limit + 1` characters (and cutting mid-word). The multibyte test asserts `<= 21` for `limit = 20`, so it accepts this violation. Take only `limit - 1` characters in that branch (and return an empty string for `limit == 0`) so the result stays within the limit.", - "confidence": 0.95, - "existing_code": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }", - "late": false, - "path": "src/findings/summarise.rs", - "rule": "incorrect-truncation-limit", - "severity": "low", - "suggestion": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{}…\", head.chars().take(limit.saturating_sub(1)).collect::()),\n }", - "title": "Honor the character limit when no word boundary exists" - }, - { - "body": "These assertions only require a trailing ellipsis and a byte length at most one over the limit. A regression that cut mid-word, such as `\"Guard the index befo…\"`, would pass. Assert the exact expected string (`\"Guard the index…\"`) so the word-boundary behaviour is actually pinned.", - "confidence": 0.9, - "existing_code": " let out = shorten(\"Guard the index before dereferencing the slice\", 20);\n assert!(out.ends_with('…'), \"{out}\");\n assert!(out.len() <= 21, \"{out}\");", - "late": false, - "path": "src/findings/summarise.rs", - "rule": "weak-test", - "severity": "low", - "suggestion": " assert_eq!(shorten(\"Guard the index before dereferencing the slice\", 20), \"Guard the index…\");", - "title": "Pin the word-boundary test to the expected truncation" - }, - { - "body": "The repository rule requires one document per `src/` module at `docs/modules//README.md`. The new `summarise` module has only an inline `//!` comment; add `docs/modules/findings/summarise/README.md` describing its role and any feature gating.", - "confidence": 0.75, - "existing_code": "pub mod summarise;", - "late": false, - "path": "src/findings/mod.rs", - "rule": "missing-docs", - "severity": "low", - "suggestion": null, - "title": "Add the required module README for summarise" - } - ], - "resolved": [], - "summary": "The change adds deployment configuration and a text-summarisation helper, but the helper has a truncation bug and an under-asserting test; merge after addressing the findings." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-96d6b84e54ee3b43.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-96d6b84e54ee3b43.json deleted file mode 100644 index 5fbf945..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-96d6b84e54ee3b43.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "96d6b84e54ee3b43", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2943, - "output_tokens": 1048, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.002191965 - }, - "value": { - "findings": [], - "summary": "The change adds configuration for embeddings retrieval using an API key from the environment; no security issues introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-fdcd6803e0c62746.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-fdcd6803e0c62746.json new file mode 100644 index 0000000..b16c716 --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0016-fdcd6803e0c62746.json @@ -0,0 +1,17 @@ +{ + "key": "fdcd6803e0c62746", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_tests", + "usage": { + "input_tokens": 5676, + "output_tokens": 103, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00028739452 + }, + "value": { + "summary": "This pull request adds a bypass script for the MongoDB kernel compatibility check, wires the embeddings config and GitHub App credentials into the Docker compose file, and introduces a `shorten`/`first_sentence` helper for truncating model-authored text. The behavioural changes are infrastructure (deploy/compose) and a well-tested utility module; no test was missed and nothing here introduces a regressable risk. Safe to merge.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-1f308da0fcce4684.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-1f308da0fcce4684.json deleted file mode 100644 index 7ee8c52..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-1f308da0fcce4684.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "1f308da0fcce4684", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3643, - "output_tokens": 2026, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.003347325 - }, - "value": { - "findings": [], - "summary": "The change adds an opt-in compose overlay script that patches and runs the MongoDB image's entrypoint to disable a non-security kernel compatibility check. It reads from a fixed path and executes no untrusted input; no security issues introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-2dc7e57b83706c42.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-2dc7e57b83706c42.json deleted file mode 100644 index 439e2b2..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-2dc7e57b83706c42.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "key": "2dc7e57b83706c42", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_description", - "usage": { - "input_tokens": 5739, - "output_tokens": 9564, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.010817145 - }, - "value": { - "findings": [ - { - "body": "`shorten` documents it shortens to at most `limit` characters, but the no-space branch returns the first `limit` characters plus the ellipsis, producing `limit + 1` characters. A caller sizing `limit` to GitHub's exact check-run title ceiling can therefore still exceed it. Reserve one character for the ellipsis before taking the head, or adjust the contract and tests if the extra character is intentional.", - "confidence": 0.9, - "existing_code": " let head: String = text.chars().take(limit).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }", - "path": "src/findings/summarise.rs", - "rule": "off-by-one", - "severity": "medium", - "suggestion": " let head: String = text.chars().take(limit.saturating_sub(1)).collect();\n match head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n }", - "title": "Keep shorten output within the stated limit" - }, - { - "body": "The title says the PR adds a summarise helper, and the body only acknowledges that helper and the end-to-end test. The diff also adds a new `[embeddings]` block to `.tinysweeper.toml`, forwards OpenRouter/Cohere keys in `docker-compose.yml`, renames the GitHub App environment variables to the names the server actually reads, mounts the config into the container, and introduces a MongoDB kernel-bypass overlay. None of these are described. Update the body to list all significant changes, or split this into the helper change and the deployment/e2e change, so reviewers can see what the diff actually does.", - "confidence": 0.95, - "existing_code": "[embeddings]\nenabled = true\nprovider = \"openrouter\"\nmodel = \"openai/text-embedding-3-small\"\ndimensions = 1536\napi_key_env = \"OPENROUTER_API_KEY\"", - "path": ".tinysweeper.toml", - "rule": "description-mismatch", - "severity": "medium", - "title": "Mention the deployment configuration in the PR description" - } - ], - "summary": "The PR adds `findings::summarise` and end-to-end deployment configuration. The helper can exceed its stated length limit, and the PR description omits the substantial deployment changes, so it is not ready to merge until both are addressed." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-67216aa5018d3c92.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-67216aa5018d3c92.json new file mode 100644 index 0000000..f926afd --- /dev/null +++ b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-67216aa5018d3c92.json @@ -0,0 +1,17 @@ +{ + "key": "67216aa5018d3c92", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_description", + "usage": { + "input_tokens": 5730, + "output_tokens": 99, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00028964208 + }, + "value": { + "summary": "This pull request adds infrastructure for end-to-end deployment: a kernel-bypass shim for MongoDB on modern Linux kernels, a compose overlay to use it, environment variable and volume fixes so the server can authenticate and load configuration, and an embeddings configuration. It also adds a `findings::summarise` helper for truncating model-authored text. The changes are well-motivated and sound; nothing blocks merge.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-da2bcfbc6612fc4f.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-da2bcfbc6612fc4f.json deleted file mode 100644 index 894c7f0..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0017-da2bcfbc6612fc4f.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "da2bcfbc6612fc4f", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2498, - "output_tokens": 846, - "cached_tokens": 2148, - "embed_tokens": 0, - "cost_usd": 0.000238544 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The new `summarise.rs` module is a small, self-contained text utility. Both `shorten` and `first_sentence` operate on in-memory strings with no network, subprocess, deserialization, filesystem, or secret-handling surface. The multibyte-slicing concern noted in comments is handled correctly: `shorten` works on a `chars()`-collected `String` and `rfind(' ')`/byte slicing on ASCII space is always on a char boundary; `first_sentence`'s `find`/slice boundaries land on ASCII `.`, also safe. No security problems introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0018-3e0fe1f1ab7eef77.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0018-3e0fe1f1ab7eef77.json deleted file mode 100644 index 9d29987..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0018-3e0fe1f1ab7eef77.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "key": "3e0fe1f1ab7eef77", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_tests", - "usage": { - "input_tokens": 5813, - "output_tokens": 8769, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.010157685 - }, - "value": { - "findings": [ - { - "body": "This test only checks that the output ends with `…` and is not longer than `limit + 1`. An implementation that truncates mid-word (for example `text.chars().take(limit).collect()` plus an ellipsis) would still pass both assertions. Assert the exact expected result, or at least that the character before the ellipsis is a space, so the word-boundary requirement is actually tested.", - "confidence": 0.9, - "existing_code": " assert!(out.ends_with('…'), \"{out}\");\n assert!(out.len() <= 21, \"{out}\");", - "path": "src/findings/summarise.rs", - "rule": "weak-test-assertion", - "severity": "low", - "suggestion": " assert_eq!(out, \"Guard the index…\");\n assert_eq!(out.chars().count(), 16);", - "title": "Assert the actual word-boundary result, not just an ellipsis" - }, - { - "body": "The documented contract is that `shorten` returns at most `limit` characters, but this assertion allows `limit + 1` (21 for a limit of 20), and the no-space branch genuinely returns 21 characters. The multibyte no-space fixture should assert `<= 20` so it catches the off-by-one; otherwise the test passes despite the contract violation.", - "confidence": 0.9, - "existing_code": " assert!(out.chars().count() <= 21, \"{out}\");", - "path": "src/findings/summarise.rs", - "rule": "weak-test-assertion", - "severity": "medium", - "title": "Enforce the at-most-limit contract on the no-space path" - } - ], - "summary": "This PR wires config and compose settings and adds a `summarise` module with in-crate tests. The new tests do not yet enforce the module's stated contracts (word-boundary truncation and an at-most-`limit` result), so they should be tightened before merge." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0018-7efb557b9ec78cdc.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0018-7efb557b9ec78cdc.json deleted file mode 100644 index 8344f63..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0018-7efb557b9ec78cdc.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "key": "7efb557b9ec78cdc", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_tests", - "usage": { - "input_tokens": 4685, - "output_tokens": 3565, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00111225 - }, - "value": { - "findings": [ - { - "body": "The `None` branch of the `match` in `shorten` is never exercised by any test. It handles input whose first `limit` characters contain no space — CJK text, long single words, URLs — and is reachable with real model output. Every test fixture either returns early (string ≤ limit) or finds a space, so if someone broke the no-space branch (e.g. returned the full string untruncated, or panicked), no test would catch it.\n\n```rust\nmatch head.rfind(' ') {\n Some(space) => format!(\"{}…\", &head[..space]),\n None => format!(\"{head}…\"),\n}\n```", - "confidence": 0.82, - "existing_code": " None => format!(\"{head}…\"),", - "path": "src/findings/summarise.rs", - "rule": "Maintain at least 80% coverage for meaningful library behaviour", - "severity": "medium", - "title": "Exercise the no-space branch of shorten" - }, - { - "body": "The `a_multibyte_title_does_not_panic` test claims to verify that multibyte truncation does not panic, but the Japanese fixture is only 17 characters — shorter than the limit of 20 — so `shorten` returns it unchanged and never reaches the character-slicing path the test is supposed to guard. The fixture passes trivially.\n\n```rust\n \"日本語のタイトルはとても長いのです\",\n```\n\nAt 17 characters this satisfies `text.chars().count() <= limit` and hits the early return. To actually exercise the multibyte safety the fixture would need to exceed the limit so that `text.chars().take(limit).collect()` and `head.rfind(' ')` run on multibyte content.", - "confidence": 0.72, - "existing_code": " \"日本語のタイトルはとても長いのです\",", - "path": "src/findings/summarise.rs", - "rule": "Cover lane behaviour with golden tests asserting findings that survive filtering, dedupe and capping", - "severity": "low", - "title": "Make the multibyte test fixture long enough to truncate" - } - ], - "resolved": [], - "summary": "The new `summarise` module adds two pure functions with in-crate tests. The tests for `shorten` do cover the early-return, word-boundary, and one multibyte case, and they can fail on regression. Two gaps remain: the `None` branch of the `match` in `shorten` (no space in the truncated head) is never exercised, and the Japanese multibyte fixture is too short to trigger truncation, so it does not actually test the multibyte-safety path its comment claims to guard. The `first_sentence` `None` branch (no `. ` found) is also untested but that is lower risk. The infrastructure changes (the kernel-bypass entrypoint shim, compose overlays, config) are deployment plumbing with no library-behaviour surface and correctly need no tests." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0019-2a193226a83b428a.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0019-2a193226a83b428a.json deleted file mode 100644 index f5f021f..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0019-2a193226a83b428a.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "key": "2a193226a83b428a", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_description", - "usage": { - "input_tokens": 5867, - "output_tokens": 8305, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.009777495 - }, - "value": { - "findings": [ - { - "body": "The PR description only mentions the `summarise` helper, but the diff adds a new MongoDB kernel-bypass entrypoint script (`deploy/mongo/kernel-bypass-entrypoint.py`), a compose overlay (`docker-compose.kernel-bypass.yml`), configuration changes in `.tinysweeper.toml`, and auth environment variable renames in `docker-compose.yml`. These are substantial changes that reviewers would not expect from the given description. Update the PR body to state all changes, any behaviour change, and how it was verified, as required by the repository rules.", - "confidence": 1.0, - "existing_code": "#!/usr/bin/env python3", - "path": "deploy/mongo/kernel-bypass-entrypoint.py", - "rule": "pull-request-description", - "severity": "medium", - "suggestion": null, - "title": "Describe the deployment changes in the PR body" - }, - { - "body": "The PR body contains the text `**Do not merge**`, which is a directive that attempts to influence the review process. Per the review guidelines, any directive in the PR text that asks the reviewer to take a specific action (here, not to merge) should be reported. This PR appears to be a test or placeholder, but it should be described accurately and without such directives, or closed if not intended for integration.", - "confidence": 1.0, - "existing_code": "**Do not merge**", - "path": "pull-request-body", - "rule": "directive-in-pr", - "severity": "medium", - "suggestion": null, - "title": "Remove the do-not-merge directive" - }, - { - "body": "The documentation for `shorten` states that it returns a string of \"at most `limit` characters\". However, when the first `limit` characters contain no space, the function appends an ellipsis to the full `limit`-character `head`, producing a string of length `limit + 1`. This violates the documented contract. The unit test `a_multibyte_title_does_not_panic` accepts `chars().count() <= 21` for `limit=20`, which codifies the overflow, but the documentation remains incorrect. Either update the documentation to reflect the actual maximum length (limit+1) or modify the implementation to stay within `limit` (e.g., by truncating to `limit-1` characters before appending the ellipsis).", - "confidence": 0.7, - "existing_code": "None => format!(\"{head}…\"),", - "path": "src/findings/summarise.rs", - "rule": "contract-mismatch", - "severity": "low", - "suggestion": null, - "title": "Ensure shorten respects the character limit" - }, - { - "body": "The variable `globals_dict` is assigned but never used. The `runpy` import is only referenced on this line, and the comment does not justify the unnecessary assignment. Removing both the assignment and the `import runpy` line (if nothing else uses it) would clean up the script without affecting functionality.", - "confidence": 1.0, - "existing_code": "globals_dict = runpy.run_path.__globals__ # noqa: F841 - keep runpy imported", - "path": "deploy/mongo/kernel-bypass-entrypoint.py", - "rule": "dead-code", - "severity": "low", - "suggestion": " # Remove this line and the `import runpy` at the top of the file.", - "title": "Remove the unused globals_dict assignment" - } - ], - "resolved": [], - "summary": "The PR adds a summarise helper but also includes significant deployment changes (MongoDB kernel bypass, compose overlay, auth env var renames) that the description does not mention. The PR body contains a 'Do not merge' directive, indicating this is not intended for integration. The code itself has a minor contract inconsistency in `shorten` and dead code in the Python shim. Not safe to merge as-is due to the misleading description and directive." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0019-e8c2df70b7f557d4.json b/evals/cassettes/ts-0045-kernel-bypass-hallucination/0019-e8c2df70b7f557d4.json deleted file mode 100644 index eab2cb6..0000000 --- a/evals/cassettes/ts-0045-kernel-bypass-hallucination/0019-e8c2df70b7f557d4.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "key": "e8c2df70b7f557d4", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_description", - "usage": { - "input_tokens": 4743, - "output_tokens": 1875, - "cached_tokens": 4126, - "embed_tokens": 0, - "cost_usd": 0.000509328 - }, - "value": { - "findings": [ - { - "body": "The PR description is inaccurate for this diff. The title and body frame the change as adding `findings::summarise` (`shorten`, `first_sentence`) for an end-to-end deployment test, but the diff introduces several significant, unmentioned changes: a new kernel-bypass entrypoint shim (`deploy/mongo/kernel-bypass-entrypoint.py`) and compose overlay (`docker-compose.kernel-bypass.yml`), a renamed and now-required set of GitHub App environment variables in `docker-compose.yml` (`TINYSWEEPER_GITHUB_APP_ID` → `TINYSWEEPER_APP_ID`, `TINYSWEEPER_GITHUB_PRIVATE_KEY` → `TINYSWEEPER_APP_PRIVATE_KEY`), a read-only mount of `.tinysweeper.toml` into the container, and an `[embeddings]` block added to `.tinysweeper.toml`. The env-var rename is a behaviour change — the old names were never read by the binary — and the repository rules require the PR to state what changed and any behaviour change. None of this is mentioned.", - "confidence": 0.95, - "existing_code": " TINYSWEEPER_APP_ID: ${TINYSWEEPER_APP_ID:?set the GitHub App id}\n TINYSWEEPER_APP_PRIVATE_KEY: ${TINYSWEEPER_APP_PRIVATE_KEY:?set the GitHub App private key}", - "path": "docker-compose.yml", - "rule": "Pull requests state what changed, any behaviour change, and how it was verified.", - "severity": "high", - "suggestion": "Rewrite the PR body to cover every change in the diff, for example:\n\n---\n\nEnd-to-end deployment test. **Do not merge** — this exists to exercise the live server and will be closed.\n\n## What changed\n\n- **`findings::summarise`**: new module with `shorten` (truncate on a word boundary, for check-run titles) and `first_sentence`, both with tests. `shorten` works on character boundaries to avoid panicking on multibyte text.\n- **GitHub App auth fix in `docker-compose.yml`**: the env vars were previously `TINYSWEEPER_GITHUB_APP_ID` / `TINYSWEEPER_GITHUB_PRIVATE_KEY`, which `server::auth::AppAuth::from_env` never reads. Renamed to `TINYSWEEPER_APP_ID` / `TINYSWEEPER_APP_PRIVATE_KEY` and made them required (`:?`), so the container now fails loud instead of silently lacking App credentials.\n- **Embeddings config (`.tinysweeper.toml`)**: added an `[embeddings]` block with `provider = \"openrouter\"`, `model = \"openai/text-embedding-3-small\"`, `dimensions = 1536`, `api_key_env = \"OPENROUTER_API_KEY\"`. Forwarded `OPENROUTER_API_KEY`, `COHERE_API_KEY`, and `VOYAGE_API_KEY` into the container.\n- **Config mount in `docker-compose.yml`**: mounted `.tinysweeper.toml` read-only into the container so `[embeddings]` is on; without it the server runs on built-in defaults with retrieval off.\n- **MongoDB kernel-bypass overlay**: new `deploy/mongo/kernel-bypass-entrypoint.py` and `docker-compose.kernel-bypass.yml` for Linux kernels ≥ 6.19, where the community-server image 8.2.12-ubi9 refuses to start `mongod` via `enforce_kernel_compatibility`. The shim patches that function to a no-op and sets `GLIBC_TUNABLES=glibc.pthread.rseq=0`; opt-in overlay only, never the default.\n\n## Behaviour changes\n\n- GitHub App env var names changed and are now required.\n- The runtime container now reads a mounted `.tinysweeper.toml` with embeddings enabled.\n\n## How verified\n\nBrought up the full stack for the first time: MongoDB 8.2 with mongot, hybrid search live, OpenRouter embeddings, and the webhook path.", - "title": "Add a helper for shortening model-authored text" - } - ], - "resolved": [], - "summary": "The description does not match the diff. The title and body describe only the `findings::summarise` helper, but the diff also adds a MongoDB kernel-bypass shim and compose overlay, renames and makes required the GitHub App environment variables (a behaviour change), mounts `.tinysweeper.toml` into the container, and adds an `[embeddings]` configuration block. None of those are mentioned in the PR body, which the repository rules require. A corrected body is proposed in the suggestion." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0001-8d8a52bdbefb3833.json b/evals/cassettes/ts-0068-description-anchored-to-code/0001-8d8a52bdbefb3833.json deleted file mode 100644 index 36fb3d9..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0001-8d8a52bdbefb3833.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "8d8a52bdbefb3833", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_rule_extraction", - "usage": { - "input_tokens": 1639, - "output_tokens": 2898, - "cached_tokens": 1536, - "embed_tokens": 0, - "cost_usd": 0.002571633 - }, - "value": { - "rules_markdown": "- One responsibility per module; core types in module-local types.rs; each port is one trait in one file.\n- Every CLI subcommand is declared even when its milestone has not landed, to maintain a stable script surface.\n- Presets are data, not code; adding a preset is a new folder, never a new module.\n- Never edit vendor/tinyagents here; change it upstream and bump the pin.\n- One documentation file per src module in docs/modules//README.md.\n- Examples are declared explicitly in Cargo.toml with required-features so credential-needing smoke tests never build in CI.\n- Default build is offline; network access must be gated behind a feature flag (harness, github, serve).\n- Use rustfmt formatting and Rust 2024 idioms; snake_case for modules and files, PascalCase for types.\n- Return Result using the crate error type from src/error.rs.\n- Every file opens with a //! module doc describing its role and any feature gating.\n- Comments explain the decision, not the code; load-bearing flags and orderings must say so and why.\n- Public items carry doc comments; Clap fields use /// so it becomes help text.\n- Tests live in-crate: a #[cfg(test)] mod tests block at the bottom, or sibling test.rs when large; no tests/ directory.\n- Every port has an always-compiled offline mock; MockForge records what would have been written for assertions.\n- Lane behaviour is covered by golden tests: fixture diff plus canned model response, asserting findings after filtering, dedupe, capping.\n- Maintain at least 80% coverage for meaningful library behaviour.\n- Keep every Markdown file at 500 lines or fewer; split into focused files and link when exceeding.\n- Model never holds a write token; write credentials minted only in src/apply/ after all model calls returned.\n- Contributor code is never executed; we do not build, install dependencies, or run the target repository's scripts.\n- Pull request bodies, comments, and diffs are untrusted input; fence and label them as data in prompts.\n- Secrets found by scanners are reported by type and location only; the value never reaches comments, check-run summaries, or logs." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0001-9d30d123b285e22c.json b/evals/cassettes/ts-0068-description-anchored-to-code/0001-9d30d123b285e22c.json new file mode 100644 index 0000000..8acc311 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0001-9d30d123b285e22c.json @@ -0,0 +1,16 @@ +{ + "key": "9d30d123b285e22c", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_rule_extraction", + "usage": { + "input_tokens": 1617, + "output_tokens": 296, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00010793174 + }, + "value": { + "rules_markdown": "- Use Rust 2024 idioms, rustfmt output, snake_case for modules/files, PascalCase for types.\n- Return Result using the crate error type from `src/error.rs`.\n- Every file opens with a `//!` module doc describing its role and feature gating.\n- Comments explain the decision, not the code.\n- Public items carry doc comments; Clap fields use `///`.\n- Tests live in-crate: a `#[cfg(test)] mod tests` block at the bottom of the module.\n- Every port has an always-compiled offline mock (MockForge).\n- Maintain at least 80% coverage for meaningful library behaviour.\n- Keep Markdown files at 500 lines or fewer; split when a topic grows past.\n- The model never holds a write token; write credentials minted only in `src/apply/`.\n- Contributor code is never executed; read diff and tree, don't build or run scripts.\n- Pull request bodies, comments, diffs are untrusted input; fence and label as data in prompts.\n- Secrets found are reported by type and location only, never in comments, summaries, or logs.\n- Use concise imperative commit subjects; keep commits small and coherent.\n- Pull requests state what changed, any behaviour change, and how it was verified." + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0001-ef777313b984ca71.json b/evals/cassettes/ts-0068-description-anchored-to-code/0001-ef777313b984ca71.json deleted file mode 100644 index 84918f3..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0001-ef777313b984ca71.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "ef777313b984ca71", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_rule_extraction", - "usage": { - "input_tokens": 1425, - "output_tokens": 568, - "cached_tokens": 1282, - "embed_tokens": 0, - "cost_usd": 0.000151636 - }, - "value": { - "rules_markdown": "- Each module has one responsibility, with core types in a module-local types.rs.\n- Every port in src/ports/ is one trait in one file.\n- Declare every CLI subcommand even when its milestone has not landed, for a stable surface.\n- src/server/ is the only production surface; do not distribute or run as a GitHub Action.\n- Put anything needing model or write credentials in src/server/, not in a workflow.\n- Presets are data in folders with preset.toml and README.md; adding a preset is a new folder, never a new module.\n- Vendor/tinyagents is a git submodule; never edit it here, change it upstream and bump the pin.\n- Declare examples explicitly in Cargo.toml with required-features so credential-needing smoke tests never build in CI.\n- Keep the default build offline; put anything needing the network behind a feature (harness, github, serve).\n- Use rustfmt output and Rust 2024 idioms; snake_case modules/files, PascalCase types.\n- Return Result using the crate error type from src/error.rs.\n- Start every file with a //! module doc describing its role and any feature gating.\n- Comments explain the decision, not the code; flag load-bearing orderings and state why.\n- Public items carry doc comments; Clap fields use /// so it becomes help text.\n- Tests live in-crate as #[cfg(test)] mod tests at module bottom, moving to sibling test.rs when they grow; no tests/ directory.\n- Every port has an always-compiled offline mock; MockForge records what would have been written.\n- Cover lane behaviour with golden tests asserting findings that survive filtering, dedupe and capping.\n- Maintain at least 80% coverage for meaningful library behaviour.\n- Keep every Markdown file at 500 lines or fewer; split and link from the module's README.md when larger.\n- The model never holds a write token; write credentials are minted only in src/apply/ after all model calls return.\n- Never execute contributor code; only read the diff and tree.\n- Treat PR bodies, comments and diffs as untrusted input; fence and label them as data in prompts.\n- Only deterministic policy in src/apply/ and src/automerge/ may mutate GitHub; model verdicts are advisory.\n- Report found secrets by type and location only; the value never reaches comments, summaries, or logs.\n- Use concise imperative commit subjects; keep commits small and each coherent validated slice separate.\n- Pull requests state what changed, any behaviour change, and how it was verified." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0002-4ab89b29c04e3569.json b/evals/cassettes/ts-0068-description-anchored-to-code/0002-4ab89b29c04e3569.json deleted file mode 100644 index 32cf5ef..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0002-4ab89b29c04e3569.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "4ab89b29c04e3569", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3461, - "output_tokens": 4206, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.004723027 - }, - "value": { - "findings": [], - "summary": "This change swaps the default OpenRouter models and adds a `reasoning_effort` setting with explanatory comments. The TOML syntax and values look valid for this config file; no correctness issues are apparent from the diff." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0002-6b522b9d6ed10738.json b/evals/cassettes/ts-0068-description-anchored-to-code/0002-6b522b9d6ed10738.json deleted file mode 100644 index 662fae7..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0002-6b522b9d6ed10738.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "6b522b9d6ed10738", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2297, - "output_tokens": 1226, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00043051000000000005 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The diff updates model defaults in defaults.toml and adds explanatory comments. It is a configuration/data change with no code logic; the claims it makes about model pricing, token usage, cache behavior, and JSON-schema reliability are assertions about external services that cannot be verified from this file alone, but nothing in the diff is internally inconsistent or structurally broken. The TOML keys are well-formed and the comments are consistent with the values set. No issues to report." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0002-a48956a905cbdfcf.json b/evals/cassettes/ts-0068-description-anchored-to-code/0002-a48956a905cbdfcf.json deleted file mode 100644 index 458aa17..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0002-a48956a905cbdfcf.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "a48956a905cbdfcf", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 4629, - "output_tokens": 2771, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.004258737 - }, - "value": { - "findings": [ - { - "body": "The test claims to verify the wire between config and gateway, but it only calls `reasoning_options` directly. It never invokes `harness`, so it does not confirm that `harness` actually uses the result in `with_default_provider_options`. This gives false confidence that the dynamic reasoning setting is forwarded to the provider. To truly test the integration, the test should call `harness` and inspect the provider options, or the options-building logic should be separated from the wiring and tested independently.", - "confidence": 0.6, - "existing_code": " assert_eq!(\n reasoning_options(&gateway.reasoning_effort),\n json!({ \"reasoning\": { \"enabled\": false } })\n );", - "path": "src/harness/openrouter.rs", - "rule": "test-gap", - "severity": "low", - "suggestion": null, - "title": "Integration test does not exercise the harness wiring" - } - ], - "summary": "The change adds configurable reasoning effort and correctly replaces a hardcoded value with a function that maps the configured effort to provider options. The production code appears sound; the only issue is that the new integration test does not actually call the harness, so it fails to verify that the option is forwarded." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0002-e91f6f655e66852e.json b/evals/cassettes/ts-0068-description-anchored-to-code/0002-e91f6f655e66852e.json new file mode 100644 index 0000000..8747012 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0002-e91f6f655e66852e.json @@ -0,0 +1,17 @@ +{ + "key": "e91f6f655e66852e", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 4410, + "output_tokens": 72, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00022250844 + }, + "value": { + "summary": "This change makes reasoning effort configurable per model deployment, replacing the hard-coded disabled reasoning with a `reasoning_options` function that respects a new `reasoning_effort` field. The tests cover the wire from config to gateway, and the change appears safe to merge.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0003-103003f05e8c9269.json b/evals/cassettes/ts-0068-description-anchored-to-code/0003-103003f05e8c9269.json deleted file mode 100644 index cba5725..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0003-103003f05e8c9269.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "key": "103003f05e8c9269", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3022, - "output_tokens": 4371, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.004675612 - }, - "value": { - "findings": [ - { - "body": "The new `reasoning_effort` field is a plain required `String`. If this type is deserialized from user config (as its module and sibling fields imply), every existing config file without `reasoning_effort` now fails to load with a missing-field error unless `#[serde(default)]` is applied elsewhere. It also makes any existing struct literal that does not use `..Default::default()` fail to compile. Provide a serde default — preferably `off`, per the doc comment — or make the field optional until rollout.", - "confidence": 0.5, - "existing_code": " pub reasoning_effort: String,", - "late": false, - "path": "src/config/types.rs", - "rule": "required-config-field", - "severity": "medium", - "suggestion": null, - "title": "Provide a default for the new reasoning_effort field" - } - ], - "resolved": [], - "summary": "This change adds and documents a `reasoning_effort` model setting. It is not safe to merge as written because the new field has no default, so existing serialized configs and struct initializers can break; give it a default (`off`) or make it optional." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0003-d44795407ec1e3eb.json b/evals/cassettes/ts-0068-description-anchored-to-code/0003-d44795407ec1e3eb.json deleted file mode 100644 index c9d963b..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0003-d44795407ec1e3eb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "d44795407ec1e3eb", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 2776, - "output_tokens": 227, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.00112897 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0003-e2a5ced825a44efc.json b/evals/cassettes/ts-0068-description-anchored-to-code/0003-e2a5ced825a44efc.json deleted file mode 100644 index ec569f9..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0003-e2a5ced825a44efc.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "key": "e2a5ced825a44efc", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 1873, - "output_tokens": 2080, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00058871 - }, - "value": { - "findings": [ - { - "body": "The new `reasoning_effort` field has no visible `#[serde(default)]`, so if this struct derives `serde::Deserialize` and existing config files lack the key, deserialization will fail for every deployment that has not been updated. The doc comment itself frames `off` as the safe recovery value, so the field should default to `\"off\"` (e.g. `#[serde(default = \"...\")]` or a typed enum with a `Default` impl) rather than being mandatory.", - "confidence": 0.55, - "existing_code": " pub reasoning_effort: String,", - "path": "src/config/types.rs", - "rule": "", - "severity": "medium", - "title": "Add a serde default for reasoning_effort to avoid breaking existing configs" - }, - { - "body": "The field is documented as accepting only `off`, `low`, `medium`, `high`, but typed as `String`. If the struct derives `Default` (common for config types), the default will be `\"\"`, which is none of the valid values and will likely be passed straight through to the model API. An enum with `#[default]` on an `Off` variant would make the default meaningful and catch typos at compile time.", - "confidence": 0.35, - "existing_code": " pub reasoning_effort: String,", - "path": "src/config/types.rs", - "rule": "", - "severity": "low", - "title": "Default-constructing this struct yields an invalid reasoning_effort" - } - ], - "resolved": [], - "summary": "The PR adds a `reasoning_effort: String` field to a public config struct. The field's doc comments are thorough, but the type itself raises two correctness concerns: existing config files may fail to deserialize if the field is required, and a `Default`-derived instance would produce an empty string that is not one of the four documented valid values. Both depend on attributes not visible in the diff, so confidence is moderated accordingly." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0003-ec3338063d329eeb.json b/evals/cassettes/ts-0068-description-anchored-to-code/0003-ec3338063d329eeb.json new file mode 100644 index 0000000..774b923 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0003-ec3338063d329eeb.json @@ -0,0 +1,17 @@ +{ + "key": "ec3338063d329eeb", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 4105, + "output_tokens": 69, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00020731298 + }, + "value": { + "summary": "Adds new model price entries for qwen, glm, and two deepseek variants, plus three tests that verify the pricing function works as documented. The change looks correct and the tests guard against the known silent-argument-ordering hazard. No bugs were found.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0004-08578bcd9582f891.json b/evals/cassettes/ts-0068-description-anchored-to-code/0004-08578bcd9582f891.json deleted file mode 100644 index 7ed7441..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0004-08578bcd9582f891.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "08578bcd9582f891", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 895, - "output_tokens": 512, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00017529 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0004-2908e203d01996cc.json b/evals/cassettes/ts-0068-description-anchored-to-code/0004-2908e203d01996cc.json deleted file mode 100644 index 3f6c90c..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0004-2908e203d01996cc.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "key": "2908e203d01996cc", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 2912, - "output_tokens": 2791, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.003529242 - }, - "value": { - "findings": [ - { - "body": "Adding `pub reasoning_effort: String` without a `#[serde(default)]` attribute makes the field required when deserializing existing config files. Users with a config written before this change will get a parse error because the field is missing. Since the field is new, backward compatibility requires a default value (likely `\"off\"` to preserve prior behavior). Consider adding `#[serde(default = \"default_reasoning_effort\")]` or a similar default.", - "confidence": 0.6, - "existing_code": " /// nothing, which is a real failure this repository has measured.\n pub reasoning_effort: String,", - "late": false, - "path": "src/config/types.rs", - "rule": "missing-serde-default", - "severity": "high", - "suggestion": null, - "title": "Add a serde default to reasoning_effort to keep existing configs valid" - } - ], - "summary": "Adds a new public `reasoning_effort` field to the config model with documentation; the field is required and will likely break deserialization of existing configuration files unless a serde default is provided." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0004-3ebbd6a0fecdb0db.json b/evals/cassettes/ts-0068-description-anchored-to-code/0004-3ebbd6a0fecdb0db.json deleted file mode 100644 index 12f28fa..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0004-3ebbd6a0fecdb0db.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "3ebbd6a0fecdb0db", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1151, - "output_tokens": 286, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.000473425 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0004-d772af2ca812230d.json b/evals/cassettes/ts-0068-description-anchored-to-code/0004-d772af2ca812230d.json new file mode 100644 index 0000000..44dd5f0 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0004-d772af2ca812230d.json @@ -0,0 +1,28 @@ +{ + "key": "d772af2ca812230d", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 3132, + "output_tokens": 295, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00018185692 + }, + "value": { + "summary": "Updates model configuration in `defaults.toml` to use a cheaper, reliable single model for both scan and deep tiers, replaces fallback models, and adds a reasoning_effort setting. The change is well-documented but introduces a potential configuration error: `reasoning_effort` is set but no corresponding runtime parser or crate support exists in the diff; if the consumer code does not accept this key it will be silently ignored or cause a config load failure. Operation otherwise appears safe.", + "findings": [ + { + "path": "src/config/defaults.toml", + "existing_code": "reasoning_effort = \"high\"", + "rule": "undefined-config-key", + "title": "Ensure reasoning_effort is accepted by the consumer", + "body": "The TOML block adds a `reasoning_effort` key, but the diff shows no change to the config struct or parser logic that would read it. If the runtime config does not expose this field, the value will be silently dropped (TOML deserializers often ignore unknown keys) or cause a parse error. Confirm the consumer code actually uses this key before merging; otherwise the setting has no effect and the comment is misleading.", + "severity": "medium", + "confidence": 0.75, + "suggestion": null + } + ] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0005-5e5e6e07b224bd0e.json b/evals/cassettes/ts-0068-description-anchored-to-code/0005-5e5e6e07b224bd0e.json deleted file mode 100644 index 07ddde3..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0005-5e5e6e07b224bd0e.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "key": "5e5e6e07b224bd0e", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 4739, - "output_tokens": 10597, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.010839127 - }, - "value": { - "findings": [ - { - "body": "This test constructs `GatewayModel` manually and then calls `reasoning_options` directly, so it would still pass if `GatewayModel::from_config` failed to copy `models.reasoning_effort`, or if `harness` ignored `self.reasoning_effort` and kept the old hard-coded disable. The comment says the test \"asserts the error\", but no error assertion is present either. To actually cover the wiring, call the real `from_config`/`harness` path and assert the resulting provider options, or factor the conversion so it can be tested directly.", - "confidence": 0.9, - "existing_code": " // SAFETY-adjacent: no env mutation. The key is read from a variable the\n // config names, so the test names one it sets nowhere and asserts the\n // error, then builds the gateway directly for the positive case.\n let gateway = GatewayModel {\n api_key: \"unused\".into(),", - "late": false, - "path": "src/harness/openrouter.rs", - "rule": "ineffective-test", - "severity": "low", - "suggestion": null, - "title": "Test does not exercise the config-to-gateway wiring it claims" - } - ], - "resolved": [], - "summary": "The change replaces the hard-coded reasoning disable with a configurable `reasoning_effort` forwarded through `GatewayModel`; the production path looks sound. One new test claims to verify the config-to-gateway wiring but only exercises `reasoning_options`, so it gives false confidence and should be fixed before merge." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0005-8c7dc6fedf29f06f.json b/evals/cassettes/ts-0068-description-anchored-to-code/0005-8c7dc6fedf29f06f.json deleted file mode 100644 index bbf4350..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0005-8c7dc6fedf29f06f.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "8c7dc6fedf29f06f", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 1063, - "output_tokens": 83, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.000258535 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0005-8cec32f46b7c9c01.json b/evals/cassettes/ts-0068-description-anchored-to-code/0005-8cec32f46b7c9c01.json deleted file mode 100644 index 079db03..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0005-8cec32f46b7c9c01.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "8cec32f46b7c9c01", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3674, - "output_tokens": 2250, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.0007521800000000001 - }, - "value": { - "findings": [ - { - "body": "The test is named and commented as testing \"the wire between config and gateway\" — the half that would leave the setting inert — but it never calls `harness()` or otherwise observes what `with_default_provider_options` receives. It constructs a `GatewayModel` by hand and then calls `reasoning_options(&gateway.reasoning_effort)`, which is the same function already exercised by the two tests above it. If `harness()` were changed to pass a hardcoded value (or to drop the call entirely), this test would still pass. It does not test what its name and comment claim.", - "confidence": 0, - "existing_code": " // `reasoning_options` is tested in isolation above; this asserts the\n // wire between config and gateway, which is the half that would leave\n // the setting inert — the same failure `max_tokens` had, where the\n // value was read, validated, documented, and then never forwarded.\n let mut models = models();\n models.reasoning_effort = \"off\".into();", - "path": "src/harness/openrouter.rs", - "rule": ", ", - "severity": "low", - "title": "The \"wire\" test does not exercise the wire it claims to" - } - ], - "resolved": [], - "summary": "The production change is sound: `reasoning_effort` flows from config through `GatewayModel` into `reasoning_options`, which correctly maps `\"off\"`/empty to disabled and everything else to an effort passthrough. One test, `the_configured_effort_reaches_the_gateway`, claims to verify the config-to-gateway wiring but only re-calls `reasoning_options` and never invokes `harness()`, so it would not catch a regression in `with_default_provider_options`." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0005-dca3571c4f7994b8.json b/evals/cassettes/ts-0068-description-anchored-to-code/0005-dca3571c4f7994b8.json new file mode 100644 index 0000000..0561eb1 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0005-dca3571c4f7994b8.json @@ -0,0 +1,16 @@ +{ + "key": "dca3571c4f7994b8", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_falsify", + "usage": { + "input_tokens": 1465, + "output_tokens": 32, + "cached_tokens": 256, + "embed_tokens": 0, + "cost_usd": 0.000064700412 + }, + "value": { + "incorrect": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0006-188d9a53900dfbce.json b/evals/cassettes/ts-0068-description-anchored-to-code/0006-188d9a53900dfbce.json deleted file mode 100644 index e029c78..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0006-188d9a53900dfbce.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "188d9a53900dfbce", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 2605, - "output_tokens": 235, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00023405000000000002 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0006-9f306866be576df3.json b/evals/cassettes/ts-0068-description-anchored-to-code/0006-9f306866be576df3.json deleted file mode 100644 index 944f9c7..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0006-9f306866be576df3.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "9f306866be576df3", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 2881, - "output_tokens": 586, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.001486975 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0006-c0050c76a554fe21.json b/evals/cassettes/ts-0068-description-anchored-to-code/0006-c0050c76a554fe21.json new file mode 100644 index 0000000..3200354 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0006-c0050c76a554fe21.json @@ -0,0 +1,17 @@ +{ + "key": "c0050c76a554fe21", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "z-ai/glm-5.2", + "schema_name": "tinysweeper_critique", + "usage": { + "input_tokens": 2704, + "output_tokens": 1079, + "cached_tokens": 2332, + "embed_tokens": 0, + "cost_usd": 0.0014099328 + }, + "value": { + "summary": "The diff adds a `reasoning_effort: String` field to the config struct with doc comments on both `max_tokens` and the new field. From what is visible in this file alone the change looks sound; potential concerns about default values or serde deserialization of the new field depend on code not shown here and cannot be confirmed.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0006-f4392b8951c5a340.json b/evals/cassettes/ts-0068-description-anchored-to-code/0006-f4392b8951c5a340.json deleted file mode 100644 index 99027cb..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0006-f4392b8951c5a340.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "f4392b8951c5a340", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3351, - "output_tokens": 4767, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.005439327 - }, - "value": { - "findings": [], - "summary": "The change swaps the default scan/deep/fallback model selections and adds a `reasoning_effort` setting. The TOML is syntactically valid and consistent with the surrounding config; no issues found in this file." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0007-0a29688e1cc61e19.json b/evals/cassettes/ts-0068-description-anchored-to-code/0007-0a29688e1cc61e19.json deleted file mode 100644 index c2a43ac..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0007-0a29688e1cc61e19.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "0a29688e1cc61e19", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 3415, - "output_tokens": 4693, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.00127151 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change adds four model price entries to `MODEL_PRICES` and four tests to `src/harness/pricing.rs`. The prices are data sourced (per the doc comment) from OpenRouter, and the tests are internally consistent with those values: `the_new_deep_tier_costs_what_the_table_says` computes `82914*0.07 + 842*0.22` per million tokens ≈ $0.00599, which lands inside the asserted `[0.005, 0.007]` band; the argument-ordering tests distinguish cached-token vs. output pricing correctly given GLM 5.2's output rate ($0.22) is far above its cache rate ($0.013); and the defaults-coverage test only requires the *selected* models to be priced, so the deliberately-unselected entries don't cause false failures. I could not see the body of `completion_cost` or `unpriced`, so I cannot certify the exact pricing formula, but the assertions the author wrote hold under the natural reading of the function and are mutually consistent. No bugs introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0007-353d2a1fbdf240b0.json b/evals/cassettes/ts-0068-description-anchored-to-code/0007-353d2a1fbdf240b0.json deleted file mode 100644 index 3c8d676..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0007-353d2a1fbdf240b0.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "353d2a1fbdf240b0", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 8868, - "output_tokens": 36545, - "cached_tokens": 5376, - "embed_tokens": 0, - "cost_usd": 0.017925596 - }, - "value": { - "findings": [ - { - "body": "The comment says cache reads are a hundredth of the input price, which for `input: 0.435` would be `0.00435`. The actual value is `0.003625`, which is `0.435 / 120` and about 17% lower than a hundredth. This either misdocuments the verified ratio or underreports cached-token cost. Update the value to `0.00435` or correct the comment and re-verify against OpenRouter.", - "confidence": 0.8, - "existing_code": " // Cache reads are a *hundredth* of the input price here, against a\n // tenth on kimi-k3. On a re-review that is most of the bill.\n cached: 0.003625,", - "path": "src/harness/pricing.rs", - "rule": "incorrect-comment", - "severity": "low", - "suggestion": null, - "title": "Correct the cached-price comment or value for deepseek-v4-pro" - } - ], - "summary": "The change adds new model price rows and tests that the default-selected models are priced, including checks for cached/output billing order. The code looks mostly sound, but one cached-price comment/value mismatch should be corrected before merging." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0007-5d5c31938dc865de.json b/evals/cassettes/ts-0068-description-anchored-to-code/0007-5d5c31938dc865de.json deleted file mode 100644 index c08baac..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0007-5d5c31938dc865de.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "5d5c31938dc865de", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_critique", - "usage": { - "input_tokens": 4324, - "output_tokens": 6751, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.007588662 - }, - "value": { - "findings": [ - { - "body": "The test `a_cached_read_is_charged_at_the_cache_rate` passes the same `input` (100,000) to both calls, but adds `cached` tokens (90,000) in the warm call. If `completion_cost` treats `input` and `cached` as separate additive quantities (as suggested by the emphasis on argument order), then the warm call charges for 190,000 prompt tokens instead of 100,000, making `warm > cold` and causing the assertion to fail. To properly compare cold vs warm, the warm call should reduce the uncached input by the number of cached tokens, e.g., `completion_cost(\"z-ai/glm-5.2\", 10_000, 90_000, 1_000)`.", - "confidence": 0.9, - "existing_code": " let cold = completion_cost(\"z-ai/glm-5.2\", 100_000, 0, 1_000);\n let warm = completion_cost(\"z-ai/glm-5.2\", 100_000, 90_000, 1_000);\n\n assert!(\n warm < cold,\n \"cached tokens must cost less: {warm} vs {cold}\"\n );", - "path": "src/harness/pricing.rs", - "rule": "logic-error", - "severity": "medium", - "suggestion": " let cold = completion_cost(\"z-ai/glm-5.2\", 100_000, 0, 1_000);\n let warm = completion_cost(\"z-ai/glm-5.2\", 10_000, 90_000, 1_000);\n\n assert!(\n warm < cold,\n \"cached tokens must cost less: {warm} vs {cold}\"\n );", - "title": "Keep total prompt tokens constant in cached read test" - } - ], - "summary": "The change adds pricing entries for several models and tests to catch unpriced defaults and verify cost calculations. One new test for cached reads appears to add cached tokens on top of input rather than replacing input, which would make the assertion fail." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0007-97d607cd8180b2b8.json b/evals/cassettes/ts-0068-description-anchored-to-code/0007-97d607cd8180b2b8.json new file mode 100644 index 0000000..09471dd --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0007-97d607cd8180b2b8.json @@ -0,0 +1,17 @@ +{ + "key": "97d607cd8180b2b8", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 4084, + "output_tokens": 63, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.0002057006 + }, + "value": { + "summary": "The change adds new model price entries and four tests verifying cost calculations and configuration completeness. No security problems are introduced: the new data is static, not derived from untrusted input, and the tests do not execute or communicate outside the harness.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0008-1665583b223eb633.json b/evals/cassettes/ts-0068-description-anchored-to-code/0008-1665583b223eb633.json deleted file mode 100644 index 902d457..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0008-1665583b223eb633.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "1665583b223eb633", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 2536, - "output_tokens": 3170, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.00358498 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0008-40d73aadf4587bd0.json b/evals/cassettes/ts-0068-description-anchored-to-code/0008-40d73aadf4587bd0.json deleted file mode 100644 index 60c40cd..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0008-40d73aadf4587bd0.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "key": "40d73aadf4587bd0", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_falsify", - "usage": { - "input_tokens": 2554, - "output_tokens": 274, - "cached_tokens": 640, - "embed_tokens": 0, - "cost_usd": 0.00107329 - }, - "value": { - "incorrect": [] - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0008-cdd9c81d90654ca1.json b/evals/cassettes/ts-0068-description-anchored-to-code/0008-cdd9c81d90654ca1.json deleted file mode 100644 index 2dda879..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0008-cdd9c81d90654ca1.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "cdd9c81d90654ca1", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2276, - "output_tokens": 246, - "cached_tokens": 1934, - "embed_tokens": 0, - "cost_usd": 0.00010320199999999999 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change swaps default model tiers and fallbacks in `defaults.toml` and adds a `reasoning_effort` setting. All values are static config data; no secrets are committed (the key still comes from `OPENROUTER_API_KEY`), and no code, network, subprocess, or auth surface changes. Nothing to report." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0008-ec37163789e29c2d.json b/evals/cassettes/ts-0068-description-anchored-to-code/0008-ec37163789e29c2d.json new file mode 100644 index 0000000..5c5575b --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0008-ec37163789e29c2d.json @@ -0,0 +1,28 @@ +{ + "key": "ec37163789e29c2d", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 2672, + "output_tokens": 185, + "cached_tokens": 1024, + "embed_tokens": 0, + "cost_usd": 0.000108606008 + }, + "value": { + "summary": "Adds a `reasoning_effort` field as a plain `String` without validation. This risks upstream breakage or silent misconfiguration. No security issue introduced.", + "findings": [ + { + "path": "src/config/types.rs", + "existing_code": " pub reasoning_effort: String,", + "rule": "unvalidated-enum", + "title": "Constrain reasoning_effort to accepted values", + "body": "The doc comment defines only `off`, `low`, `medium`, `high` as valid, but the type is `String`. Any value compiles and is silently forwarded to the API. Use an enum or validate at the boundary.", + "severity": "low", + "confidence": 0.95, + "suggestion": null + } + ] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0009-23736351a1125283.json b/evals/cassettes/ts-0068-description-anchored-to-code/0009-23736351a1125283.json deleted file mode 100644 index 105fad9..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0009-23736351a1125283.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "23736351a1125283", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 4303, - "output_tokens": 393, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.002048067 - }, - "value": { - "findings": [], - "summary": "The diff adds pricing entries for several new models and tests that validate cost calculations and that all default models are priced. No security issues are introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0009-5fd8ceb66d310e3f.json b/evals/cassettes/ts-0068-description-anchored-to-code/0009-5fd8ceb66d310e3f.json deleted file mode 100644 index cda4ec7..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0009-5fd8ceb66d310e3f.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "5fd8ceb66d310e3f", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 1852, - "output_tokens": 538, - "cached_tokens": 1592, - "embed_tokens": 0, - "cost_usd": 0.00015725600000000001 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change adds a single documented `reasoning_effort: String` field to an existing config struct in `src/config/types.rs`. It introduces no new network calls, subprocess invocations, deserialization, path handling, auth changes, secrets, dependency changes, or CI permission changes. The field is a plain config value typed and documented in the appropriate types module per the repository's own rules. No security findings." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0009-64f5967d5015765f.json b/evals/cassettes/ts-0068-description-anchored-to-code/0009-64f5967d5015765f.json deleted file mode 100644 index b7a508a..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0009-64f5967d5015765f.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "64f5967d5015765f", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3440, - "output_tokens": 814, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.001762852 - }, - "value": { - "findings": [], - "summary": "The change updates the default language-model selections, fallback chain, and reasoning effort in `src/config/defaults.toml`. It introduces no new secrets, network endpoints, authentication changes, or untrusted-input handling, and is safe to merge." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0009-9f3a263d8d7c3f65.json b/evals/cassettes/ts-0068-description-anchored-to-code/0009-9f3a263d8d7c3f65.json new file mode 100644 index 0000000..08fcc36 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0009-9f3a263d8d7c3f65.json @@ -0,0 +1,17 @@ +{ + "key": "9f3a263d8d7c3f65", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 4389, + "output_tokens": 125, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00022666154 + }, + "value": { + "summary": "This change re-enables a configurable `reasoning` block in OpenRouter requests, replacing a hard-coded disable with a runtime option. The new field is read from configuration, forwarded through the gateway builder, and tested. No security problems were introduced: the input reaches a controlled JSON sink but is not interpreted as code or used in a dangerous way; the field is an enum-like string and the JSON is constructed by `serde_json::json!`, which is safe. The tests cover the edge cases explicitly. The change looks sound.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0010-8f032c383f3cd7ce.json b/evals/cassettes/ts-0068-description-anchored-to-code/0010-8f032c383f3cd7ce.json deleted file mode 100644 index 4d6f02b..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0010-8f032c383f3cd7ce.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "8f032c383f3cd7ce", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3001, - "output_tokens": 271, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.001099477 - }, - "value": { - "findings": [], - "summary": "The change adds documentation and a new `reasoning_effort` field to the model configuration type; it introduces no security-sensitive behavior in this file." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0010-95abf2e743a888f9.json b/evals/cassettes/ts-0068-description-anchored-to-code/0010-95abf2e743a888f9.json deleted file mode 100644 index 11a3879..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0010-95abf2e743a888f9.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "95abf2e743a888f9", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3330, - "output_tokens": 498, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.001716162 - }, - "value": { - "findings": [], - "summary": "This change replaces the model tier and fallback configuration with cheaper, validated models and enables reasoning. No security issues are introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0010-c861c8ec196f73ea.json b/evals/cassettes/ts-0068-description-anchored-to-code/0010-c861c8ec196f73ea.json deleted file mode 100644 index 78fb61b..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0010-c861c8ec196f73ea.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "c861c8ec196f73ea", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3653, - "output_tokens": 305, - "cached_tokens": 3141, - "embed_tokens": 0, - "cost_usd": 0.000143773 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change replaces a hardcoded reasoning-off setting with a config-derived `reasoning_effort` field and passes it through `reasoning_options`, which builds structured JSON via `serde_json::json!`. The value originates from local config, not untrusted input, and is inserted as a JSON string value, so there is no injection surface. The API key continues to be excluded from the `Debug` impl, and no new network calls, subprocesses, deserialization paths, secrets in logs, dependency changes, or permission widenings are introduced. Looks sound." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0010-d8a6ea9d1612bf43.json b/evals/cassettes/ts-0068-description-anchored-to-code/0010-d8a6ea9d1612bf43.json new file mode 100644 index 0000000..81641a5 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0010-d8a6ea9d1612bf43.json @@ -0,0 +1,17 @@ +{ + "key": "d8a6ea9d1612bf43", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "z-ai/glm-5.2", + "schema_name": "tinysweeper_security", + "usage": { + "input_tokens": 3107, + "output_tokens": 312, + "cached_tokens": 2656, + "embed_tokens": 0, + "cost_usd": 0.0006467424 + }, + "value": { + "summary": "This change updates the default model configuration in defaults.toml, switching both scan and deep tiers to z-ai/glm-5.2, updating fallback models, and enabling reasoning_effort. It introduces no security issues: no secrets, no new network calls, no auth or permission changes.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0011-093689929f9f5cdf.json b/evals/cassettes/ts-0068-description-anchored-to-code/0011-093689929f9f5cdf.json deleted file mode 100644 index d63e6f8..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0011-093689929f9f5cdf.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "093689929f9f5cdf", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 3394, - "output_tokens": 1210, - "cached_tokens": 3020, - "embed_tokens": 0, - "cost_usd": 0.00033163999999999996 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The change adds model price entries to a static table and accompanying tests in `src/harness/pricing.rs`. No untrusted input reaches a dangerous sink, no network or subprocess calls are introduced, no deserialization of untrusted data occurs, and no secrets or widened permissions are involved. The new tests parse the crate's own `DEFAULTS` constant and assert pricing invariants. The change looks sound." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0011-41900b70dd0627fd.json b/evals/cassettes/ts-0068-description-anchored-to-code/0011-41900b70dd0627fd.json new file mode 100644 index 0000000..13d9763 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0011-41900b70dd0627fd.json @@ -0,0 +1,17 @@ +{ + "key": "41900b70dd0627fd", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_tests", + "usage": { + "input_tokens": 6921, + "output_tokens": 121, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00034998418 + }, + "value": { + "summary": "The change updates the default model configuration to use GLM 5.2 for both scan and deep tiers, adds a reasoning_effort config field and wires it through the OpenRouter gateway, and adds test coverage for the new reasoning_options function and model pricing. All behavioural changes are tested: the new reasoning_options function has unit tests, the config-to-gateway wire is tested, the default model set is checked for price entries, and the new pricing logic is pinned against real measured figures. No test gaps were found.", + "findings": [] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0011-99e578a2574f6d7a.json b/evals/cassettes/ts-0068-description-anchored-to-code/0011-99e578a2574f6d7a.json deleted file mode 100644 index 076523d..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0011-99e578a2574f6d7a.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "99e578a2574f6d7a", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 4718, - "output_tokens": 1800, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.003176602 - }, - "value": { - "findings": [], - "summary": "The change adds a configurable reasoning_effort field to GatewayModel and uses it to construct provider options via a safe JSON builder. No security issues are introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0011-de8143183d9a6731.json b/evals/cassettes/ts-0068-description-anchored-to-code/0011-de8143183d9a6731.json deleted file mode 100644 index 32c33d7..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0011-de8143183d9a6731.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "de8143183d9a6731", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 2891, - "output_tokens": 746, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.001740957 - }, - "value": { - "findings": [], - "summary": "The diff adds documentation comments to `max_tokens` and introduces a new public field `reasoning_effort` with documentation. No code logic is changed, and the new field does not introduce a security vulnerability." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0012-124e3ad01415e1c2.json b/evals/cassettes/ts-0068-description-anchored-to-code/0012-124e3ad01415e1c2.json new file mode 100644 index 0000000..cd02ba2 --- /dev/null +++ b/evals/cassettes/ts-0068-description-anchored-to-code/0012-124e3ad01415e1c2.json @@ -0,0 +1,48 @@ +{ + "key": "124e3ad01415e1c2", + "model_requested": "deepseek/deepseek-v4-flash", + "model_answered": "deepseek/deepseek-v4-flash", + "schema_name": "tinysweeper_description", + "usage": { + "input_tokens": 8048, + "output_tokens": 760, + "cached_tokens": 0, + "embed_tokens": 0, + "cost_usd": 0.00046749248 + }, + "value": { + "summary": "This change moves the scan and deep tiers to GLM 5.2, makes reasoning effort configurable, updates the fallback chain, and extends pricing tables with the new models, all backed by measured cost data and test coverage; it is safe to merge.", + "findings": [ + { + "path": "src/config/types.rs", + "existing_code": " pub reasoning_effort: String,", + "rule": "untyped-string-config", + "title": "Represent reasoning_effort as an enum instead of String", + "body": "`reasoning_effort` is stored as a raw `String` but is only ever compared against a fixed set of values (`\"off\"`, `\"\"`, `\"low\"`, `\"medium\"`, `\"high\"`, `\"max\"`). A typo (`\"hgh\"` -> passes validation, becomes `effort: \"hgh\"` in the request) would silently fall through to the `effort =>` branch of `reasoning_options` and produce an unsupported effort string. Parse it into an enum with a `Display` / `FromStr` implementation, or validate it in the config deserializer and reject unknown values at startup.", + "severity": "medium", + "confidence": 0.8, + "suggestion": null + }, + { + "path": "src/harness/openrouter.rs", + "existing_code": "fn reasoning_options(effort: &str) -> serde_json::Value {\n match effort.trim() {\n \"off\" | \"\" => json!({ \"reasoning\": { \"enabled\": false } }),\n effort => json!({ \"reasoning\": { \"effort\": effort } }),\n }\n}", + "rule": "missing-input-validation", + "title": "Reject unknown effort values instead of passing them through silently", + "body": "The catch-all arm `effort =>` passes any unknown string (e.g. `\"hgh\"`, `\"max\"`, `\"extreme\"`) directly as the `effort` field to OpenRouter. The PR body itself says `\"max\"` is undocumented and produced malformed JSON. This should either validate the effort against a known set, or log a warning before passing it through. At minimum, document the valid values in the function signature.", + "severity": "medium", + "confidence": 0.9, + "suggestion": null + }, + { + "path": "src/config/defaults.toml", + "existing_code": "reasoning_effort = \"high\"", + "rule": "incomplete-validation", + "title": "Document that max reasoning effort is unsupported and may break outputs", + "body": "The PR body warns that `reasoning_effort = \"max\"` is undocumented, passed through rather than validated, and produced malformed JSON in testing. The config file sets `reasoning_effort = \"high\"` and the comment block explains the hazard of thinking-heavy models, but it does not explicitly tell an operator that `\"max\"` is not a supported value and will cause failures. Add a line to the comment above or to the type's doc comment warning against `\"max\"`.", + "severity": "low", + "confidence": 0.9, + "suggestion": null + } + ] + } +} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0012-32974d59dd73473c.json b/evals/cassettes/ts-0068-description-anchored-to-code/0012-32974d59dd73473c.json deleted file mode 100644 index 068694e..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0012-32974d59dd73473c.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "key": "32974d59dd73473c", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_tests", - "usage": { - "input_tokens": 6380, - "output_tokens": 2244, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.0009402799999999999 - }, - "value": { - "findings": [ - { - "body": "The test `the_configured_effort_reaches_the_gateway` claims to assert \"the wire between config and gateway\" but never calls `from_config`. It constructs a `GatewayModel` by hand, copying `models.reasoning_effort` into the `reasoning_effort` field, then reads that same field back through `reasoning_options`. The assertion can only fail if `String::clone` is broken. The actual wiring it claims to guard — forwarding the config field in `from_config`:\n\n```\n reasoning_effort: models.reasoning_effort.clone(),\n```\n\nis not exercised. A regression that drops this line (the exact failure the test's own comment cites as the motivation: `max_tokens` was \"read, validated, documented, and then never forwarded\") would pass this test.", - "confidence": 0.92, - "existing_code": " let gateway = GatewayModel {\n api_key: \"unused\".into(),\n base_url: models.base_url.clone(),\n fallbacks: models.fallback.clone(),\n reasoning_effort: models.reasoning_effort.clone(),\n };\n\n assert_eq!(\n reasoning_options(&gateway.reasoning_effort),\n json!({ \"reasoning\": { \"enabled\": false } })\n );", - "path": "src/harness/openrouter.rs", - "rule": "missing test for new branching path", - "severity": "medium", - "title": "Gateway-wiring test is a tautology; it bypasses from_config" - } - ], - "resolved": [], - "summary": "The pricing tests are solid: they pin real cost figures, guard the argument order of `completion_cost`, and cross-check defaults against the price table. The `reasoning_options` unit tests cover both match arms. However, `the_configured_effort_reaches_the_gateway` does not test the wire it claims to: it builds a `GatewayModel` by hand and reads back a field it just set, so it cannot catch a regression where `from_config` forgets to forward `reasoning_effort` — precisely the failure the test's own commentary says it exists to prevent." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0012-7042f2628e522625.json b/evals/cassettes/ts-0068-description-anchored-to-code/0012-7042f2628e522625.json deleted file mode 100644 index e1972c9..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0012-7042f2628e522625.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "7042f2628e522625", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 4413, - "output_tokens": 3596, - "cached_tokens": 1024, - "embed_tokens": 0, - "cost_usd": 0.004606447 - }, - "value": { - "findings": [], - "summary": "The change adds model pricing entries and in-crate tests for cost estimation; no security problems introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0012-7bde9d206ba49dcc.json b/evals/cassettes/ts-0068-description-anchored-to-code/0012-7bde9d206ba49dcc.json deleted file mode 100644 index fe456fd..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0012-7bde9d206ba49dcc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "key": "7bde9d206ba49dcc", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_security", - "usage": { - "input_tokens": 4608, - "output_tokens": 734, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.002477412 - }, - "value": { - "findings": [], - "summary": "The change introduces a configurable reasoning_effort field and forwards it to the gateway provider options. The logic is straightforward and the tests cover the key paths; no security issues were found in the modified file." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0013-2d1bd1de53c9ba41.json b/evals/cassettes/ts-0068-description-anchored-to-code/0013-2d1bd1de53c9ba41.json deleted file mode 100644 index 6a185fb..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0013-2d1bd1de53c9ba41.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "key": "2d1bd1de53c9ba41", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_tests", - "usage": { - "input_tokens": 7250, - "output_tokens": 10819, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.01256628 - }, - "value": { - "findings": [ - { - "body": "This test is titled as though it verifies that configured reasoning effort reaches the gateway, but it manually constructs a `GatewayModel` and then calls `reasoning_options` on the field it just copied. It never invokes the config-to-gateway constructor or the `harness` method, so deleting the `reasoning_effort: models.reasoning_effort.clone()` forwarding line from `GatewayModel::new` would leave this test green. The comment also says the test \"asserts the error\", but no error assertion exists. Replace it with a test that builds the gateway from `models` through the real constructor and asserts the provider options the harness produces.", - "confidence": 0.95, - "existing_code": " let gateway = GatewayModel {\n api_key: \"unused\".into(),\n base_url: models.base_url.clone(),\n fallbacks: models.fallback.clone(),\n reasoning_effort: models.reasoning_effort.clone(),\n };", - "path": "src/harness/openrouter.rs", - "rule": "ineffective-test", - "severity": "medium", - "title": "Call the config-to-gateway constructor instead of rebuilding the value" - } - ], - "summary": "The change swaps the default OpenRouter model, makes reasoning effort configurable, and adds pricing rows plus several sound unit tests. One test that claims to cover config-to-gateway forwarding constructs the gateway by hand and never exercises the constructor, so a regression in that wiring would still pass; that test should be replaced before merge." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0013-6a2e2ad299636656.json b/evals/cassettes/ts-0068-description-anchored-to-code/0013-6a2e2ad299636656.json deleted file mode 100644 index a2ca95d..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0013-6a2e2ad299636656.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "key": "6a2e2ad299636656", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_tests", - "usage": { - "input_tokens": 7140, - "output_tokens": 6833, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.008884962 - }, - "value": { - "findings": [ - { - "body": "The test titled `the_configured_effort_reaches_the_gateway` never constructs a harness or inspects the provider options it sends. It copies the field into a new `GatewayModel` and calls `reasoning_options` directly, which is exactly what the isolated tests above already cover. If `harness` stopped applying `reasoning_effort` — the failure this test exists to catch — the assertion would still pass. Call `gateway.harness(...)` and assert the resulting provider options contain the configured reasoning block, or move the options assembly into a seam that `harness` is known to use.", - "confidence": 0.9, - "existing_code": " let gateway = GatewayModel {\n api_key: \"unused\".into(),\n base_url: models.base_url.clone(),\n fallbacks: models.fallback.clone(),\n reasoning_effort: models.reasoning_effort.clone(),\n };\n\n assert_eq!(\n reasoning_options(&gateway.reasoning_effort),\n json!({ \"reasoning\": { \"enabled\": false } })\n );", - "path": "src/harness/openrouter.rs", - "rule": "test-does-not-exercise-behaviour", - "severity": "medium", - "suggestion": null, - "title": "Make the wired-effort test exercise harness construction" - }, - { - "body": "`the_new_deep_tier_costs_what_the_table_says` computes the cost for the literal `\"z-ai/glm-5.2\"` rather than the model actually named by `config.models.deep`. The test's premise is that the selected deep tier is cheap, but if `defaults.toml` later points `deep` at an already-priced expensive model (e.g. the `qwen` row), the `every_model_the_defaults_select_has_a_price` test still passes and this test keeps checking the old model, so the cost regression would be silent. Parse the defaults in this test and use `config.models.deep` as the model argument.", - "confidence": 0.7, - "existing_code": " let cost = completion_cost(\"z-ai/glm-5.2\", 82_914, 0, 842);\n\n assert!(\n (0.005..0.007).contains(&cost),\n \"expected roughly $0.006 for the 83k-token security call, got {cost:.5}\"\n );", - "path": "src/harness/pricing.rs", - "rule": "test-does-not-exercise-behaviour", - "severity": "medium", - "suggestion": null, - "title": "Pin the deep-tier cost test to the selected model" - } - ], - "summary": "The change adds configurable reasoning effort and new model pricing with tests, but two tests do not protect the behaviour they claim to, so the PR is not fully covered." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0013-e9b356c74601f13b.json b/evals/cassettes/ts-0068-description-anchored-to-code/0013-e9b356c74601f13b.json deleted file mode 100644 index fe75e47..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0013-e9b356c74601f13b.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "key": "e9b356c74601f13b", - "model_requested": "z-ai/glm-5.2", - "model_answered": "z-ai/glm-5.2", - "schema_name": "tinysweeper_description", - "usage": { - "input_tokens": 7501, - "output_tokens": 1764, - "cached_tokens": 6675, - "embed_tokens": 0, - "cost_usd": 0.0005326749999999999 - }, - "value": { - "findings": [], - "resolved": [], - "summary": "The description is accurate and thorough. Every claim in the PR body maps to the diff: both `scan` and `deep` move to `z-ai/glm-5.2`; the fallback chain changes from `kimi-k2.7-code`/`minimax-m2.1` to `deepseek-v4-pro`/`minimax-m3`; `reasoning_effort` is a new configurable `String` field on `Models`, wired through `GatewayModel` into `reasoning_options`, with `\"off\"`/`\"\"` disabling reasoning and any other value passed through as an effort level; prices for every named model (plus `deepseek-v4-flash` and `qwen3.8-max`) are added to `harness::pricing`; and the test suite gains coverage for the reasoning wire, argument-order correctness in cost calculation, and a guard that every model named in `defaults.toml` has a price. The body's acknowledgment that `\"max\"` passes through unvalidated is consistent with the code, which does no validation. The one unmentioned addition — a price row for `deepseek/deepseek-v4-flash` — is covered by the general statement that entries exist for unselected models so one-line config changes do not silently break the budget ceiling. No problems introduced." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0014-7519cc81b4847f86.json b/evals/cassettes/ts-0068-description-anchored-to-code/0014-7519cc81b4847f86.json deleted file mode 100644 index b369b27..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0014-7519cc81b4847f86.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "key": "7519cc81b4847f86", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_description", - "usage": { - "input_tokens": 8377, - "output_tokens": 14227, - "cached_tokens": 0, - "embed_tokens": 0, - "cost_usd": 0.016021485 - }, - "value": { - "findings": [ - { - "body": "The test comment says it asserts the wire between config and gateway, but it manually builds `GatewayModel` and only calls `reasoning_options` again. A regression in the constructor that stopped copying `reasoning_effort` would still pass this test. Call the actual constructor and assert on the resulting gateway, or better, on the provider options sent.", - "confidence": 0.9, - "existing_code": " let gateway = GatewayModel {\n api_key: \"unused\".into(),\n base_url: models.base_url.clone(),\n fallbacks: models.fallback.clone(),\n reasoning_effort: models.reasoning_effort.clone(),", - "late": false, - "path": "src/harness/openrouter.rs", - "rule": "ineffective-test", - "severity": "low", - "suggestion": null, - "title": "Use the real constructor in the reasoning-effort wiring test" - } - ], - "resolved": [], - "summary": "The model configuration, reasoning-effort plumbing, and pricing table updates match the PR description and look sound. The only issue is a low-severity test that claims to verify config-to-gateway wiring but does not." - } -} \ No newline at end of file diff --git a/evals/cassettes/ts-0068-description-anchored-to-code/0014-e9702f894c13c6ba.json b/evals/cassettes/ts-0068-description-anchored-to-code/0014-e9702f894c13c6ba.json deleted file mode 100644 index 2459309..0000000 --- a/evals/cassettes/ts-0068-description-anchored-to-code/0014-e9702f894c13c6ba.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "key": "e9702f894c13c6ba", - "model_requested": "deepseek/deepseek-v4-pro-0813", - "model_answered": "deepseek/deepseek-v4-pro-0813", - "schema_name": "tinysweeper_description", - "usage": { - "input_tokens": 8267, - "output_tokens": 7315, - "cached_tokens": 384, - "embed_tokens": 0, - "cost_usd": 0.009794547 - }, - "value": { - "findings": [ - { - "body": "The `reasoning_effort` string is passed through to OpenRouter without validation. If a user sets an unsupported value like `\"max\"` or a typo, the API call fails or behaves unexpectedly. The PR even notes `\"max\"` was passed through rather than validated. Restrict to `off`, `low`, `medium`, `high` (possibly `minimal`) or use an enum.", - "confidence": 0.9, - "existing_code": " effort => json!({ \"reasoning\": { \"effort\": effort } }),", - "path": "src/harness/openrouter.rs", - "rule": "unvalidated-config-value", - "severity": "low", - "suggestion": null, - "title": "Validate reasoning_effort against allowed values" - }, - { - "body": "Adding `reasoning_effort` as a required field means existing custom `config.toml` files that don't include it will fail to deserialize. Since the previous config schema had no such field, this is a breaking change for deployments. Add `#[serde(default = \"...\")]` or ensure config merging fills it from defaults.", - "confidence": 0.6, - "existing_code": " pub reasoning_effort: String,", - "path": "src/config/types.rs", - "rule": "breaking-config-change", - "severity": "medium", - "suggestion": null, - "title": "Add serde default for reasoning_effort to avoid breaking configs" - } - ], - "summary": "The change swaps model defaults to GLM 5.2, makes reasoning effort configurable, updates pricing, and adds tests. Overall sound, but two issues: missing validation for reasoning_effort could send invalid values to the API, and adding a required config field may break existing custom configs." - } -} \ No newline at end of file diff --git a/src/app/apply.rs b/src/app/apply.rs index 2b33fe1..042d1ad 100644 --- a/src/app/apply.rs +++ b/src/app/apply.rs @@ -314,7 +314,16 @@ fn title_for(findings: usize, summary: &str) -> String { } fn render_lane_summary(lane: &crate::app::review::LaneProposal) -> String { - let mut out = crate::findings::render::lane_summary(&lane.summary, &lane.findings, VERSION); + // `Neutral` is exactly "this lane formed no opinion" — not implemented, + // skipped as a draft, or every model call in the chain failed. Anything + // else reached a verdict, including a clean one. + let reached_a_verdict = lane.conclusion != crate::forge::types::CheckConclusion::Neutral; + let mut out = crate::findings::render::lane_summary( + &lane.summary, + &lane.findings, + VERSION, + reached_a_verdict, + ); // Resolved findings are reported, not discarded. An author who fixed // something needs to see that it was noticed; otherwise the only signal a diff --git a/src/config/defaults.toml b/src/config/defaults.toml index 11145e2..2d3c6ed 100644 --- a/src/config/defaults.toml +++ b/src/config/defaults.toml @@ -127,9 +127,10 @@ deep = "deepseek/deepseek-v4-flash" # is one mechanical call, `flash` is one of N concurrent opinions whose # *aggregate* has to stay under the price of the single `deep` call it stands # in for. Blended against the ~80% prompt-cache hit this prompt layout achieves, -# and on the same pinned DeepSeek endpoint as the tiers above, flash costs -# $0.030/M where the pro snapshot costs $0.090/M — so three reviewers cost -# roughly what one did. +# and on the same pinned endpoint as the tiers above, flash costs roughly a +# third of what the pro snapshot did — so three reviewers cost about what one +# did. Note "the same endpoint" is StreamLake/DeepInfra, not DeepSeek: see +# `[models.provider]` below for why that distinction stopped every review once. # # Same `json_object` caveat as everything else in this block: the flash endpoint # refuses a strict schema request exactly as the pro snapshot does. @@ -286,27 +287,60 @@ budget_usd_per_pr = 1.0 # already single-provider by virtue of the dated snapshot; `flash` is not, and # is the tier a council multiplies. # -# `deepseek` is both the cheapest endpoint and — see `structured_output` below — -# the reason this deployment is on `json_object` at all. Pinning it is only -# viable *because* of that setting: measured against the live API, the DeepSeek -# endpoint refuses a strict-schema request, so a deployment that pins this -# provider while asking for `schema` gets `404 No endpoints found` on every -# call, for every model. That combination was shipped once and every lane went -# neutral against a real pull request before anything caught it, which is why -# the two keys are documented together. -# -# `allow_fallbacks = false` makes the pin a guarantee rather than a preference; -# `models.fallback` is the safety net, and it switches model rather than -# provider. Note this is a *routing* pin and does not fix an account whose -# allowed-providers list excludes DeepSeek — that failure is the one the `scan` -# comment above describes, and it is fixed at openrouter.ai/settings/privacy. +# **Pin only providers that actually serve every model in the chain.** This is +# the trap, and it has now been sprung twice. `order` names a *routing* +# preference; with `allow_fallbacks = false` it is a hard constraint, so a model +# with no endpoint on any listed provider is `404 No endpoints found` — not a +# reroute, not a downgrade, a dead call. The pin applies to the whole chain, so +# when it is wrong `models.fallback` cannot save it: the safety net switches +# model, never provider, and every fallback 404s for exactly the same reason. +# +# `order = ["deepseek"]` was that mistake. DeepSeek first-party serves the dated +# `deepseek-v4-pro-0813` snapshot and **does not serve `deepseek-v4-flash`** — +# the vendor prefix names who trained the model, not who hosts it. Once both +# tiers moved to Flash, every primary call and both fallbacks 404'd, every lane +# went `Neutral`, and each pull request was told there was nothing to review. +# A silent all-clear over unreviewed code is the worst failure this repo has. +# +# StreamLake and DeepInfra both serve all four ids this deployment can ask for — +# the two tiers, and both fallbacks — and both accept `json_object`. Verified +# against the live API on 2026-08-23; re-verify before changing this line, with +# +# curl -s https://openrouter.ai/api/v1/models//endpoints \ +# | jq -r '.data.endpoints[].tag' +# +# and confirm the id you are pinning appears there. StreamLake leads because it +# is the cheaper of the two; DeepInfra follows so a single provider's outage +# degrades the price rather than the review. `harness::pricing` records +# DeepInfra's dearer rate for the same reason — see the note on the flash entry. # # This table must stay *below* every scalar key in `[models]`: a TOML sub-table # ends the parent table, so anything written after it is parsed as a key of # `models.provider` instead. [models.provider] -order = ["deepseek"] +order = ["streamlake", "deepinfra"] allow_fallbacks = false +# The rung that stops this pin being a single point of failure. +# +# `models.fallback` switches *model* and every rung of it inherits the `order` +# above, so a pin that cannot serve these models fails the whole ladder for one +# reason — which is exactly how a week of reviews came back empty. With this on, +# one final attempt is made with no pin at all once every priced route has +# failed. +# +# It costs nothing on a healthy deployment because it is only reached after +# everything else has failed, and the trade when it *is* reached is deliberate: +# an unpinned call is billed at a price `harness::pricing` cannot predict, so +# the cost line for that review is an estimate. `budget_usd_per_pr` still bounds +# it, the summary still names the model that answered, and reaching this rung +# logs at `warn` — a wrong cost line is recoverable, a silent all-clear over +# unreviewed code is not. +# +# Set it `false` only where an unpinned call is genuinely unacceptable — a +# data-residency rule that some provider in the pool would break. That is a +# deployment choosing a failed review over an unpinned one, which is coherent, +# but it should be a decision rather than a default. +last_resort_unpinned = true # Several reviewers on one lane's evidence, folded into one review. # diff --git a/src/config/test.rs b/src/config/test.rs index 1a23822..66eeba6 100644 --- a/src/config/test.rs +++ b/src/config/test.rs @@ -153,11 +153,50 @@ fn the_shipped_defaults_pin_the_upstream_provider() { // no provider routing must not have a stray `provider` block sent). let config: Config = DEFAULTS.parse::().unwrap().try_into().unwrap(); - assert_eq!(config.models.provider.order, vec!["deepseek".to_string()]); + assert_eq!( + config.models.provider.order, + vec!["streamlake".to_string(), "deepinfra".to_string()] + ); assert!( !config.models.provider.allow_fallbacks, "a pin the gateway may route around is not a pin" ); + assert!( + config.models.provider.last_resort_unpinned, + "a pin every rung of the ladder inherits needs an unpinned rung below it" + ); +} + +#[test] +fn every_shipped_model_can_be_served_by_a_pinned_provider() { + // The outage this exists to make impossible. `order = ["deepseek"]` was + // shipped while every configured model was served by StreamLake and + // DeepInfra and by DeepSeek not at all — the vendor prefix on + // `deepseek/deepseek-v4-flash` names who trained it, not who hosts it. Every + // rung of the ladder 404'd with `No endpoints found`, every lane went + // Neutral, and each pull request was told there was nothing to review for a + // week. + // + // Which providers serve which model is catalogue data this test cannot + // reach offline, so what is checked here is the shape that made the mistake + // survivable: the pin names providers, not vendors, and the ladder has a + // rung that drops it. `PROVIDER_IS_NOT_A_VENDOR` lists the gateway-side + // names that read like a model vendor and are therefore the ones somebody + // reaches for by mistake. + const PROVIDER_IS_NOT_A_VENDOR: &[&str] = + &["deepseek", "minimax", "z-ai", "moonshotai", "qwen"]; + + let config: Config = DEFAULTS.parse::().unwrap().try_into().unwrap(); + + for provider in &config.models.provider.order { + assert!( + !PROVIDER_IS_NOT_A_VENDOR.contains(&provider.as_str()), + "`{provider}` is a model vendor, and pinning it routes only to the \ + subset of models it also hosts. Confirm every id in `[models]` has \ + an endpoint there before pinning it: \ + curl -s https://openrouter.ai/api/v1/models//endpoints" + ); + } } #[test] diff --git a/src/config/types.rs b/src/config/types.rs index 143c10d..5348961 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -422,7 +422,7 @@ pub struct Labels { /// Cache reads are the reason the *cheapest headline* provider is not /// automatically the right pin — see `defaults.toml`, where the choice is /// argued against measured numbers. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct ProviderRouting { /// Providers to try, in order, by their gateway-side name. Empty leaves @@ -437,6 +437,33 @@ pub struct ProviderRouting { /// net for a genuine outage is `models.fallback`, which switches *model* /// and is priced accordingly. pub allow_fallbacks: bool, + /// Whether the last rung of the model ladder drops the pin entirely. + /// + /// On by default, and it is the rung that keeps a pin from being a single + /// point of failure. Every other rung — the primary and each + /// `models.fallback` — carries the same `order`, so a pin naming a provider + /// that does not serve these models 404s all of them identically and the + /// ladder protects nothing. That is not hypothetical: it is what shipped, + /// and every review for a week reported no findings over code no model had + /// read. + /// + /// The trade is deliberate and one-directional. Reaching this rung means + /// every priced route already failed, so the choice is an unpinned review + /// at a price `harness::pricing` cannot predict, or no review at all — and + /// a wrong cost line is recoverable in a way a silent all-clear is not. + /// `budget_usd_per_pr` still bounds the call, the response reports which + /// model actually answered, and reaching this rung logs at `warn`. + pub last_resort_unpinned: bool, +} + +impl Default for ProviderRouting { + fn default() -> Self { + Self { + order: Vec::new(), + allow_fallbacks: false, + last_resort_unpinned: true, + } + } } impl ProviderRouting { @@ -444,6 +471,15 @@ impl ProviderRouting { pub fn is_empty(&self) -> bool { self.order.iter().all(|p| p.trim().is_empty()) } + + /// The same routing with no pin, for the ladder's last-resort rung. + pub fn unpinned() -> Self { + Self { + order: Vec::new(), + allow_fallbacks: true, + last_resort_unpinned: false, + } + } } /// How a lane's structured answer is obtained from the model. diff --git a/src/findings/render.rs b/src/findings/render.rs index 86c02bd..646c450 100644 --- a/src/findings/render.rs +++ b/src/findings/render.rs @@ -71,13 +71,33 @@ pub fn confidence_badge(confidence: f64) -> String { } /// Render one lane's findings as a check-run summary. -pub fn lane_summary(summary: &str, findings: &[Finding], version: &str) -> String { +/// +/// `reached_a_verdict` is false for a lane that did not run — disabled, skipped +/// as a draft, or one whose every model call failed. It exists because the two +/// cases render identically otherwise and mean opposite things: "we looked and +/// found nothing" against "we never looked". A provider misconfiguration once +/// 404'd every call in the chain for a week and each lane published `No +/// findings.` over unreviewed code, which is the most expensive way this +/// reviewer can be wrong. An empty finding list is a result only when a +/// reviewer actually produced one. +pub fn lane_summary( + summary: &str, + findings: &[Finding], + version: &str, + reached_a_verdict: bool, +) -> String { let mut out = String::with_capacity(1024); out.push_str(summary.trim()); out.push_str("\n\n"); if findings.is_empty() { - out.push_str("No findings.\n"); + out.push_str(if reached_a_verdict { + "No findings.\n" + } else { + "**This lane did not review the change, so it has no verdict.** \ + The absence of findings below is not an all-clear — see the reason \ + above.\n" + }); out.push_str(&footer(version)); return out; } @@ -429,11 +449,31 @@ mod tests { #[test] fn an_empty_review_says_so_and_stops() { - let out = lane_summary("Looks sound.", &[], "0.1.0"); + let out = lane_summary("Looks sound.", &[], "0.1.0", true); assert!(out.contains("No findings.")); assert!(!out.contains("
"), "nothing to fold away"); } + #[test] + fn a_lane_that_never_ran_does_not_report_an_all_clear() { + // The bug this exists to stop: a provider pin that 404'd every model in + // the chain made every lane skip, and each one published "No findings." + // over code nobody had read. An empty finding list is a result only + // when a reviewer produced one. + let out = lane_summary( + "No files could be reviewed; see the listed provider failures.", + &[], + "0.1.0", + false, + ); + assert!( + !out.contains("No findings."), + "a lane with no verdict must not read as a clean one: {out}" + ); + assert!(out.contains("no verdict"), "{out}"); + assert!(out.contains("not an all-clear"), "{out}"); + } + #[test] fn findings_are_listed_worst_first() { let out = lane_summary( @@ -443,6 +483,7 @@ mod tests { finding(Severity::Critical, "Remove the key"), ], "0.1.0", + true, ); let critical = out.find("Remove the key").expect("present"); let low = out.find("Rename this").expect("present"); @@ -452,7 +493,12 @@ mod tests { #[test] fn a_pipe_in_a_title_cannot_break_the_table() { // Titles are model output, and a bare `|` ends the cell. - let out = lane_summary("…", &[finding(Severity::High, "Handle a | b")], "0.1.0"); + let out = lane_summary( + "…", + &[finding(Severity::High, "Handle a | b")], + "0.1.0", + true, + ); let row = out.lines().find(|l| l.contains("Handle a")).expect("row"); assert!(row.contains("\\|"), "{row}"); } @@ -463,6 +509,7 @@ mod tests { "…", &[finding(Severity::High, "Fix ")], "0.1.0", + true, ); assert!(!out.contains("