From 135cdf17f2bd6a6fc399dce420a1b3d72dab99e5 Mon Sep 17 00:00:00 2001 From: Dhar Rawal Date: Sat, 8 Aug 2026 18:34:30 -0500 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20v2.31.0=20=E2=80=94=20replace=20spe?= =?UTF-8?q?edict=20with=20sqlite3=20and=20require=20Python=203.13+=20(fix-?= =?UTF-8?q?lvl)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the abandoned speedict/RocksDB dependency in favor of stdlib sqlite3 (WAL, JSON values, float32 BLOB utterance cache). Unblocks Python 3.13 installs, removes pickle-on-disk from first-party stores, and fixes process-exclusive LOCK failures under concurrent writers. requires-python is now >=3.13,<3.15; pre-existing .rdb / cache.db RocksDB dirs are abandoned. Co-authored-by: Cursor --- .beads/interactions.jsonl | 9 + .beads/issues.jsonl | 16 +- README.md | 5 +- .../intent_detection.py | 24 +-- fastworkflow/cache_matching.py | 96 +++++---- fastworkflow/kvstore.py | 187 ++++++++++++++++++ fastworkflow/run_fastapi_mcp/__main__.py | 12 +- .../run_fastapi_mcp/conversation_store.py | 64 ++---- fastworkflow/workflow.py | 25 ++- poetry.lock | 90 +++------ pyproject.toml | 5 +- tests/soak/memory_soak.py | 4 +- tests/test_cache_matching_sqlite.py | 51 +++++ tests/test_conversation_store_concurrency.py | 60 ++++++ tests/test_enablecache_kvstore.py | 45 +++++ tests/test_fastapi_memory_bounds.py | 35 ++-- tests/test_kvstore.py | 137 +++++++++++++ 17 files changed, 650 insertions(+), 215 deletions(-) create mode 100644 fastworkflow/kvstore.py create mode 100644 tests/test_cache_matching_sqlite.py create mode 100644 tests/test_conversation_store_concurrency.py create mode 100644 tests/test_enablecache_kvstore.py create mode 100644 tests/test_kvstore.py diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index a709891..11de941 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -529,3 +529,12 @@ {"id":"int-1a604fb1bb652a247f93660f85c3105b","kind":"field_change","created_at":"2026-08-08T18:11:15.685177178Z","actor":"Dhar Rawal","issue_id":"fix-1in.4","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}} {"id":"int-b3c36247b3e8862b12e87c057c19ad29","kind":"field_change","created_at":"2026-08-08T18:11:16.39822319Z","actor":"Dhar Rawal","issue_id":"fix-1in","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} {"id":"int-b53eb85f236734101eaa670d8de6c6ad","kind":"field_change","created_at":"2026-08-08T19:44:59.853945913Z","actor":"Dhar Rawal","issue_id":"fix-ry7","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"PR #56 applied with Sourcery review fixes; targeted + full pytest green"}} +{"id":"int-6ff3fca77a76d8abddb06a72a48eab60","kind":"field_change","created_at":"2026-08-08T21:58:50.658033307Z","actor":"Dhar Rawal","issue_id":"fix-lvl.8","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-f83ec8304c5f573d83444cb0d6cfc9a2","kind":"field_change","created_at":"2026-08-08T21:58:54.909404133Z","actor":"Dhar Rawal","issue_id":"fix-lvl.1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}} +{"id":"int-b75136c7fe586dd723ef9d79debdb3a9","kind":"field_change","created_at":"2026-08-08T21:58:55.654856178Z","actor":"Dhar Rawal","issue_id":"fix-lvl.2","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-51badfb2ebc8019682b80ff8c7f1e57d","kind":"field_change","created_at":"2026-08-08T21:58:56.596857655Z","actor":"Dhar Rawal","issue_id":"fix-lvl.3","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-a2b8c5f36f4de5d3f68dfc01830f9e5a","kind":"field_change","created_at":"2026-08-08T21:58:57.291280264Z","actor":"Dhar Rawal","issue_id":"fix-lvl.4","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-b089cab161efe38dd7e4d018d564ea0d","kind":"field_change","created_at":"2026-08-08T21:58:58.09581554Z","actor":"Dhar Rawal","issue_id":"fix-lvl.5","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-bb14552030f257d7ff413535c2394f05","kind":"field_change","created_at":"2026-08-08T21:58:58.842982682Z","actor":"Dhar Rawal","issue_id":"fix-lvl.6","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-cf0e8785fef0883771ec8057f02a4f79","kind":"field_change","created_at":"2026-08-08T21:58:59.663124247Z","actor":"Dhar Rawal","issue_id":"fix-lvl.7","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} +{"id":"int-4f8cc573e4d1dcde4033d4da6211a960","kind":"field_change","created_at":"2026-08-08T21:59:00.341228885Z","actor":"Dhar Rawal","issue_id":"fix-lvl","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 5cfad2a..d521dff 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -6,6 +6,14 @@ {"_type":"issue","id":"fix-551.2","title":"R2: full determinism — seed everything, record provenance","description":"ADDRESSES F2 (S1). No manual_seed, random.seed, np.random.seed, or set_seed call exists anywhere in the package. The only determinism is random_state=42 on the train/test split. Persona selection is free-running (train/generate_synthetic.py:43: selected_indices = random.sample(range(len(persona_dataset)), num_personas)) and model initialisation is unseeded. Consequence: 20.6% verdict churn between two identical runs (92 of 446 held-out routing cases). This blocks incremental training (a command retrained today is not comparable to its neighbours trained last week), blocks utterance caching (no provenance — see D6, R2 is a HARD prerequisite for R6), and blocks any convergence loop.\n\nDO (spec R2):\n- Add a single TRAINING_SEED config value that seeds `random`, `numpy`, and `torch` (including CUDA where applicable). Follow the existing env-var conventions — see the fastworkflow-config-and-flags skill; do not invent a parallel config mechanism.\n- Persist, PER COMMAND, the seed and the PERSONA IDS ACTUALLY USED. This provenance record is what R6's cache fingerprint is keyed on, so its shape is load-bearing for a later issue, not just a log.\n- Document that unseeded runs on a benchmark of a few hundred cases exhibit ~20% verdict churn, and that differences below that floor are not interpretable from a single run.\n\nVERIFY: grep -rn \"manual_seed\\|random.seed\\|set_seed\\|np.random.seed\" --include=*.py fastworkflow/ ; grep -n \"random.sample\" fastworkflow/train/generate_synthetic.py\n\nACCEPTANCE: two full training runs of tests/example_workflow with the same TRAINING_SEED produce byte-identical (or at minimum verdict-identical) artifacts, and the per-command provenance record round-trips.","notes":"DELIVERED (wave 1, not yet integrated). fastworkflow/train/determinism.py plus a reworked generate_synthetic.py; 27 integration tests in tests/test_training_determinism.py.\n\nProvides seed_everything (random / numpy / torch / CUDA / transformers), derived_seed for stable sub-seeds, the TRAINING_SEED env var, and a process-wide ProvenanceRecorder. The recorder exists because generate_diverse_utterances CANNOT return provenance: its signature is public API, called from user-authored command files in every workflow. It pushes into a module-level sink that the trainer installs for the duration of a run and collects afterwards.\n\nCAUTION -- seeding alone does NOT deliver determinism, so this issue is not done when the module lands. See AR5 in docs/intent_training_improvements_spec.md section 10, and blocker fix-9mo (p0), which covers BOTH consequences of non-deterministic set iteration:\n 1. LABEL LOSS: core-command labels silently vanish from ancestor contexts depending on PYTHONHASHSEED, which can make a command unroutable.\n 2. ROW ORDER: model_pipeline_training.py:872-877 appends wildcard rows in set order, and :914 train_test_split(random_state=42) shuffles by POSITION -- so the fixed seed is false reassurance. Empirically confirmed: same 30 rows, 3 PYTHONHASHSEED values, 3 different test partitions.\n\nThis blocks paired comparison of two runs, and therefore blocks R1 (is score X better than Y?) and D6 (utterance cache reuse). fix-9mo must close before fix-551.2 can.\n\n--- BLOCKER ADDED: seeding is not sufficient, measured ---\nTwo runs at TRAINING_SEED=42, same code and env, produced DIFFERENT training data:\n0/5 commands had identical utterance sets, with row counts differing by up to 5\n(add_two_numbers 24 vs 29). Synthetic utterances come from a live LLM, which no seed\ncontrols. See the measurement recorded on fix-551.9.\n\nR2 therefore has TWO blockers, not one:\n fix-9mo ordering (set iteration -\u003e label loss and row order)\n fix-551.9 utterance persistence (R6), now p0, without which the training DATA\n varies run to run and no seed can make the artifacts reproducible\n\nR2's acceptance test should be: two consecutive runs at the same seed produce\nbyte-identical per-context artifacts. Do not close this issue on \"seed_everything is\nwired in\".\nCLOSING R2. Evidence, stated at the strength it actually has:\n\nCLAIM: two runs at a fixed TRAINING_SEED produce identical artifacts, measured on hello_world and messaging_app_4, on a CUDA machine WITHOUT deterministic-algorithm enforcement. Not the broader claim 'training is deterministic'.\n\nIt took three findings to get here, and only the first was foreseen:\n1. Seeding alone does nothing -- 0/5 commands identical at the same seed, because the LLM redraws the training data every run. This inverted D6's sequencing: R6 blocks R2, not the reverse.\n2. There were TWO LLM generation paths, not one. R6 fixed utterances; generate_dspy_examples was still unseeded at temperature 0.9 (fix-czb). Parameter files went 0/1 -\u003e 1/1 on hello_world and 0/5 -\u003e 5/5 on messaging_app_4.\n3. Fine-tuning was ALREADY reproducible and needed no change: artifacts were byte-identical in every condition measured (24/24 and 96/96), including cache-off runs where the training data itself differed. determinism.seed_everything is sufficient here.\n\nMETHODOLOGY CORRECTION worth carrying forward (spec section M6): measuring reproducibility on hello_world ALONE is not evidence. Its parameter fields are already alphabetical, so it structurally cannot detect key-ordering divergence -- an earlier version of the param cache scored 5/5 there while leaving 3/5 artifacts differing on messaging_app_4. Any future determinism claim needs a workflow with non-alphabetical fields.\n\nRESIDUAL RISK, not papered over: nothing sets torch.use_deterministic_algorithms or cudnn.deterministic, so a different GPU, cuDNN version, or batch shape could reintroduce drift. Two workflows measured, not all.","status":"closed","priority":0,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-02T14:35:45Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-08-02T20:23:47Z","close_reason":"Determinism achieved: seed + R6 utterance cache + fix-czb param cache; verified on 2 workflows","dependencies":[{"issue_id":"fix-551.2","depends_on_id":"fix-551.9","type":"blocks","created_at":"2026-08-02T13:21:42Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-551.2","depends_on_id":"fix-551","type":"parent-child","created_at":"2026-08-02T09:35:45Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-551.2","depends_on_id":"fix-czb","type":"blocks","created_at":"2026-08-02T13:17:47Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":2,"dependent_count":2,"comment_count":0} {"_type":"issue","id":"fix-551.1","title":"Adversarial review of the intent-training spec; promote to doc of record","description":"GATE (house lifecycle stage 1 of 7, see the fastworkflow-docs-and-positioning skill §6). docs/intent_training_improvements_spec.md is currently marked \"Proposal, awaiting adversarial review\" and is NOT a doc of record. Design-bearing children of this epic (R1 holdout design, R4 versioning layout, R5 closure rule, R7 wildcard split) should not be implemented against an unreviewed decision log.\n\nDO:\n- Run the adversarial design review per the fastworkflow-proof-and-analysis-toolkit skill against spec §5 (D1-D8) and §4 (R1-R9). Attack, specifically: (a) D1 — is whole-persona holdout actually a generalisation test, or does PersonaHub produce enough cross-persona phrasing overlap that it leaks too? (b) D5 — is the upward closure rule complete, or does the `base` inheritance axis create a second closure direction not covered? (c) D8 — does versioning-before-selective-training actually make merging safe, or does it just make rollback possible? (d) R7.2's budget ratio — is \"relative to the average real command class\" the right sizing rule, or is it a fitted constant from one workflow?\n- Re-verify every volatile fact with the one-liners in spec §9 against the CURRENT HEAD, not v2.23.0/23dbe35, and update any drifted file:line references.\n- Confirm §8 (explicitly not established) is still accurate and that no child issue's description smuggles one of those three non-results in as justification.\n- On acceptance, update the spec status header and record the review outcome.\n\nDO NOT: soften §8. The three non-results are recorded precisely so nobody rediscovers them as facts.","status":"closed","priority":0,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-02T14:35:25Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-08-02T15:26:13Z","close_reason":"Adversarial review complete; docs/intent_training_improvements_spec.md promoted to doc of record. The review invalidated D1, D5 and D8 and R7.2's sizing rule, and surfaced two new findings: AR5 (fix-9mo, p0 - set-iteration order silently drops core-command labels and makes runs unreproducible) and AR6 (fix-lfz, p2 - R7.2 share figures do not reconcile, not citable). Full record in spec section 10.","dependencies":[{"issue_id":"fix-551.1","depends_on_id":"fix-551","type":"parent-child","created_at":"2026-08-02T09:35:24Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":4,"comment_count":0} {"_type":"issue","id":"fix-85g.8","title":"Step 1 integration tests: long-running task simulation (wait-or-defer + single-flight)","description":"Integration tests (NO mocks; real test workflow) simulating a workflow with a deliberately LONG-running command. Build a test workflow/command whose _process_command sleeps longer than wait_seconds (configurable) to deterministically simulate the ~10-min LLM call. Assert: (1) a turn exceeding wait_seconds returns a deferred response (running + turn_key) WITHOUT orphaning the execution; (2) a retry with the same args returns the SAME turn_key and starts NO second execution (assert the long command body runs exactly once, e.g. via a call counter/log); (3) the execution still completes and persists after the request returned; (4) §3.3: a retry arriving before /initialize startup completes receives exec_state=running + startup_turn_key, never an empty startup_output; (5) the fast path returns the result inline when the command is quick; (6) no ctx mutation outside runtime.lock; (7) process_message DeprecationWarning no longer emitted. Follow tests/ conventions (integration only).","notes":"TEST SCOPE NARROWING (per user): Test ONLY /initialize with a startup_action so NO workflow training is required (no trained intent models needed — a startup_action dispatches a command directly by name, bypassing NLU/intent detection). Build/extend a small test workflow with a command whose _process_command sleeps for a configurable duration to simulate the long-running LLM call. Drive everything through POST /initialize (startup_action -\u003e submit_turn(kind='initialize_startup')). Concretely assert: (1) FAST PATH — when the startup command is quick (sleep \u003c wait_seconds), /initialize returns 200 inline with startup_output + exec_state=DONE; (2) DEFER — when sleep \u003e wait_seconds, /initialize returns 202 with startup_turn_key + exec_state=running WITHOUT orphaning the execution; (3) SINGLE-FLIGHT RETRY (§3.3) — a second /initialize for the same channel arriving before startup completes hits the 'already exists' branch and returns the SAME startup_turn_key + exec_state=running (NEVER an empty startup_output), and the sleeping command body runs EXACTLY ONCE (assert via a call counter / log); (4) the deferred execution still completes and persists after the request returned; (5) the process_message DeprecationWarning is no longer emitted. Skip the /invoke_agent and /invoke_assistant agent-path assertions here (those require trained models) — cover them separately if/when training fixtures exist.\nWORKFLOW TEMPLATE (per user): The hello_world example is a good template for the small test workflow. See tests/hello_world_workflow/ (and fastworkflow/examples/hello_world/) — a minimal application/ + _commands/ pair (e.g. add_two_numbers.py with a Signature.Input/Output + ResponseGenerator). Clone that structure into a new fixture (e.g. tests/long_running_workflow/) and make the command's _process_command sleep for a configurable duration (read from the Input params or an env var) to simulate the long LLM call. Because the test drives it via /initialize startup_action (direct dispatch by command name), no plain_utterances/generate_utterances/training is needed — the Signature can be minimal. Reuse the existing tests/ conftest fixtures and conventions.","status":"closed","priority":0,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-06-23T19:19:46Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-06-23T20:01:53Z","close_reason":"tests/test_fastapi_turns_async.py: fast-path inline, defer+single-flight (one execution across retries), pointer-based 409. Driven via /initialize startup_action against tests/hello_world_workflow (no training). Full suite 457 passed.","dependencies":[{"issue_id":"fix-85g.8","depends_on_id":"fix-85g.3","type":"blocks","created_at":"2026-06-23T14:20:44Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-85g.8","depends_on_id":"fix-85g.4","type":"blocks","created_at":"2026-06-23T14:20:45Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-85g.8","depends_on_id":"fix-85g.7","type":"blocks","created_at":"2026-06-23T14:20:50Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-85g.8","depends_on_id":"fix-85g.6","type":"blocks","created_at":"2026-06-23T14:20:48Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-85g.8","depends_on_id":"fix-85g","type":"parent-child","created_at":"2026-06-23T14:19:46Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-85g.8","depends_on_id":"fix-85g.5","type":"blocks","created_at":"2026-06-23T14:20:47Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":5,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"fix-lvl.7","title":"Verification: concurrency, no-pickle, install, perf gate","description":"Design §6 checklist:\n1. Install/import on advertised Python versions (esp. 3.13)\n2. No pickle protocol markers or class names on disk after write\n3. 4-process ConversationStore concurrency (kvbench4 pattern; report via exit codes not Queue-after-join)\n4. Round-trip per module for all key shapes\n5. KeyError del semantics\n6. Mutate-while-iterating keys()\n7. cache_matching perf gate @1000 utterances\n\nPrefer integration tests under tests/; do not wipe ___command_info.","acceptance_criteria":"Checklist items covered by tests or documented manual verification; suite green for touched paths","status":"closed","priority":1,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:48Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:59:00Z","closed_at":"2026-08-08T21:59:00Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.7","depends_on_id":"fix-lvl.6","type":"blocks","created_at":"2026-08-08T16:53:47Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.7","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:47Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"fix-lvl.5","title":"Migrate conversation_store to KVStore (.sqlite3)","description":"Design §4 Step 4 (do last among stores — durable data). _get_db() → KVStore. Filename {channel_id}.rdb → {channel_id}.sqlite3. Abandon old .rdb (no migration).\n\nDelete unreachable legacy inline-turns path (~lines 64–74) after confirming no other writer produces inline turns. Check del db[turn_key] KeyError semantics.","acceptance_criteria":"Uses .sqlite3; no speedict; legacy inline turns path removed if unused; KeyError semantics preserved","status":"closed","priority":1,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:47Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:58Z","closed_at":"2026-08-08T21:58:58Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.5","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:46Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.5","depends_on_id":"fix-lvl.1","type":"blocks","created_at":"2026-08-08T16:53:46Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"fix-lvl.6","title":"Drop speedict dependency and widen requires-python","description":"Design §4 Step 5. Remove speedict\u003e=0.3.12,\u003c0.4 from pyproject.toml (and poetry.lock). Add nothing — sqlite3 is stdlib. Widen requires-python from \u003e=3.11,\u003c3.14 to at least \u003c3.15.\n\nRelease notes: mention abandoned .rdb files; note downstream consumers (e.g. xray rocksdict shim) can delete their workaround.","acceptance_criteria":"poetry/pip resolve without speedict; requires-python includes 3.13; no speedict in lockfile","status":"closed","priority":1,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:47Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:59Z","closed_at":"2026-08-08T21:58:59Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.6","depends_on_id":"fix-lvl.3","type":"blocks","created_at":"2026-08-08T16:53:47Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.6","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:47Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.6","depends_on_id":"fix-lvl.4","type":"blocks","created_at":"2026-08-08T16:53:47Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.6","depends_on_id":"fix-lvl.2","type":"blocks","created_at":"2026-08-08T16:53:47Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.6","depends_on_id":"fix-lvl.5","type":"blocks","created_at":"2026-08-08T16:53:47Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":4,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"fix-lvl.3","title":"Migrate intent_detection NLU cache to KVStore","description":"Design §4 Step 2 (easiest real store). Six Rdict(cache_path) sites → KVStore. Keys: suggested_commands, flag_type, utterance_count. Pure cache; miss = recompute.","acceptance_criteria":"No speedict/Rdict in intent_detection.py; round-trip of the three key shapes works","status":"closed","priority":1,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:46Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:57Z","closed_at":"2026-08-08T21:58:57Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.3","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:45Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.3","depends_on_id":"fix-lvl.1","type":"blocks","created_at":"2026-08-08T16:53:45Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"fix-lvl.4","title":"Migrate cache_matching to float32 BLOB utterance_cache schema","description":"Design §4 Step 3 — MUST NOT be a mechanical KVStore/JSON swap (measured 3–6x regression).\n\nOwn table:\n CREATE TABLE utterance_cache (\n k TEXT PRIMARY KEY,\n meta TEXT NOT NULL, -- JSON: command_mapping, feedback dates\n vec BLOB NOT NULL -- numpy float32 embedding .tobytes()\n );\nWrite: np.asarray(embedding, dtype=np.float32).tobytes()\nRead: np.frombuffer(vec, dtype=np.float32)\n\nOne row per utterance (no whole-cache blob rewrite). Perf gate: 1000 utterances match+store cycle \u003c\u003c 238ms speedict baseline (target ~1.8ms).","acceptance_criteria":"No speedict in cache_matching.py; embeddings never JSON-encoded; perf gate under baseline","status":"closed","priority":1,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:46Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:57Z","closed_at":"2026-08-08T21:58:57Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.4","depends_on_id":"fix-lvl.1","type":"blocks","created_at":"2026-08-08T16:53:46Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.4","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:46Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"fix-lvl.1","title":"Add fastworkflow/kvstore.py (SQLite WAL JSON KVStore)","description":"Implement KVStore in fastworkflow/kvstore.py per design §3.\n\nAPI: __setitem__, __getitem__, __delitem__, __contains__, get, keys, close, context manager.\nValues JSON-serialisable only. PRAGMA journal_mode=WAL, synchronous=NORMAL, busy_timeout.\ncheck_same_thread=False (sqlite3.threadsafety==3).\nkeys() must materialise via fetchall() before iterating (conversation_store mutates while iterating).\n__delitem__ raises KeyError on missing key (dict semantics).\n\nNo callers yet — pure additive.","acceptance_criteria":"Module importable; unit tests cover 8 operations + KeyError + concurrent keys iteration","status":"closed","priority":1,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:45Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:55Z","started_at":"2026-08-08T21:53:57Z","closed_at":"2026-08-08T21:58:55Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.1","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:44Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":4,"comment_count":0} +{"_type":"issue","id":"fix-lvl.2","title":"Migrate enablecache/workflow.py to KVStore","description":"Design §4 Step 1. Swap Rdict for KVStore in enablecache. Update speedict block comment (lines ~17–39).\n\nJSON constraint: catch TypeError on cache write and raise clear message that @enablecache requires JSON-serialisable return. Do NOT reintroduce pickle.\n\nenablecache is dead code today (no callers) — blast radius none. Out of scope: cache key str(args)+str(kwargs) collision (file separate issue).","acceptance_criteria":"No speedict import in workflow.py; enablecache uses KVStore; TypeError message on non-JSON return","status":"closed","priority":1,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:45Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:56Z","closed_at":"2026-08-08T21:58:56Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.2","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:45Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-lvl.2","depends_on_id":"fix-lvl.1","type":"blocks","created_at":"2026-08-08T16:53:45Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"fix-lvl","title":"Replace speedict with stdlib sqlite3","description":"Replace abandoned speedict/RocksDB with stdlib sqlite3 (WAL). Unblocks Python 3.13+ installs, removes pickle RCE surface, and fixes process-exclusive lock concurrency failures.\n\nDesign of record: /home/drawal/rl/fastworkflow-speedict-to-sqlite-migration.md (analysed against fastworkflow 2.30.1, 2026-08-08).\n\nDecision: go straight to sqlite3 — do NOT route through rocksdict or sqlitedict. No data migration; abandon existing .rdb stores.\n\nWhy: speedict has no 3.13 wheel and no sdist; upstream abandoned (Speedb acquired by Redis); pickle on disk; RocksDB exclusive LOCK fails/hangs under multi-process.\n\nScope modules: workflow.py (enablecache), intent_detection.py, cache_matching.py (BLOB schema required), conversation_store.py. New module: fastworkflow/kvstore.py.\n\nAcceptance: pip install works on 3.11–3.14; no pickle markers on disk; 4-process ConversationStore concurrency succeeds; cache_matching match+store @1000 utterances \u003c\u003c 238ms baseline; speedict removed from pyproject; requires-python widened to at least \u003c3.15.","acceptance_criteria":"- No speedict import or dependency remains\n- requires-python allows 3.13 (and preferably 3.14)\n- KVStore (JSON) used for enablecache, intent_detection, conversation_store\n- cache_matching uses float32 BLOB + JSON meta schema (not all-JSON)\n- Verification checklist in design doc §6 passes\n- Release notes mention abandoned .rdb files and downstream shim deletion","notes":"Landed: kvstore.py (KVStore + UtteranceCacheStore), all four call sites migrated, speedict removed, requires-python \u003e=3.11,\u003c3.15.\n\nFollow-ups discovered during verification:\n- Existing RocksDB dirs named cache.db blocked sqlite open → NLU paths now *.sqlite3\n- command_router.predict() returns ndarray → coerced to list[str] before JSON store\n- Follow-up issue fix-snl for enablecache key collision","status":"closed","priority":1,"issue_type":"epic","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:30Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T22:00:37Z","closed_at":"2026-08-08T21:59:00Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-ry7","title":"Apply PR #56: context-aware command visibility (with review fixes)","description":"Apply https://github.com/radiantlogicinc/fastworkflow/pull/56 locally onto current main, incorporating Sourcery review feedback, then verify.\n\n## PR summary\nTwo related fixes that make the command lists shown to the agent and the planner context-aware:\n- Executor: re-scope the ReAct agent's available_commands to the current context whenever it changes (set_current_*, go_up, reset_context) via a context-change observer on the Workflow, so the agent never acts on a stale command list after a mid-trajectory switch.\n- Planner: build_query_with_next_steps now uses get_all_contexts_command_display_text so the planner sees every available context's commands.\n\n## Files changed in PR\n- fastworkflow/command_metadata_api.py (+67)\n- fastworkflow/utils/react.py (+5)\n- fastworkflow/workflow.py (+19)\n- fastworkflow/workflow_agent.py (+15/-1)\n- fastworkflow/workflow_execution_context.py (+10)\n\nHead commit: 8937222895e12206ac4526365bad580ff0666cdb\nBranch: sanchit056:agentic_enhancements_v2\n\n## REQUIRED review fixes (must apply, not just cherry-pick)\n1. Context-change listeners on Workflow: add remove/unsubscribe API and/or weakrefs so short-lived workflows/agents do not leak via long-lived listener lists that only append.\n2. In _notify_context_change: log with exc_info=True (or full traceback), not just `{exc}`, while still never breaking context switching.\n3. In command_metadata_api.py get_all_contexts_command_display_text fallback except: log warning with exc_info=True before returning base_text; use existing module logger conventions (import logging / getLogger if needed).\n\n## Acceptance\n- PR changes applied and review comments addressed\n- Targeted tests for this PR's behavior (context-aware command visibility / listener refresh / planner all-contexts display)\n- Full pytest regression suite green (do NOT run two full suites at once; suite ~24 min; activate .venv)\n- Close this bead and export: bd export -o .beads/issues.jsonl\n- Do NOT commit/push unless asked\n\n## Notes\n- Local remote: origin=dharrawal/fastworkflow, upstream=radiantlogicinc/fastworkflow\n- Use mgrep for searches, not Grep/WebSearch\n- Conservative beads profile: no commit/push without approval","notes":"Applying PR #56 head 8937222 onto main; will add Sourcery review fixes (unsubscribe API, exc_info logging) then targeted tests + full suite.\nPR applied + review fixes (remove listener, WeakMethod, exc_info logging, WEC.close unsubscribe). Targeted tests: 8 passed in tests/test_context_aware_command_visibility.py. Starting full pytest suite.\nFull suite green: 1704 passed, 14 skipped in 2837s (~47m). Closing.","status":"closed","priority":1,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T18:54:42Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T19:45:00Z","started_at":"2026-08-08T18:55:04Z","closed_at":"2026-08-08T19:45:00Z","close_reason":"PR #56 applied with Sourcery review fixes; targeted + full pytest green","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-1in.4","title":"Bump to 2.30.0 and verify full pytest suite","description":"Bump pyproject.toml version to 2.30.0 (behaviour change). Run the full suite alone (no concurrent suites — OOM risk). Record pass counts in epic close notes.","acceptance_criteria":"version=2.30.0; full suite green with zero failures.","notes":"Version bumped to 2.30.0 in pyproject.toml. Starting full suite.\nFull suite: 1696 passed, 14 skipped, 0 failed in 2817.51s. PYTEST_EXIT=0. Version 2.30.0.","status":"closed","priority":1,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T17:21:49Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T18:11:16Z","started_at":"2026-08-08T17:23:55Z","closed_at":"2026-08-08T18:11:16Z","close_reason":"Closed","labels":["bug","validation"],"dependencies":[{"issue_id":"fix-1in.4","depends_on_id":"fix-1in.2","type":"blocks","created_at":"2026-08-08T12:22:00Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-1in.4","depends_on_id":"fix-1in","type":"parent-child","created_at":"2026-08-08T12:21:49Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-1in.4","depends_on_id":"fix-1in.1","type":"blocks","created_at":"2026-08-08T12:22:00Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":2,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-1in.2","title":"Add regression tests for direct-action validation parity","description":"Add integration tests (no mocks of FastWorkflow components) that lock the invariant: create() path and perform_action path both reject with the hook's own message when a context precondition fails.\n\nCover:\n1. Command with parameters + context precondition missing → ValueError with hook message\n2. Same with context present → success\n3. Command with no/empty parameters + context precondition → still rejected (exercises the old if action.parameters guard)\n\nPrefer a small dedicated fixture workflow under tests/ so the hook message is unambiguous and no LLM is required.","acceptance_criteria":"New tests fail on unfixed code and pass after the fix; assert on hook message text.","notes":"Added tests/direct_action_validation_workflow + tests/test_direct_action_validation.py (6 passed).","status":"closed","priority":1,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T17:21:44Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T17:22:58Z","started_at":"2026-08-08T17:22:57Z","closed_at":"2026-08-08T17:22:57Z","close_reason":"Closed","labels":["bug","validation"],"dependencies":[{"issue_id":"fix-1in.2","depends_on_id":"fix-1in","type":"parent-child","created_at":"2026-08-08T12:21:43Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-1in.2","depends_on_id":"fix-1in.1","type":"blocks","created_at":"2026-08-08T12:21:59Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} @@ -192,11 +200,13 @@ {"_type":"issue","id":"fix-vof.17","title":"R41: Disambiguate the live response path (in-memory payloads vs store handles)","description":"FINDING: section 10.1 offloads payloads at persistence; section 10.3 says the live xray mapping fetches payloads lazily from the PayloadStore by handle. One reading pays put-then-get per turn for bytes already in memory; the other (inline from memory) makes the lazily-by-handle language wrong and balloons bundled-server model_dump responses with all gallery payloads. OPEN QUESTIONS: (1) live path maps from the in-memory TurnResult with handles reserved for the review reader (recommended), or serve live from handles for response-size control; (2) per-response payload size cap for the bundled server JSON/SSE bodies. Interacts with R45 eager offload. See docs/turn_result_design_review.md R41.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","design":"RESOLVED 2026-06-11 (with Dhar). (1) LIVE RESPONSE SERVED FROM RAM: the runner/mapping reads payloads from the in-memory TurnResult; offload happens on a serialized COPY at the persistence boundary (R10 copy-on-serialize), purely for review. Zero store reads on the hot path. Section 10.3's fetching-lazily-by-handle language corrected to apply ONLY to the review reader, never the live path. (2) BUNDLED-SERVER RESPONSES: payloads inline up to a configurable cap (MAX_INLINE_PAYLOAD, generous default ~10MB); above the cap the response carries the A10 envelope marked not-inlined-available-via-review-record (path depends on the future R32 read API, documented as such). (3) R45 CONSEQUENCE (noted here, finalized there): with the live path pinned to RAM, payloads must stay in memory until the response is built - eager offload buys no memory relief, so R45 collapses to boundary-offload with a documented memory profile. Recorded in design doc Amendment A16 and review doc R41.","notes":"Q and A (after ELI5 kitchen/warehouse with explicit R45 coupling): serve-from-RAM + configurable cap, both recommended options. R45 (fix-vof.43) is now mostly determined.","status":"closed","priority":2,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:23Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-11T19:37:23Z","closed_at":"2026-06-11T19:39:51Z","close_reason":"Finding finalized: live path serves from RAM, configurable inline cap, handles are review-only; recorded in bead, review doc R41, design doc Amendment A16","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.17","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:22Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-vof.18","title":"R43: Define the command_output_queue / CLI contract after the redesign","description":"FINDING: the queue transport has its own contract the design never updates: _finalize_agent_output, _process_message and _process_action all enqueue the final CommandOutput; Topology A enqueues mid-turn clarifications directly; the CLI (run/__main__.py) consumes the queue and iterates command_responses. OPEN QUESTIONS: (1) queue carries per-event CommandOutputs plus a terminal TurnResult (recommended - mirrors live-then-final), or TurnResult only; (2) enumerate ChatSession.keep_alive and CLI renderer changes in section 11. See docs/turn_result_design_review.md R43.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","status":"closed","priority":2,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:23Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-11T19:48:46Z","closed_at":"2026-06-11T20:03:58Z","close_reason":"Finding finalized: queue carries status-stamped TurnResults only, trace queue untouched, sentinel pairing rule fixes fix-5fv; recorded in bead, review doc R43, design doc Amendment A19","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.18","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:23Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-vof.15","title":"R38: Give /post_feedback a home compatible with write-once review records","description":"FINDING: /post_feedback (run_fastapi_mcp/__main__.py:1276-1316) mutates the last in-memory conversation turn AFTER completion - after the write-once review record is sealed. Feedback is first-order observability data and the design never mentions it. OPEN QUESTIONS: choose (a) mutable feedback field on review records (violates write-once); (b) separate feedback keyspace keyed by turn_key (recommended); (c) feedback stays in ConversationStore with a turn_key link (fits R37 consolidation). See docs/turn_result_design_review.md R38.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","design":"RESOLVED 2026-06-11 (with Dhar) - SEPARATE FEEDBACK RECORDS. Feedback is its own small record type in the unified keyspace: fw:feedback:{channel}:{conv_id}:{turn_key} holding score, comment, timestamp. (1) Turn records stay STRICTLY write-once - R16 enforcement and audit immutability hold with no carve-outs. (2) Same prefix scans serve everything: review reader and A1 memory rebuild fetch turns + feedback cards in one conversation-prefix scan; A8/A12 retention deletes cards with their envelopes automatically. (3) Conventions matching today: re-posting overwrites (last-write-wins on the single feedback key); /post_feedback targets the latest completed turn of the active conversation (arbitrary-turn feedback = future R32 read-API territory); feedback enters the agent-memory projection for completed turns exactly as the dspy.History feedback slot does today (A1 projection constraint satisfied via the read-time join). Recorded in design doc Amendment A18 and review doc R38.","notes":"Q and A (after ELI5 sealed-envelopes/sticky-notes): separate feedback records, recommended option. Original option (c) (ConversationStore home) was dead post-A1; its spirit maps to this resolution within the unified store.","status":"closed","priority":2,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:22Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-11T19:44:27Z","closed_at":"2026-06-11T19:46:12Z","close_reason":"Finding finalized: separate feedback records in the unified keyspace, turn records stay write-once; recorded in bead, review doc R38, design doc Amendment A18","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.15","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:21Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-vof.15","depends_on_id":"fix-vof.3","type":"blocks","created_at":"2026-06-10T16:29:39Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"fix-vof.16","title":"R39: Model cancelled turns (/cancel_pending)","description":"FINDING: /cancel_pending (run_fastapi_mcp/__main__.py:1133-1160) abandons a suspended ask_user turn and clears pending state - a fourth terminal state the design does not model. OPEN QUESTIONS: (1) do cancelled turns write a review record with status=cancelled carrying the partial event sequence (recommended - it is real history); (2) who cleans up payloads offloaded at the suspend boundary, now referenced by nothing (ties to R24). Depends on the R11 status enum. See docs/turn_result_design_review.md R39.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","design":"RESOLVED 2026-06-10 (with Dhar). (1) RECORD, NOT SHRED: /cancel_pending and the A2 auto-cancel-on-switch paths write a turn record with status=cancelled under the turn's ORIGINAL conversation, carrying the partial event sequence (commands executed so far + the unanswered clarification question; event shape per R1) and the payload handles already offloaded at suspend. Payload ownership transfers from the pending blob to the cancelled record - no cleanup step needed; R24's orphan concern dissolves for the cancel path (abandoned path remains with R6.3). Sequence under per-session lock: serialize partial TurnResult (status=cancelled) -\u003e write record -\u003e clear pending blob. Today's cancel_pending() (workflow_execution_context.py:289) only resets in-memory state; it gains the record write. (2) R7 interplay: if review persistence is disabled by config, cancel falls back to shredding, and only that mode performs an explicit suspend-payload delete. (3) AGENT-MEMORY PROJECTION RULE (clarifies A1.2): rebuilt dspy.History projects from status=completed records ONLY; cancelled/failed/abandoned records are review-only and never enter agent working memory - matches today's behavior. Recorded in design doc Amendment A4 and review doc R39.","notes":"Q and A (after ELI5): record vs shred = record (with R7 config fallback); memory projection = completed only. Cross-refs: R1 (unanswered-question event shape), R24 (cancel-path orphans dissolved), R6.3 (abandoned path still open), R7 (config switch).","status":"closed","priority":2,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:22Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-06-10T23:01:50Z","close_reason":"Finding finalized: cancelled turns recorded under original conversation; completed-only memory projection; recorded in bead, review doc R39, design doc Amendment A4","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.16","depends_on_id":"fix-vof.14","type":"blocks","created_at":"2026-06-10T16:29:40Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-vof.16","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:22Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"fix-vof.16","title":"R39: Model cancelled turns (/cancel_pending)","description":"FINDING: /cancel_pending (run_fastapi_mcp/__main__.py:1133-1160) abandons a suspended ask_user turn and clears pending state - a fourth terminal state the design does not model. OPEN QUESTIONS: (1) do cancelled turns write a review record with status=cancelled carrying the partial event sequence (recommended - it is real history); (2) who cleans up payloads offloaded at the suspend boundary, now referenced by nothing (ties to R24). Depends on the R11 status enum. See docs/turn_result_design_review.md R39.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","design":"RESOLVED 2026-06-10 (with Dhar). (1) RECORD, NOT SHRED: /cancel_pending and the A2 auto-cancel-on-switch paths write a turn record with status=cancelled under the turn's ORIGINAL conversation, carrying the partial event sequence (commands executed so far + the unanswered clarification question; event shape per R1) and the payload handles already offloaded at suspend. Payload ownership transfers from the pending blob to the cancelled record - no cleanup step needed; R24's orphan concern dissolves for the cancel path (abandoned path remains with R6.3). Sequence under per-session lock: serialize partial TurnResult (status=cancelled) -\u003e write record -\u003e clear pending blob. Today's cancel_pending() (workflow_execution_context.py:289) only resets in-memory state; it gains the record write. (2) R7 interplay: if review persistence is disabled by config, cancel falls back to shredding, and only that mode performs an explicit suspend-payload delete. (3) AGENT-MEMORY PROJECTION RULE (clarifies A1.2): rebuilt dspy.History projects from status=completed records ONLY; cancelled/failed/abandoned records are review-only and never enter agent working memory - matches today's behavior. Recorded in design doc Amendment A4 and review doc R39.","notes":"Q and A (after ELI5): record vs shred = record (with R7 config fallback); memory projection = completed only. Cross-refs: R1 (unanswered-question event shape), R24 (cancel-path orphans dissolved), R6.3 (abandoned path still open), R7 (config switch).","status":"closed","priority":2,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:22Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-06-10T23:01:50Z","close_reason":"Finding finalized: cancelled turns recorded under original conversation; completed-only memory projection; recorded in bead, review doc R39, design doc Amendment A4","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.16","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:22Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-vof.16","depends_on_id":"fix-vof.14","type":"blocks","created_at":"2026-06-10T16:29:40Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-vof.13","title":"R10: Answer aliasing - copy-on-serialize rule and headline/gallery dedup","description":"FINDING: in the deterministic path answer IS command_outputs[-1].command_response (same object). In-place offload would corrupt the live TurnResult through both aliases; and the xray mapping would render the same payload twice (headline source and gallery entry). Blind deepcopy is also wrong - artifacts can hold live app objects and command_parameters holds a Pydantic model. OPEN QUESTIONS: (1) confirm selective-copy serializer rule (no in-place mutation, no blind deepcopy); (2) confirm mapping dedup rule (skip gallery entries whose command_response is the answer object). See docs/turn_result_design_review.md R10.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","design":"RESOLVED 2026-06-11 (with Dhar). (1) SELECTIVE COPY-ON-SERIALIZE (stated; implied by A10/A16): the serializer never mutates live TurnResult/CommandOutput objects and never blind-deep-copies them (artifacts can hold live application objects; command_parameters holds a typed model). It builds a new structure - small fields copied, values converted per A10's contract (threshold offload to envelope, model_dump for parameters, strict rejection otherwise) - originals untouched. A16's pristine-in-RAM live path depends on this. (2) HEADLINE NEVER CARRIES PAYLOADS (supersedes the reviewer's earlier identity-skip suggestion): the headline ResponseTuple is always narrative text + metadata; the gallery always contains ALL payload-bearing outputs in turn order. One uniform invariant across agent/assistant/action turns; the faithful reading of decision 21 (the UI, not the framework, picks the featured payload); deterministic-turn duplication eliminated by construction. Recorded in design doc Amendment A20 and review doc R10.","notes":"Q and A (after ELI5 one-object-two-names): headline-never-carries-payloads chosen - the reviewer's own recommendation updated mid-pass from identity-skip after testing it against decision 21's UI-decides intent.","status":"closed","priority":2,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:21Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-11T20:08:39Z","closed_at":"2026-06-11T20:12:21Z","close_reason":"Finding finalized: selective copy-on-serialize, headline never carries payloads; recorded in bead, review doc R10, design doc Amendment A20","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.13","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:20Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-vof.14","title":"R11: Introduce first-class TurnResult.status (replace awaiting_user artifact sniffing)","description":"FINDING: suspension is signaled today by artifacts[awaiting_user]=True set in _awaiting_user_output (workflow_execution_context.py:518-520) and sniffed by the runner and utils.py:355-357 - a stringly-typed protocol. The redesign is the moment to replace it. OPEN QUESTIONS: (1) confirm TurnStatus enum: completed, awaiting_user, failed, cancelled, abandoned; (2) confirm it replaces _output_is_awaiting_user and the runner branch condition; (3) scope - does the artifacts protocol get removed in the same release. Gates R6, R39, R40. See docs/turn_result_design_review.md R11.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","design":"RESOLVED 2026-06-10 (with Dhar). (1) TurnResult.status: TurnStatus enum with ALL FIVE values: completed, awaiting_user, failed, cancelled, abandoned. cancelled is required by R9 auto-cancel-on-switch; abandoned is reserved now (written only if the R6.3 stale-pending sweep is built) so readers never need a schema bump. (2) The artifacts[awaiting_user] protocol is REMOVED IMMEDIATELY in the release that introduces TurnResult - no dual-publish window. _awaiting_user_output stops stamping the artifact; _output_is_awaiting_user sniffing (utils.py:354-357, xray runner) is deleted; the branch becomes turn.status == AWAITING_USER. Contained because every consumer of the old signal is in-repo or user-owned and is being rewritten for the TurnResult return type in the same release anyway; does not by itself force a big-bang release in R46. (3) NO FURTHER CLEANUP (user decision): all four CommandOutput predicates and the NLU-internal artifact handshake (command_handled / command_name / cmd_parameters between wildcard.py and CommandExecutor) carry into the redesigned model unchanged per design section 5.2. For the record: command_aborted and not_what_i_meant were verified to have zero consumers in framework+tests (definitions only, __init__.py:78,86); retained deliberately as public API. Formalizing the handshake is possible future work, out of scope. Recorded in design doc Amendment A3 and review doc R11.","notes":"Q and A: enum = all five; artifact removal = immediate (user chose against dual-publish recommendation); cleanup scope = none (user chose conservative option after ELI5 of the sticky-note protocol and the dead-predicate evidence). Unblocks R6 (fix-vof.8), R39 (fix-vof.16), R40 (fix-vof.11).","status":"closed","priority":2,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:21Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-10T21:59:27Z","closed_at":"2026-06-10T22:54:34Z","close_reason":"Finding finalized: 5-value TurnStatus, immediate artifact removal, no further cleanup; recorded in bead, review doc R11, design doc Amendment A3","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.14","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:21Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":3,"comment_count":0} {"_type":"issue","id":"fix-vof.12","title":"R9: Add conversation scoping to the review namespace","description":"FINDING: one channel spans multiple conversations (/new_conversation, ConversationStore conversation ids), so TurnReviewStore.list(channel_id) interleaves turns across conversations with no way to scope a review. OPEN QUESTIONS: (1) conversation id in the turn key vs required record metadata with filtered listing; (2) source of the id (ConversationStore owns the counter - interacts with R37); (3) define behavior when the conversation switches while a turn is suspended. See docs/turn_result_design_review.md R9.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","design":"RESOLVED 2026-06-10 (with Dhar). (1) Namespace resolved structurally by R37/A1: conversation id is a key component of every turn record (fw:turn:{channel}:{conv_id}:{sortable-ts}-{uuid}); per-conversation listing is a prefix scan; channel-wide observability scans use the channel prefix. (2) Eager conversation-id reservation: active_conversation_id is guaranteed at session creation (restore-last or reserve-new), required because R16 mints the turn key at logical-turn start; deployments that never rotate conversations operate in a single implicit conversation. (3) Switch policy = AUTO-CANCEL: verified that neither /new_conversation (__main__.py:1174) nor /activate_conversation (:1349) checks awaiting_user or the pending store today. Decision: when a turn is suspended, both endpoints first cancel it - record the partial turn under its ORIGINAL conversation with status=cancelled (R39 semantics), clean up pending blob and suspend-offloaded payloads (R24) - then switch. Carry-across rejected (diverges agent memory from store under A1); 409-block rejected (UX). (4) Implementation note: cancel-then-switch must run under the per-session lock, which these endpoints do not currently acquire. Recorded in design doc Amendment A2 and review doc R9.","notes":"Q and A: switch-while-suspended policy = auto-cancel (recommended option). Eager id reservation and channel-prefix listing settled without user input (conventional defaults). Cross-refs: R39 cancelled status, R24 orphan cleanup, R16 key minting.","status":"closed","priority":2,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:20Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-10T21:53:28Z","closed_at":"2026-06-10T21:56:29Z","close_reason":"Finding finalized: structural namespace via A1, eager id reservation, auto-cancel on switch; recorded in bead, review doc R9, design doc Amendment A2","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.12","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:20Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-vof.12","depends_on_id":"fix-vof.3","type":"blocks","created_at":"2026-06-10T16:29:39Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-6b4","title":"Add TTL/reaper for orphaned suspended-session blobs","description":"In Topology B, when an agent suspends on ask_user and the user never returns, the durable pending-state blob persists indefinitely. SessionStateStore.save is called on suspend and on LRU eviction (eviction only SAVES, never deletes), and there is no TTL or background reaper. The only cleanup path is the /cancel_pending endpoint (ctx.cancel_pending() + store.clear()). Over time, abandoned blobs accumulate in DiskSessionStateStore (JSON files) and RedisSessionStateStore. Add a TTL/expiry mechanism: e.g. Redis key TTL on save, and a reaper/age-based sweep for the disk backend, with a configurable max-age. Ensure resuming refreshes the TTL.","acceptance_criteria":"Abandoned pending blobs are eventually removed without manual cancel_pending; TTL is configurable; resuming an active suspension refreshes its expiry; tests cover expiry on both disk and redis backends.","notes":"Surfaced during trajectory-serialization review. Related: fastworkflow/session_state_store.py, fastworkflow/run_fastapi_mcp/utils.py (persist_pending_after_turn, _evict_oldest_if_needed).\nReaper contract addition from R24 final confirmation (2026-06-11): when deleting a stale pending blob, also delete the in-flight turn's payload prefix (fw:payload:{channel}:{conv}:{turn_key}:*) - trivial under A8 turn-scoped keys.\nIMPLEMENTED v2.29.0.\n\nORDERING MATTERED, AND NOT IN THE DIRECTION THE ISSUE IMPLIED. Before fix-g03.25 removed the awaiting_user pin, reaping a pending blob was harmless but useless: the pinned in-memory copy kept the session working, so the reaper deleted only the crash-recovery path while the memory leak continued. After the pin was removed the blob can be the ONLY copy of a suspended conversation, which makes the reaper both effective and destructive. Doing this first would have been strictly worse than doing nothing.\n\nDESIGN, mirroring fix-jtr rather than inventing a second shape.\n- PendingRetentionPolicy(max_age_seconds=7d, max_entries=10000). Stated, not hidden -- decision 16 forbids a hidden durable-state TTL, not a TTL. Age alone is not a byte bound (steady state is abandonment_rate x window x blob_size, a plateau set by a rate the operator does not control), so the count cap is what makes size independent of arrival rate and it is on by default. 7 days because what is being deleted is a user half-finished conversation: too long costs disk, too short costs their work.\n- reap(policy, protected_channel_ids, now, dry_run) -\u003e PendingReapOutcome(reclaimed, protected, scanned, unreadable). Invoked, never scheduled: a timer inside the store would be a second writer on a channel whose only writer is meant to be the owning process.\n- Protection is enforced at ChannelSessionManager.reap_pending_state(), which passes set(_sessions) | set(_leases). The store cannot know what is live; the manager is the only layer that does.\n- The count cap counts protected entries in the TOTAL but never deletes them, or a process holding many live sessions would silently raise its own cap.\n- Save time is stamped by the STORE (_saved_at), deliberately not in SCHEMA_VERSION: when a blob was written is a fact about storage, not about the session, and putting it in the schema would make every retention change a migration.\n- Disk falls back to file mtime when _saved_at is absent, so blobs written before this change age out normally instead of being immortal for want of a field. Redis has no per-key write time and therefore no fallback.\n- A blob whose age cannot be established is reported and LEFT IN PLACE. Reclaiming on a guess is how a reaper eats live state.\n- Wired into the existing lifespan reaper task under its own try, so a failure in one namespace cannot silently disable retention in the other.\n\nVERIFICATION. tests/test_pending_state_retention.py (12) + a manager-level wiring test in test_checkpoint_integration.py. Nine-mutation matrix, all caught on the first run: protection ignored, cap counting only candidates, oldest-first reversed, unreadable reclaimed, no mtime fallback, dry_run deleting, save not stamping, manager protecting nothing, and channel_id parsed from the filename.\n\nTEST UPDATED, NOT REMOVED: test_disk_session_state_store_roundtrip asserted exact dict equality on load, which the save stamp breaks. It now asserts every saved field survives plus the stamp, which is what it was actually for.\n\nDISCOVERED: fix-7hn (P1) -- DiskSessionStateStore._json_path is not injective, so tenant/user-1 and tenant_user-1 share one blob. Cross-session exposure, pre-existing, same defect class the checkpoint store already fixed. The reaper sidesteps it by reading channel_id from inside the blob.","status":"closed","priority":2,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-06-04T19:35:36Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-08-07T06:10:34Z","close_reason":"Implemented in v2.29.0. Abandoned suspended sessions are now reclaimed under a stated PendingRetentionPolicy by an invoked reaper, on the same terms fix-jtr set for the checkpoint namespace.","labels":["cleanup","storage","topology-b"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"fix-snl","title":"enablecache cache key collides on equal-repr / kwargs order","description":"Out of scope for speedict→sqlite migration (fix-lvl). enablecache keys with str(args)+str(kwargs), which collides for arguments with equal repr and is sensitive to kwargs ordering. Improve with a stable hash (e.g. json.dumps(args/kwargs, sort_keys=True, default=str) or hashlib) when someone next touches the decorator.","status":"open","priority":3,"issue_type":"chore","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:58:50Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:50Z","dependencies":[{"issue_id":"fix-snl","depends_on_id":"fix-lvl","type":"discovered-from","created_at":"2026-08-08T16:58:50Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"fix-lvl.8","title":"File follow-up: enablecache key collision (str args/kwargs)","description":"Out-of-scope note from design §4 Step 1: enablecache cache key is str(args)+str(kwargs), which collides for equal-repr args and is kwargs-order sensitive. Track as separate improvement; do not fix in migration.","status":"closed","priority":3,"issue_type":"chore","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T21:53:48Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T21:58:51Z","closed_at":"2026-08-08T21:58:51Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-lvl.8","depends_on_id":"fix-lvl","type":"parent-child","created_at":"2026-08-08T16:53:48Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-9nd","title":"v2.29.1 flipped the DSPy disk cache on under a CVE-floors subject","description":"NOT NECESSARILY WRONG -- filed so the decision is visible rather than buried.\n\nCommit 38ceceb's subject is 'raise CVE floors, local scanners, OpenVEX for diskcache'. It also, in fastworkflow/run_fastapi_mcp/server_memory.py:\n - flipped enable_disk_cache=False -\u003e True (capped at 1 GiB)\n - INVERTED the drift check in check_policy_in_force(), from 'disk cache re-enabled' to 'disk cache disabled'\n - removed the comment stating the disk cache was off BECAUSE DSPy's is pickle-backed and 30 GB is not a sane container default, replacing that rationale with an OpenVEX acceptance of CVE-2025-69872.\n\nThe security rationale is therefore reversed, and the drift check now asserts the opposite invariant. That may well be the right call -- 1 GiB is bounded and OpenVEX is a real answer to the pickle concern -- but a behaviour change to a server memory policy is not what the subject line describes, and it is the direct cause of the one hard test failure in the v2.29.1 run (a probe that assumed a cold cache).\n\nDO: nothing to the code unless you disagree with the flip. This exists so that anyone reading server_memory.py's history finds the decision, and so the reversal of the stated security rationale is on the record rather than only in a diff.","notes":"Accepted as-is: keep DSPy disk cache ON (1 GiB cap + OpenVEX for CVE-2025-69872). No code change.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T16:10:19Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T17:09:44Z","closed_at":"2026-08-08T17:09:44Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-k8d","title":"Deprecations that will break on the next dependency bump","description":"Surfaced by the v2.29.1 upgrade run (dspy 3.3, transformers 5.14, pandas 3.0). None is broken today; each has a named removal.\n\n1. StarletteDeprecationWarning: 'Using httpx with starlette.testclient is deprecated; install httpx2 instead'. Affects EVERY TestClient test, which is most of the FastAPI suite.\n2. FieldValidationInfo imported at fastworkflow/command_directory.py:8 -- deprecated in Pydantic V2, REMOVED in V3.\n3. ast.Str and elt.s at fastworkflow/build/documentation_generator.py:91-92 -- REMOVED in Python 3.14.\n4. tokenizers WordPiece.__init__ deprecation.\n\nWorth doing together rather than one at a time when each breaks: they are all mechanical, and the failure mode for 2 and 3 is an ImportError/AttributeError at import, which given that 'import fastworkflow' pulls the whole stack means the package stops importing at all rather than degrading.","notes":"## Completed 2026-08-08\n\n### Fixed\n1. FieldValidationInfo → ValidationInfo in fastworkflow/command_directory.py (kept).\n2. ast.Str/elt.s → ast.Constant/elt.value in fastworkflow/build/documentation_generator.py (kept).\n3. httpx2\u003e=2.0.0 added: optional main dep + server/fastapi extras + test group; poetry lock refreshed; locked httpx2==2.9.1 (+ httpcore2, truststore). Starlette TestClient httpx deprecation cleared (0 httpx-related warnings).\n\n### Won't-fix (documented)\n4. tokenizers WordPiece.__init__ deprecation: originates in upstream transformers BertTokenizer (tokenization_bert.py) when AutoTokenizer loads BERT vocab.txt. We do not construct WordPiece. No in-repo hack; wait for transformers to migrate to WordPiece.from_file.\n\n### Tests\n- test_command_directory.py + test_command_directory_cme.py + test_build/test_readme_generation.py: 5 passed (prior pass).\n- test_fastapi_service.py: 31 passed; TestClient smoke shows no Starlette httpx deprecation.","status":"closed","priority":3,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T16:10:19Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T17:11:15Z","started_at":"2026-08-08T16:55:42Z","closed_at":"2026-08-08T17:11:15Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-ylf.10","title":"Dependabot #99 torch.jit.script — already on patched 2.13.0","description":"## Finding\nDependabot alert **#99** (Low): PyTorch `torch.jit.script` memory corruption (GHSA-rrmf-rvhw-rf47 / CVE-2025-3000).\n\n## Analysis\n- Advisory: affected ≤2.12.1; **first patched 2.13.0**\n- Locked: `torch==2.13.0+cpu` (pyproject `^2.7.1`, source pytorch-cpu)\n- Local pip-audit after CVE floors does **not** report torch\n- Alert likely stale relative to current lock, or Dependabot does not treat `2.13.0+cpu` local version as satisfying 2.13.0\n\n## Recommendation\n1. After 2.29.1 poetry.lock is on default branch, wait for Dependabot re-scan.\n2. If #99 remains open, dismiss as **fixed in 2.13.0** (note +cpu local tag).\n3. No version bump required unless a newer advisory appears.\n4. Reachability: we use torch for intent-model training/inference, not typically `torch.jit.script` on untrusted models — Low severity matches.\n\n## Effort\nNone for code; dismiss-only after push.","notes":"No code change: lock already torch 2.13.0+cpu. Remaining work is Dependabot dismiss after push.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-08T14:49:23Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T14:49:25Z","closed_at":"2026-08-08T14:49:25Z","close_reason":"Closed","labels":["cve","dependencies","security"],"dependencies":[{"issue_id":"fix-ylf.10","depends_on_id":"fix-ylf","type":"parent-child","created_at":"2026-08-08T09:49:22Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -213,7 +223,7 @@ {"_type":"issue","id":"fix-6r5","title":"Persona-source row matching is outside the utterance-cache fingerprint","description":"DISCOVERED FROM fix-k0i.43 (which closed the four functions it named; this is the residue).\n\nDomainConditionedPersonaSource.rows and _matches are not digested. Editing which rows a keyword matches changes the persona pool a command is generated from, without invalidating any cache entry -- so a developer tunes the matching, retrains, and silently gets the old personas back.\n\nfix-k0i.43 widened _DIGESTED_GENERATION_SOURCES to derived_seed, select_persona_indices, resolve_personas and PersonaSource.select, which covers the SELECTION algorithm. What is still uncovered is the SOURCE's own matching behaviour.\n\nWHY IT WAS LEFT: closing it properly means digesting the active source's own methods, which makes the fingerprint source-dependent -- a different shape from the current 'digest these named module-level functions' design. That is a design decision, not an oversight.\n\nLow severity while there is effectively one source in use.","status":"closed","priority":3,"issue_type":"bug","owner":"drawal@radiantlogic.com","created_at":"2026-08-07T17:37:35Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T01:19:54Z","closed_at":"2026-08-08T01:19:54Z","close_reason":"Fixed by making the fingerprint source-dependent, but resolved from the source's CLASS rather than its instance state.\n\nThe issue framed source-dependence as 'a different shape from the current design'. On inspection that does not hold: the persona_source fingerprint input was ALREADY a function of the installed source, because active_persona_source_label() consults the process-wide source and returns name#fingerprint(). Only a third component was added: name#\u003ccontent fingerprint\u003e#\u003cpool-code digest\u003e.\n\nThe digest comes from PersonaSource.pool_source_digest(), which hashes the source text of every function the concrete class and its bases define, found BY REFLECTION. Reflection rather than a name list is the whole point -- a name list is exactly what left _matches and rows out in the first place: fix-k0i.43 named four functions, two more existed, and nothing failed. Adding DomainConditionedPersonaSource._matches to _DIGESTED_GENERATION_SOURCES was rejected for that reason, and because it would make EVERY workflow regenerate over a keyword filter only a workflow with personas.json can reach.\n\nDeterminism verified independently rather than argued: the digest is identical across PYTHONHASHSEED 0/12345/99999 in fresh subprocesses, and is computable from an uninitialised instance -- confirming it reads only type(self), so no instance attribute and no id() can reach it. Ordered by MRO then attribute name so neither __dict__ order nor the hash salt can move it. Properties are unwrapped to .fget and staticmethod/classmethod to .__func__, because inspect.getsource rejects the descriptors and source_digest would otherwise have quietly recorded 'source-unavailable' -- a digest that stops noticing edits while still looking like one.\n\nInvalidation is scoped: utterance-cache entries are invalidated only for workflows WITH a personas.json. A workflow without one keeps its exact existing key, pinned by a test asserting its persona_source input still has no '#'. The default PersonaHub draw keeps label None per decision D6.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-3xf","title":"Empty channel_id reaches the storage layer and now 500s instead of 400ing","description":"DISCOVERED FROM fix-7hn. encode_path_component('') raises ValueError, so a request carrying channel_id='' now fails inside the storage layer rather than at the API boundary.\n\nThis is strictly better than what it replaced -- the old _json_path('') returned '_pending.json', ONE SHARED FILE for every empty id, which is the same cross-session defect class fix-7hn just closed. Failing loudly beats silently sharing a bucket.\n\nBut it fails in the wrong place. No HTTP handler in run_fastapi_mcp validates that channel_id is non-empty, so the caller sees a 500 (an internal error, implying our bug) where they should see a 400 (their malformed request). It also means the error text is a storage-layer message rather than something an API consumer can act on.\n\nFIX: validate channel_id at the API boundary -- non-empty, and probably a length cap since the encoder hashes beyond the filesystem name limit. The Pydantic request models are the natural home. Keep the ValueError as the backstop; the point is that it should be unreachable from a well-formed request.\n\nLow severity: reachable only by a client sending an empty channel_id, which no normal client does.","status":"closed","priority":3,"issue_type":"bug","owner":"drawal@radiantlogic.com","created_at":"2026-08-07T16:56:15Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T04:17:22Z","closed_at":"2026-08-08T04:17:22Z","close_reason":"Fixed at the boundary with ChannelId = Annotated[str, Field(min_length=1)] on InitializationRequest and GenerateMCPTokenRequest -- the two places a client-supplied channel id enters. SessionData deliberately left alone: its channel_id comes from a JWT this server minted, so constraining it would turn a bad-but-authentic token into a 500 instead of the current 401.\n\nLENGTH CAP DELIBERATELY SKIPPED, with reasoning rather than omission: the encoder's 200-byte ceiling is on the ENCODED name, which percent-escaping can triple, so any raw-length number derived from it would reject ordinary ids to prevent nothing -- and oversized ids are a supported hash-and-recheck path with a passing test at 400 chars. No principled limit exists, and that is recorded in the comment rather than a number being invented.\n\nThe mutation surfaced a SECOND defect the issue did not name: with min_length removed, /admin/generate_mcp_token returned 200 and happily minted a token for an empty channel id that would then 500 on first use. That endpoint had no validation at all.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-xm1","title":"reap() can report reclaiming a pending blob it did not remove","description":"DISCOVERED FROM fix-7hn (pre-existing; not introduced by it).\n\nSessionStateStore.reap() enumerates entries via iter_entries(), which reads channel_id from INSIDE each blob (deliberately -- the storage key is not reversible). It then calls clear(channel_id), which derives the path FROM that id.\n\nIf a blob's embedded channel_id does not map back to its own filename, clear() removes nothing and reclaimed is incremented anyway. Reachable when a blob was hand-copied, restored from a backup under a different name, or written by an older tool.\n\nConsequence: the reaper reports reclaiming N and the namespace does not shrink by N, and the offending blob is immortal -- enumerated forever, never removed. An operator watching the metric would conclude retention is working.\n\nFIX OPTIONS: (a) have iter_entries yield the storage path alongside the channel_id and have reap unlink the path it actually enumerated; (b) verify after clear() that the entry is gone and count only confirmed removals, logging the mismatch. (a) is more direct and also removes reap's dependency on the blob's contents being self-consistent.\n\nSeverity low: reclaimed is a metric rather than a guarantee, and the situation requires a blob that did not come from this store. fix-7hn's two-path clear() makes it marginally less likely by also trying the legacy name.","status":"closed","priority":3,"issue_type":"bug","owner":"drawal@radiantlogic.com","created_at":"2026-08-07T16:56:15Z","created_by":"Dhar Rawal","updated_at":"2026-08-08T01:17:05Z","closed_at":"2026-08-08T01:17:05Z","close_reason":"Fixed with option (a). iter_entries now yields PendingEntry(channel_id, saved_at, storage_key) and reap removes entry.storage_key through a new remove_at(storage_key) -\u003e bool. Selection is still by channel_id -- that is what protected_channel_ids is expressed in and the only thing comparable against it -- but REMOVAL no longer re-derives a path from a blob's own claim about which channel it belongs to, which was the whole defect.\n\nreclaimed now counts CONFIRMED removals on a real pass (a dry run still projects len(doomed)), and PendingReapOutcome gained a failures field for blobs selected but refused, with a per-blob warning naming the path.\n\nThe backends stay consistent by construction: remove_at is ABSTRACT on SessionStateStore, so neither can inherit channel-keyed removal. Verified independently -- remove_at is in __abstractmethods__ and both classes override it. Both implementations also refuse a key from outside their own namespace, and each backend's clear() now routes through remove_at, so there is one unlink per backend rather than two paths that can drift.\n\nTwo improvements fell out: the reaper can no longer take a live blob along with a stale one (the old clear-based removal unlinked both the current and legacy names), and the count cap de-duplicates per BLOB rather than per channel, so two blobs claiming one channel each count and each get removed.\n\n6 mutations, all detected. The load-bearing test hand-writes a blob at restored-from-backup.pending.json whose embedded id derives neither that name nor its legacy form, and asserts the file is gone, reclaimed == 1, failures == 0, and the namespace no longer enumerates it. One reclaimed mutation initially SLIPPED THROUGH and drove a real extra test where a competing pass removes the blobs first.\n\nSTILL IMMORTAL, deliberately: a blob with a readable _saved_at but no channel_id is counted unreadable and left. Removal by storage key would now make reaping it possible, but protection is keyed by channel id and a blob that cannot be attributed cannot be checked against the protected set. Right call -- the metric does not lie about it either way.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"fix-g03.25","title":"Follow-up: complete logical-turn and CME continuation serialization (unpins awaiting sessions; needed for multi-pod)","description":"Sections 11.6 and 19.11; decision 26 [R2-8, R2-10 absorption]. NOT required for Release B v1 - v1 pins awaiting and continuation sessions instead.\n\nThe existing pending snapshot stores suspension flags, ReAct state, NLU stage, action log and conversation turns. It does NOT store the WEC logical-turn accumulator initialized at workflow_execution_context.py:104-114: _turn_outputs, _turn_key, _turn_started_at, the original and refined message, suspended duration and suspension start, entry workflow and context, and the agent result. Resume deliberately skips _begin_turn() (:547-549) precisely because it expects the old accumulator, so after rehydration the logical turn takes a NEW FALLBACK KEY and loses pre-suspension command outputs, ask-user entry, artifacts and timing.\n\nThe deterministic CME continuation is also incomplete: serialize_state() stores only nlu_stage, while clarification and parameter extraction depend on the CME context keys command, command_name and stored_parameters (fastworkflow/_workflows/command_metadata_extraction/_commands/wildcard.py:45, :141-144; parameter_extraction.py:72, :155-165). Workflow.end_command_processing() deletes command and stored_parameters and resets the stage (workflow.py:291-303), so a mid-extraction snapshot that omits them cannot be distinguished from a completed one after restore.\n\nThis is a PRE-EXISTING defect in today's suspended-state restore, not something Release B introduces.\n\nWhy it still matters with pinning [R2-10 absorption]: pinning removes the controlled-eviction half of the two-record consistency problem but not the crash half, and it does NOT help multi-pod - another pod still cold-rehydrates from the store, so the incomplete restore remains reachable. Redis-backed multi-pod deployments need this completeness work before they can rely on suspension across pods.","acceptance_criteria":"- Exact TurnOutput equivalence across suspend -\u003e evict -\u003e rehydrate -\u003e resume.\n- missing-parameter -\u003e evict -\u003e answer completes correctly.\n- Only once both pass may the section 11.2 eligibility clause be relaxed so awaiting/continuation sessions become evictable.","notes":"IMPLEMENTED v2.28.0.\n\nWHAT THE ISSUE GOT RIGHT AND WHAT IT MISSED. The listed gaps were all real: serialize_state carried neither the accumulator (_turn_outputs, _turn_key, timings, entry workflow/context, agent result) nor the CME keys (command, command_name, stored_parameters), only nlu_stage. What the issue framed as a multi-pod/crash concern was also a live memory bound: checkpoint.assess() pinned every awaiting session, decision 6 declines an idle TTL, and only the user returning or /cancel_pending clears it -- so an abandoned ask_user held a runtime for the life of the process. Release C lowering the cap 2000 -\u003e 50 made that ~40x more reachable. The state was never unsaved; the blob is written at the end of the suspending turn. It was written lossily.\n\nIMPLEMENTATION.\n- WEC._serialize_turn_accumulator / _apply_turn_accumulator. CommandOutput list via model_dump(mode=json) / model_validate; datetimes ISO. _turn_agent_result is distilled to {exhausted: bool} because the finalize path reads only that attribute plus whether it is None -- it is a dspy Prediction that no strict encoder would accept whole.\n- WEC._serialize_cme_continuation / _apply_cme_continuation. stored_parameters round-trips through the command Input class looked up from command_name via RoutingRegistry, rebuilt with model_construct (NOT model_validate: the saved instance was itself model_construct-ed and holds NOT_FOUND sentinels in typed fields, e.g. a str in an int field, which is exactly what validation rejects). An unresolvable class resets to intent detection with a warning rather than stranding the session at PARAMETER_EXTRACTION, where wildcard.py reads context[command_name] unconditionally.\n- has_open_command() deliberately does NOT key off command_name: end_command_processing() clears command and stored_parameters but LEAVES command_name, so keying off it marks every session that ever ran a command as mid-extraction forever. Found by a test, not by reading.\n- Both persist writers (turns._persist_after_turn and utils.persist_pending_after_turn) now save when has_open_command(), not only when awaiting. Mid-extraction sessions are not awaiting_user, so the old writers CLEARED their blob -- a pre-existing silent loss of partially extracted parameters on eviction.\n- checkpoint.assess(runtime, store) replaces the awaiting_user pin with: pin iff (awaiting or mid-extraction) AND the blob is not in the store. Omitting the store pins, which is the safe reading for a caller that cannot check.\n- SCHEMA_VERSION 1 -\u003e 2. A v1 blob is refused by the fail-closed guard from fix-4od rather than migrated: it is a suspended turn minutes old, and the fields it lacks are the ones that made restoring it wrong.\n\nVERIFICATION. tests/test_turn_and_cme_continuation.py (10) + 4 new in tests/test_checkpoint_integration.py. 8-mutation matrix, all caught; two survived the first run and drove real test additions -- a duplicated policy tested in only one of its two copies, and a model_construct guard tested with a value that validated either way. Behaviour was checked differentially against the live path (same sequence with and without a restore in the middle produces identical output) rather than against an idealised expectation.\n\nTEST CHANGED, NOT REMOVED: test_a_suspended_channel_is_never_evicted asserted decision 26 and is now test_a_suspended_channel_is_evicted_now_that_its_snapshot_is_complete, with the pin half preserved in test_suspended_session_pins_when_its_state_never_reached_the_store.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-05T22:00:40Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-08-07T02:33:52Z","close_reason":"Implemented in v2.28.0. Schema 2 of the pending blob now carries the logical-turn accumulator and the CME continuation keys, and the unconditional awaiting_user pin is gone.","labels":["follow-up","memory-bounds","release-b"],"dependencies":[{"issue_id":"fix-g03.25","depends_on_id":"fix-g03.17","type":"blocks","created_at":"2026-08-05T17:03:54Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-g03.25","depends_on_id":"fix-g03","type":"parent-child","created_at":"2026-08-05T17:00:39Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"fix-g03.25","title":"Follow-up: complete logical-turn and CME continuation serialization (unpins awaiting sessions; needed for multi-pod)","description":"Sections 11.6 and 19.11; decision 26 [R2-8, R2-10 absorption]. NOT required for Release B v1 - v1 pins awaiting and continuation sessions instead.\n\nThe existing pending snapshot stores suspension flags, ReAct state, NLU stage, action log and conversation turns. It does NOT store the WEC logical-turn accumulator initialized at workflow_execution_context.py:104-114: _turn_outputs, _turn_key, _turn_started_at, the original and refined message, suspended duration and suspension start, entry workflow and context, and the agent result. Resume deliberately skips _begin_turn() (:547-549) precisely because it expects the old accumulator, so after rehydration the logical turn takes a NEW FALLBACK KEY and loses pre-suspension command outputs, ask-user entry, artifacts and timing.\n\nThe deterministic CME continuation is also incomplete: serialize_state() stores only nlu_stage, while clarification and parameter extraction depend on the CME context keys command, command_name and stored_parameters (fastworkflow/_workflows/command_metadata_extraction/_commands/wildcard.py:45, :141-144; parameter_extraction.py:72, :155-165). Workflow.end_command_processing() deletes command and stored_parameters and resets the stage (workflow.py:291-303), so a mid-extraction snapshot that omits them cannot be distinguished from a completed one after restore.\n\nThis is a PRE-EXISTING defect in today's suspended-state restore, not something Release B introduces.\n\nWhy it still matters with pinning [R2-10 absorption]: pinning removes the controlled-eviction half of the two-record consistency problem but not the crash half, and it does NOT help multi-pod - another pod still cold-rehydrates from the store, so the incomplete restore remains reachable. Redis-backed multi-pod deployments need this completeness work before they can rely on suspension across pods.","acceptance_criteria":"- Exact TurnOutput equivalence across suspend -\u003e evict -\u003e rehydrate -\u003e resume.\n- missing-parameter -\u003e evict -\u003e answer completes correctly.\n- Only once both pass may the section 11.2 eligibility clause be relaxed so awaiting/continuation sessions become evictable.","notes":"IMPLEMENTED v2.28.0.\n\nWHAT THE ISSUE GOT RIGHT AND WHAT IT MISSED. The listed gaps were all real: serialize_state carried neither the accumulator (_turn_outputs, _turn_key, timings, entry workflow/context, agent result) nor the CME keys (command, command_name, stored_parameters), only nlu_stage. What the issue framed as a multi-pod/crash concern was also a live memory bound: checkpoint.assess() pinned every awaiting session, decision 6 declines an idle TTL, and only the user returning or /cancel_pending clears it -- so an abandoned ask_user held a runtime for the life of the process. Release C lowering the cap 2000 -\u003e 50 made that ~40x more reachable. The state was never unsaved; the blob is written at the end of the suspending turn. It was written lossily.\n\nIMPLEMENTATION.\n- WEC._serialize_turn_accumulator / _apply_turn_accumulator. CommandOutput list via model_dump(mode=json) / model_validate; datetimes ISO. _turn_agent_result is distilled to {exhausted: bool} because the finalize path reads only that attribute plus whether it is None -- it is a dspy Prediction that no strict encoder would accept whole.\n- WEC._serialize_cme_continuation / _apply_cme_continuation. stored_parameters round-trips through the command Input class looked up from command_name via RoutingRegistry, rebuilt with model_construct (NOT model_validate: the saved instance was itself model_construct-ed and holds NOT_FOUND sentinels in typed fields, e.g. a str in an int field, which is exactly what validation rejects). An unresolvable class resets to intent detection with a warning rather than stranding the session at PARAMETER_EXTRACTION, where wildcard.py reads context[command_name] unconditionally.\n- has_open_command() deliberately does NOT key off command_name: end_command_processing() clears command and stored_parameters but LEAVES command_name, so keying off it marks every session that ever ran a command as mid-extraction forever. Found by a test, not by reading.\n- Both persist writers (turns._persist_after_turn and utils.persist_pending_after_turn) now save when has_open_command(), not only when awaiting. Mid-extraction sessions are not awaiting_user, so the old writers CLEARED their blob -- a pre-existing silent loss of partially extracted parameters on eviction.\n- checkpoint.assess(runtime, store) replaces the awaiting_user pin with: pin iff (awaiting or mid-extraction) AND the blob is not in the store. Omitting the store pins, which is the safe reading for a caller that cannot check.\n- SCHEMA_VERSION 1 -\u003e 2. A v1 blob is refused by the fail-closed guard from fix-4od rather than migrated: it is a suspended turn minutes old, and the fields it lacks are the ones that made restoring it wrong.\n\nVERIFICATION. tests/test_turn_and_cme_continuation.py (10) + 4 new in tests/test_checkpoint_integration.py. 8-mutation matrix, all caught; two survived the first run and drove real test additions -- a duplicated policy tested in only one of its two copies, and a model_construct guard tested with a value that validated either way. Behaviour was checked differentially against the live path (same sequence with and without a restore in the middle produces identical output) rather than against an idealised expectation.\n\nTEST CHANGED, NOT REMOVED: test_a_suspended_channel_is_never_evicted asserted decision 26 and is now test_a_suspended_channel_is_evicted_now_that_its_snapshot_is_complete, with the pin half preserved in test_suspended_session_pins_when_its_state_never_reached_the_store.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-05T22:00:40Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","closed_at":"2026-08-07T02:33:52Z","close_reason":"Implemented in v2.28.0. Schema 2 of the pending blob now carries the logical-turn accumulator and the CME continuation keys, and the unconditional awaiting_user pin is gone.","labels":["follow-up","memory-bounds","release-b"],"dependencies":[{"issue_id":"fix-g03.25","depends_on_id":"fix-g03","type":"parent-child","created_at":"2026-08-05T17:00:39Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-g03.25","depends_on_id":"fix-g03.17","type":"blocks","created_at":"2026-08-05T17:03:54Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-k0i.50","title":"S3: dead-code sweep across the new modules (~350 lines beyond the decided deletions)","description":"Everything below has zero production callers (grep-verified by reviewers) and is NOT covered by the decided deletions (fix-k0i.13/.19/.22): artifact_versioning.py format_versions_table/describe_version/human_age/human_duration/human_size (~115 lines, CLI-era) plus VersionInfo.size_bytes/_dir_size_bytes which run a full tree walk of EVERY version on each list_versions call including the one inside retain_current_and_previous on every train (wasted I/O); prune_versions keep= branch (~15); selective_training.py only_commands/only_contexts + descendants_of speculative API (~50) and the _resolve_cache_mode/_MODE_REUSE mirror unreachable from __main__ (~30); utterance_cache.py normalize_mode/_unknown_mode_warned/resolve_cache_mode (~35 — modes only arrive as internal constants); generate_synthetic.select_persona_indices production-dead duplicate of PersonaHubSource.select (~20, kept as test oracle — decide); personas.py num_personas_hint plumbing (~15, always 0 — resolved by fix-k0i.38 either way); generate_param_examples.py commented-out old transform block (:483-528, -46), stale regex-era comments in the AST loop (~10), merged duplicate required/optional invalid-params loops (-12), dead 'command in locals()' check (-2), save_examples_to_file/save_examples_to_json dead pair (-26), raw print() debugging to logger.debug (~8 sites); training_report.py get_min_training_rows/get_min_seed_utterances constant wrappers (~15). Rule: for each item either delete or wire — do not keep dead code certified by live tests. Split into per-module commits; no behavior change, full suite green after each.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-03T20:23:19Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T20:44:26Z","closed_at":"2026-08-07T20:44:26Z","close_reason":"Resolved, with two of the issue's own candidates found to be wrong. VERIFIED INDEPENDENTLY: format_versions_table's only two 'production' references are COMMENTS, not calls -- it, describe_version, human_age, human_size and human_duration (~130 lines) had zero production callers and were reachable only from their own tests. Deleted with user approval, along with their five tests; the R4 provenance (what they were, why they existed, and that VersionInfo.size_bytes should be made lazy per fix-44d before any replacement) is preserved in the artifact_versioning module docstring, and the two comments that referenced the deleted function were corrected rather than left dangling. NOT DELETED, because the issue was wrong about them: get_min_training_rows has a real caller at training_report.py:715, and only_commands/only_contexts are documented in-code at selective_training.py:843 as a deliberate programmatic affordance. COLLISION NOTE: fix-k0i.32 had just corrected the retention footer INSIDE format_versions_table and a test pinned it; deletion supersedes that fix, so the test was removed with a note pointing at the pruning tests that assert the policy for real.","dependencies":[{"issue_id":"fix-k0i.50","depends_on_id":"fix-k0i","type":"parent-child","created_at":"2026-08-03T15:23:19Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-k0i.49","title":"S3: AR2 base+parent compose case covered only by hand-built dicts — build the sibling fixture the spec prescribes","description":"test_base_and_parent_axes_compose (tests/test_selective_training.py:83-112) exercises the shipped close_dirty_contexts but with hand-built context maps; no real-workflow reproduction exists — messaging_app_4 has no grandchild, and the spec's prescribed reproduction (AR2: 'give TodoList a second child that does not inherit from TodoItem' — the sibling whose wildcard class fills with TodoItem/* utterances) was never built. A regression in how the PLANNER DERIVES context_commands/context_ancestors for the compose case (e.g. Carriers from the raw base graph instead of resolved commands() — the exact anti-decision AR2 names) is invisible to the pure test and unreachable by the shipped fixtures. DO: add the sibling context to a copy-based test workflow (or extend tests/duplicate_capability_workflow) and drive compute_training_plan end-to-end: edit the base-inherited command, assert the sibling is planned dirty.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-03T20:22:55Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T20:44:26Z","closed_at":"2026-08-07T20:44:26Z","close_reason":"Fixed. Built the grandchild the spec prescribes on a copy-based workflow: a PremiumSession context whose parent is PremiumUser, which inherits User's commands through base -- so base and parent axes genuinely compose rather than being simulated with hand-built dicts. Four tests, including one verifying the fixture really has both axes and one driving compute_training_plan end to end. Mutation (closing the wildcard axis over the first ancestor only) failed exactly one test -- the new grandchild test -- while the pure-data test_base_and_parent_axes_compose passed. That isolates the gap the finding named.","dependencies":[{"issue_id":"fix-k0i.49","depends_on_id":"fix-k0i","type":"parent-child","created_at":"2026-08-03T15:22:55Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-k0i.48","title":"S3: test hygiene bundle — cleanup not in finally, dead assertions, stale docstrings, brittle --help substring","description":"Four small pins from the review: (a) _cleanup(...) at the end of both new standalone train tests (test_train_modern_stack.py:363,441) is not in try/finally — a failure leaks ./___workflow_contexts into the repo root; (b) tests/test_heldout_evaluation.py:741,754: 'assert_benchmark_disjoint_from_seeds(cases, seeds) is None' is a bare expression, the is-None comparison is dead (semi-harmless: a raise still fails the test) — wrap in assert or drop the comparison; (c) test_selective_training_integration.py:759 docstring says '(relative path, size, mtime)' but the code hashes sha256 — fix the doc, the code is right; (d) test_duplicate_detection.py:533: _run_cli('--help') asserting 'report' not in stdout breaks the moment any unrelated help text contains that substring — assert on the subcommand list structure instead.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-03T20:22:35Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T20:44:26Z","closed_at":"2026-08-07T20:44:26Z","close_reason":"Fixed all four. Cleanup moved into try/finally in both standalone train tests and the trained_hello_world fixture; the two dead '... is None' bare expressions became real assertions; the stale '(relative path, size, mtime)' docstring now says sha256 and explains why stat-based signatures cannot work with carry-forward hardlinks; the --help test parses argparse's subcommand list instead of grepping prose. Mutations: returning overlaps from assert_benchmark_disjoint_from_seeds failed both previously-dead assertions, and adding a real 'report' subcommand failed the new CLI test while putting 'report' in an unrelated subcommand's help text would have failed the OLD test spuriously and passes the new one. HONEST NEGATIVE on 48(a): with try/finally removed and a failure injected, the repo root stayed clean anyway -- the new fix-k0i.16 preflight now aborts before the workflow is created, so nothing in a hello_world train writes ./___workflow_contexts. The try/finally is still correct and does guarantee cleanup on failure, but the specific leak this item names is no longer reproducible.","dependencies":[{"issue_id":"fix-k0i.48","depends_on_id":"fix-k0i","type":"parent-child","created_at":"2026-08-03T15:22:34Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -234,7 +244,7 @@ {"_type":"issue","id":"fix-k0i.33","title":"S3: three silent-failure suppressions — heldout write, provenance save, corrupt manifest pruning the recovery version","description":"(1) model_pipeline_training.py:1622: write_report wrapped in contextlib.suppress(OSError) with NO log — a failed write leaves heldout_evaluation.json stale/absent while the run claims success, and the selective merge (train/__main__.py:268) then merges against the wrong baseline; capture_heldout_evaluation silently returns None next run. (2) train/__main__.py:250-256: recorder.save() equally suppressed — the publish gate at :284 then evaluates the PREVIOUS run's training_provenance.json, passing or failing on data describing a different run. (3) artifact_versioning read_manifest returns {} for unreadable JSON (:514-525), so on the no-op path previous_previous=None and retain_current_and_previous prunes the ACTUAL previous successful version — implicit destruction triggered by a damaged manifest.json, against R4's never-destroy-implicitly rule. DO: (1)+(2) log loudly at ERROR and, for (2), refuse the publish gate on a failed save (the gate's input is gone); (3) treat an unreadable manifest as fatal for retention (skip pruning, warn), never as 'no previous'.","status":"closed","priority":3,"issue_type":"bug","owner":"drawal@radiantlogic.com","created_at":"2026-08-03T20:17:40Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T20:24:19Z","closed_at":"2026-08-07T20:24:19Z","close_reason":"Fixed all three. (1) The heldout report write moved into _write_heldout_report, which logs at ERROR and prints a WARNING naming the consequence -- the report is stale, so the NEXT selective run merges against the wrong baseline. (2) The provenance save moved into _save_run_provenance and a failed recorder.save() now RAISES TrainingDataError instead of being suppressed, because the publication gate reads that file and would otherwise pass or fail on the previous run's records. The version-internal copy failing stays non-fatal (the models are worth publishing) but logs what the version can no longer report. (3) read_manifest logs at ERROR, and a new manifest_is_damaged distinguishes CORRUPT from ABSENT so retain_current_and_previous refuses to prune when the current version's manifest is damaged -- previously a corrupt manifest read as 'no versions yet' and the recovery point was pruned. 8 tests provoking real filesystem failures; 4 mutations, all detected, plus a companion test that retention still prunes normally so 'keep everything' cannot pass.","dependencies":[{"issue_id":"fix-k0i.33","depends_on_id":"fix-k0i","type":"parent-child","created_at":"2026-08-03T15:17:39Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-k0i.32","title":"S3: code comments and docs reference CLI surface that was cut — rollback/prune commands, benchmark flag, rejected_examples.json dump","description":"Four doc-rot spots pointing users at tooling the 2026-08-03 UX decision deleted: (1) train/__main__.py:271-272 'rolling back with versions publish \u003cold\u003e' — no such command; a developer with a bad model has no rollback instruction that works (the actual path is artifact_versioning.publish_version via Python). (2) artifact_versioning.py:1029-1031 'use versions prune with an explicit request'. (3) docs/intent_benchmark_format.md:17-19 'The orchestrator should expose a CLI flag to point at a different path' — no flag exists; the fixed default path (\u003cworkflow\u003e/intent_benchmark.json) is the contract. (4) generate_param_examples.py:754-757 stale comment describing the deleted rejected_examples.json debugging dump (removal is pinned by a test). Also selective_training.py:438-442 build_context_maps docstring claims core commands are 'deliberately absent from an ancestor's contribution' while :463 unions core_commands into every context — conservative in effect, wrong as documentation. DO: one sweep correcting all five; state the real rollback procedure where the fake command was cited.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-03T20:17:22Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T20:24:22Z","closed_at":"2026-08-07T20:24:22Z","close_reason":"Fixed the four spots in the training modules: the 'versions publish \u003cold\u003e' rollback comment in train/__main__.py now states the real procedure (artifact_versioning.publish_version against an id from list_versions), plus the 'versions prune' footer in format_versions_table, describe_version's 'for versions show', and migrate_legacy_to_version's 'or a versions list'. The footer was wrong twice over -- there is no versions CLI, AND 'nothing removes them implicitly' denied the automatic retention that runs on every train. No comments were deleted; each was replaced with an accurate one. Mutation restoring the old footer: detected. Remaining spots are in files this workstream did not own: docs/intent_benchmark_format.md (a CLI flag that does not exist), generate_param_examples.py:754 (the deleted rejected_examples.json dump) and selective_training.py:438 (a build_context_maps docstring contradicting line 463).","dependencies":[{"issue_id":"fix-k0i.32","depends_on_id":"fix-k0i","type":"parent-child","created_at":"2026-08-03T15:17:22Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-k0i.31","title":"S3: stale fix-eia claims in convergence skill (self-contradiction) and command-authoring rule","description":"CONFIRMED. .claude/skills/fastworkflow-intent-training-convergence/SKILL.md contradicts itself: lines 94-101 correctly state the benchmark validators now run (fix-eia FIXED, disjointness raises BenchmarkLeakError before training), but Precondition 1 (:136-141) and Pitfalls (:357-360) still instruct 'The package ships the check but does not yet run it (fix-eia), so enforce it yourself'. .claude/rules/command-authoring.md repeats the stale claim ('as of 2026-08-02 nothing calls it during training (fix-eia), so run it yourself'). fix-eia is closed with call sites at model_pipeline_training.py:887-913 (per its close note; current lines ~996-1043). A reader either wastes effort hand-running the check or distrusts a protection that exists. DO: update both docs to the fixed state; while there, remove scripts/__pycache__/*.pyc from the skill dir (team-private dir hygiene). These are team-private/untracked docs — no commit involved (Rule 1 untouched).","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-03T20:17:02Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T16:25:25Z","closed_at":"2026-08-07T16:25:25Z","close_reason":"Fixed. The 'not called during training' half was the stale one: model_pipeline_training.py calls assert_benchmark_disjoint_from_seeds (deliberately uncaught) and find_near_duplicate_benchmark_cases / validate_routing_cases / validate_escalation_cases before the training loop begins. Corrected three spots in the convergence skill (Precondition 1, Precondition 3, Pitfalls) and one in .claude/rules/command-authoring.md, keeping the still-true advice to run the check yourself in CI for an earlier signal. Also removed stale untracked bytecode from the skill's scripts dir, and fixed the 'Five gaps' heading that sat above seven bullets.","dependencies":[{"issue_id":"fix-k0i.31","depends_on_id":"fix-k0i","type":"parent-child","created_at":"2026-08-03T15:17:02Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"fix-551.17","title":"R9b: detect and report near-duplicate command capabilities","description":"ADDRESSES F14 (S3). Some workflows expose the same capability twice — on IDO, ControlsMonitor/list_findings and Directory/search_control_findings answer the same question. No amount of utterance engineering separates them. They present as PERMANENT BENCHMARK FAILURES and send developers chasing an unfixable defect.\n\nDO (spec R9): after training, cluster label centroids and warn on near-duplicate commands, NAMING THE PAIR so the developer can merge, alias, or accept them.\n\nThe value here is diagnostic honesty: turning an unfixable-looking accuracy failure into a design observation the developer can act on.","notes":"R9b DELIVERED (reporting-only; no integration into the training path, by design). Near-duplicate defined operationally as a property of the TRAINING DATA rather than of the commands' meanings: two commands are near-duplicates when a classifier restricted to that pair, trained on their own utterances, cannot separate them. Implemented as leave-one-out balanced nearest-centroid accuracy in TF-IDF space with document frequency computed across ALL commands, which is what suppresses false positives on command families sharing boilerplate. Chance is 0.5 by construction, so the threshold is the coin-flip line, not a fitted constant. Evidence: on retail (19 commands, 79 pairs) zero duplicates and one low-severity overlap; minimum separability 0.56 and median 1.00 across 171 scored pairs. The positive-control workflow's deliberately duplicated pair scores 0.00 and is flagged; the hard negative in that workflow is not. Documented blind spot: it is a lexical instrument, so a duplicate pair with genuinely disjoint vocabulary is invisible -- covered instead by find_confusable_commands, which asks the trained router what it actually does via a predict_fn.\nR9b SHIPPED, and the missing piece was the call site. duplicate_detection.py was complete and unit-tested but had NO caller anywhere in fastworkflow/ -- a developer had no way to run it, which is indistinguishable from the feature not existing. Now wired as `fastworkflow duplicates \u003cworkflow\u003e [--json] [--fail-on-duplicates]`.\n\nWiring it surfaced two defects no unit test could have caught, which is the argument for insisting on call sites: (1) the scan needs fastworkflow.init() before the routing definition resolves -- it failed with \"'NoneType' object has no attribute 'get_definition'\"; (2) find_confusable_commands() requires a trained model's predict_fn, so it CANNOT run in a pre-training scan at all. The command therefore does the lexical scan only, which is the half that works before you have spent an afternoon training two commands no data separates.\n\n--fail-on-duplicates gates on outright duplicates only. Overlapping and confusable pairs are frequently correct -- two commands SHOULD look similar when they do similar things -- and failing on them would teach developers to bypass the check. Verified discriminating: exits 1 on tests/duplicate_capability_workflow, exits 0 on retail (which has an overlapping pair and no duplicate). 5 CLI integration tests run the real binary in a subprocess, because the failure being guarded against is that the entry point is unreachable -- a test importing the function would have passed all through the period the command did not exist.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-02T14:38:12Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-08-02T17:15:04Z","closed_at":"2026-08-02T20:46:32Z","close_reason":"Shipped and integrated; see notes","dependencies":[{"issue_id":"fix-551.17","depends_on_id":"fix-551.5","type":"blocks","created_at":"2026-08-02T09:38:34Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-551.17","depends_on_id":"fix-551","type":"parent-child","created_at":"2026-08-02T09:38:11Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"fix-551.17","title":"R9b: detect and report near-duplicate command capabilities","description":"ADDRESSES F14 (S3). Some workflows expose the same capability twice — on IDO, ControlsMonitor/list_findings and Directory/search_control_findings answer the same question. No amount of utterance engineering separates them. They present as PERMANENT BENCHMARK FAILURES and send developers chasing an unfixable defect.\n\nDO (spec R9): after training, cluster label centroids and warn on near-duplicate commands, NAMING THE PAIR so the developer can merge, alias, or accept them.\n\nThe value here is diagnostic honesty: turning an unfixable-looking accuracy failure into a design observation the developer can act on.","notes":"R9b DELIVERED (reporting-only; no integration into the training path, by design). Near-duplicate defined operationally as a property of the TRAINING DATA rather than of the commands' meanings: two commands are near-duplicates when a classifier restricted to that pair, trained on their own utterances, cannot separate them. Implemented as leave-one-out balanced nearest-centroid accuracy in TF-IDF space with document frequency computed across ALL commands, which is what suppresses false positives on command families sharing boilerplate. Chance is 0.5 by construction, so the threshold is the coin-flip line, not a fitted constant. Evidence: on retail (19 commands, 79 pairs) zero duplicates and one low-severity overlap; minimum separability 0.56 and median 1.00 across 171 scored pairs. The positive-control workflow's deliberately duplicated pair scores 0.00 and is flagged; the hard negative in that workflow is not. Documented blind spot: it is a lexical instrument, so a duplicate pair with genuinely disjoint vocabulary is invisible -- covered instead by find_confusable_commands, which asks the trained router what it actually does via a predict_fn.\nR9b SHIPPED, and the missing piece was the call site. duplicate_detection.py was complete and unit-tested but had NO caller anywhere in fastworkflow/ -- a developer had no way to run it, which is indistinguishable from the feature not existing. Now wired as `fastworkflow duplicates \u003cworkflow\u003e [--json] [--fail-on-duplicates]`.\n\nWiring it surfaced two defects no unit test could have caught, which is the argument for insisting on call sites: (1) the scan needs fastworkflow.init() before the routing definition resolves -- it failed with \"'NoneType' object has no attribute 'get_definition'\"; (2) find_confusable_commands() requires a trained model's predict_fn, so it CANNOT run in a pre-training scan at all. The command therefore does the lexical scan only, which is the half that works before you have spent an afternoon training two commands no data separates.\n\n--fail-on-duplicates gates on outright duplicates only. Overlapping and confusable pairs are frequently correct -- two commands SHOULD look similar when they do similar things -- and failing on them would teach developers to bypass the check. Verified discriminating: exits 1 on tests/duplicate_capability_workflow, exits 0 on retail (which has an overlapping pair and no duplicate). 5 CLI integration tests run the real binary in a subprocess, because the failure being guarded against is that the entry point is unreachable -- a test importing the function would have passed all through the period the command did not exist.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-02T14:38:12Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-08-02T17:15:04Z","closed_at":"2026-08-02T20:46:32Z","close_reason":"Shipped and integrated; see notes","dependencies":[{"issue_id":"fix-551.17","depends_on_id":"fix-551","type":"parent-child","created_at":"2026-08-02T09:38:11Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-551.17","depends_on_id":"fix-551.5","type":"blocks","created_at":"2026-08-02T09:38:34Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-551.8","title":"F11: shared encoder with per-context heads — cut 276 MB/context","description":"ADDRESSES F11 (S3). Each context directory holds a full DistilBERT copy PLUS a BERT-tiny (largemodel.pth, tinymodel.pth, two threshold files, a label encoder) — roughly 276 MB per context. 33 contexts -\u003e 8.6 GB for a 160-command workflow.\n\nA shared encoder with per-context classification heads would cut this by roughly two orders of magnitude, and would make selective retraining cheap: rewriting one context would touch a few MB of head weights instead of 276 MB.\n\nDO:\n- Evaluate and, if it holds up, implement a shared-encoder / per-context-head architecture.\n- Measure the accuracy consequence on BOTH axes (R1) with a paired test — a shared encoder is a real modelling change, not a packaging change, and must clear the same bar as R7.\n\nSCOPED AS P3 DELIBERATELY: it is a cost and ergonomics win that makes versioning (R4) and selective training (R5) cheaper, but neither depends on it. Do not let it block them.\n\nVERIFY current cost: du -sh \u003cworkflow\u003e/___command_info/*/ | sort -h | tail -5","notes":"RETIRED — NOT A LOSSLESS PACKAGING REFACTOR. One Tiny+Distil context is 287,342,019 bytes (274.0 MiB); 33 contexts are ~8.83 GiB. Frozen shared encoders plus per-context heads could reduce that to ~348 MiB (96.1%, 26x). But current training fine-tunes every encoder independently: sampled contexts shared 0/41 identical Tiny tensors and 0/104 Distil tensors, so existing artifacts cannot be split losslessly. Frozen shared encoders change the trained model and may reduce routing/escalation accuracy; joint multi-context training invalidates every head and defeats R5; adapters add major runtime/serialization/dependency complexity. R4/R5 already hardlink carried-forward contexts, reducing version duplication. Without a pre-registered five-seed non-inferiority experiment, storage savings do not justify this modeling redesign. Documented as spec M14; no replacement task created.\nRETIRED — NOT A LOSSLESS PACKAGING REFACTOR. One Tiny+Distil context is 287,342,019 bytes (274.0 MiB); 33 contexts are ~8.83 GiB. Frozen shared encoders plus per-context heads could reduce that to ~348 MiB (96.1%, 26x). But current training fine-tunes every encoder independently: sampled contexts shared 0/41 identical Tiny tensors and 0/104 Distil tensors, so existing artifacts cannot be split losslessly. Frozen shared encoders change the trained model and may reduce routing/escalation accuracy; joint multi-context training invalidates every head and defeats R5; adapters add major runtime/serialization/dependency complexity. R4/R5 already hardlink carried-forward contexts, reducing version duplication. Without a pre-registered five-seed non-inferiority experiment, storage savings do not justify this modeling redesign. Documented as spec M14; no replacement task created.\nRETIRED: SHARED ENCODERS REQUIRE MODEL REDESIGN, not artifact deduplication. Per context Tiny+Distil is 274 MiB; 33 contexts ~8.83 GiB. Frozen shared encoders could cut this ~96%, but current contexts share 0/41 Tiny and 0/104 Distil tensors because all encoder weights are independently fine-tuned. Frozen encoders risk accuracy; joint training defeats R5; adapters add major complexity. R4/R5 already hardlink carried artifacts. Without a five-seed non-inferiority experiment, do not exchange known quality for disk. Retired in spec M14; no replacement task.","status":"closed","priority":3,"issue_type":"task","owner":"drawal@radiantlogic.com","created_at":"2026-08-02T14:36:58Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-08-03T01:42:25Z","closed_at":"2026-08-03T02:01:50Z","close_reason":"Closed","dependencies":[{"issue_id":"fix-551.8","depends_on_id":"fix-551","type":"parent-child","created_at":"2026-08-02T09:36:57Z","created_by":"Dhar Rawal","metadata":"{}"},{"issue_id":"fix-551.8","depends_on_id":"fix-551.7","type":"blocks","created_at":"2026-08-02T09:38:34Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-vof.42","title":"R44: Capture both raw and refined user messages","description":"FINDING: _refine_user_query rewrites the user message using conversation history before the agent sees it; TurnResult.user_message is undefined on this axis, and refinement bugs are a classic why-did-the-agent-do-that cause. OPEN QUESTIONS: store raw + refined per user input (folds into the R1 event/exchange shape). See docs/turn_result_design_review.md R44.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","status":"closed","priority":3,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:36Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-11T22:04:12Z","closed_at":"2026-06-11T22:04:56Z","close_reason":"Finding finalized: refined_user_message dedicated optional field; recorded in bead, review doc R44, design doc Amendment A36","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.42","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:35Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"fix-vof.43","title":"R45: Decide boundary offload vs eager offload (memory profile)","description":"FINDING: accumulate-all retains every payload in RAM until turn end (today they are garbage immediately after text extraction); chart payloads are unbounded (full-frame df.write_csv, only tables are row-capped); multiply by concurrent sessions. OPEN QUESTIONS: (1) accept-and-document vs eager offload (D-prime: offload above-threshold payloads at capture, keep envelopes in memory - bounds RAM, cheapens suspend, write-only I/O off the critical decision path); (2) interaction with R41 (eagerly offloaded payloads must still serve the live response). See docs/turn_result_design_review.md R45.\n\nMISSION (review, not implement): resolve the OPEN QUESTIONS collaboratively with the user, then finalize this finding into an unambiguous, detailed and complete issue description plus a design recommendation of matching quality. Record outcomes by updating this bead and the finding in docs/turn_result_design_review.md, and reconcile the affected sections of docs/turn_result_design.md. No code changes.","status":"closed","priority":3,"issue_type":"task","assignee":"Dhar Rawal","owner":"drawal@radiantlogic.com","created_at":"2026-06-10T21:29:36Z","created_by":"Dhar Rawal","updated_at":"2026-08-07T10:16:55Z","started_at":"2026-06-11T19:41:18Z","closed_at":"2026-06-11T19:42:10Z","close_reason":"Finding finalized: boundary offload per A16, memory profile documented, envelope-entry fetch fallback for rehydrated turns; recorded in bead, review doc R45, design doc Amendment A17","labels":["turn-result-design"],"dependencies":[{"issue_id":"fix-vof.43","depends_on_id":"fix-vof","type":"parent-child","created_at":"2026-06-10T16:29:36Z","created_by":"Dhar Rawal","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/README.md b/README.md index 3281054..14edbda 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ This is the fastest way to see fastWorkflow in action.

```sh -# 1. Install (Linux/macOS; on Windows use WSL. Python 3.11+) +# 1. Install (Linux/macOS; on Windows use WSL. Python 3.13+) pip install fastworkflow # 2. Fetch the hello_world example + env file templates @@ -494,9 +494,10 @@ pip install "fastworkflow[training]" # adds HuggingFace datasets for the train ``` **Notes** -- Linux/macOS only — on Windows use WSL. Python 3.11+. +- Linux/macOS only — on Windows use WSL. Python 3.13–3.14 (stdlib `sqlite3` replaced the abandoned `speedict`/RocksDB dependency that blocked 3.13 installs). - Installs PyTorch; the first install may take a few minutes. - `fastworkflow train` needs the optional HuggingFace `datasets` package (`pip install datasets`, or `poetry install --with dev` from this repo). +- On-disk conversation stores are now `{channel_id}.sqlite3` under `SPEEDDICT_FOLDERNAME/channel_conversations`. NLU caches use `*.sqlite3` under `___convo_info/`. Pre-existing RocksDB `.rdb` / `cache.db` directories are unused and may be deleted. Downstream shims that aliased `speedict.Rdict` to `rocksdict.Rdict` can be removed. The core depends on **plain** `litellm` (client only — no proxy server stack), so it co-installs cleanly with downstream apps that pin a plain `litellm`. Server-only deps live behind the `server` extra. diff --git a/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py b/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py index 265d579..daf2d87 100644 --- a/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py +++ b/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py @@ -4,12 +4,12 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from pydantic import BaseModel -from speedict import Rdict import fastworkflow from fastworkflow.utils.logging import logger from fastworkflow import NLUPipelineStage from fastworkflow.cache_matching import cache_match, store_utterance_cache +from fastworkflow.kvstore import KVStore from fastworkflow.model_pipeline_training import ( CommandRouter ) @@ -194,7 +194,7 @@ def _get_cache_path(workflow_id, convo_path): base_dir = convo_path # Create directory if it doesn't exist os.makedirs(base_dir, exist_ok=True) - return os.path.join(base_dir, f"{workflow_id}.db") + return os.path.join(base_dir, f"{workflow_id}.sqlite3") @staticmethod def _get_cache_path_cache(convo_path): @@ -204,7 +204,7 @@ def _get_cache_path_cache(convo_path): base_dir = convo_path # Create directory if it doesn't exist os.makedirs(base_dir, exist_ok=True) - return os.path.join(base_dir, "cache.db") + return os.path.join(base_dir, "cache.sqlite3") # Store the suggested commands with the flag type @staticmethod @@ -217,10 +217,11 @@ def _store_suggested_commands(cache_path, command_list, flag_type): command_list: List of suggested commands flag_type: Type of constraint (1=ambiguous, 2=misclassified) """ - db = Rdict(cache_path) + db = KVStore(cache_path) try: - db["suggested_commands"] = command_list - db["flag_type"] = flag_type + # predict() returns a numpy ndarray of labels; JSON needs plain strs. + db["suggested_commands"] = [str(c) for c in list(command_list)] + db["flag_type"] = int(flag_type) finally: db.close() @@ -230,7 +231,7 @@ def _get_suggested_commands(cache_path): """ Get the list of suggested commands for the constrained selection """ - db = Rdict(cache_path) + db = KVStore(cache_path) try: return db.get("suggested_commands", []) finally: @@ -238,7 +239,7 @@ def _get_suggested_commands(cache_path): @staticmethod def _get_count(cache_path): - db = Rdict(cache_path) + db = KVStore(cache_path) try: return db.get("utterance_count", 0) # Default to 0 if key doesn't exist finally: @@ -246,7 +247,7 @@ def _get_count(cache_path): @staticmethod def _print_db_contents(cache_path): - db = Rdict(cache_path) + db = KVStore(cache_path) try: print("All keys in database:", list(db.keys())) for key in db.keys(): @@ -261,7 +262,7 @@ def _store_utterance(cache_path, utterance, label): Returns: The utterance count used """ # Open the database (creates if doesn't exist) - db = Rdict(cache_path) + db = KVStore(cache_path) try: # Get existing counter or initialize to 0 @@ -291,12 +292,11 @@ def _read_utterance(cache_path, utterance_id): """ Read a specific utterance from the database """ - db = Rdict(cache_path) + db = KVStore(cache_path) try: return db.get(utterance_id)['utterance'] finally: db.close() - @staticmethod def resolve_fully_qualified_command_name( command_name: Optional[str], command_name_dict: dict[str, str]) -> Optional[str]: diff --git a/fastworkflow/cache_matching.py b/fastworkflow/cache_matching.py index 11bc6a1..4a58def 100644 --- a/fastworkflow/cache_matching.py +++ b/fastworkflow/cache_matching.py @@ -1,11 +1,12 @@ import numpy as np from sklearn.metrics.pairwise import cosine_similarity -import fastworkflow import torch -from speedict import Rdict import mmh3 # mmh33 implementation from datetime import datetime from functools import lru_cache +import weakref + +from fastworkflow.kvstore import UtteranceCacheStore # --------------------------------------------------------------------- # In-process memoisation for expensive DistilBERT embeddings. @@ -26,7 +27,6 @@ def _cached_embedding(model_id: int, text: str): raise RuntimeError("ModelPipeline instance no longer alive; cache invalid.") return _compute_embedding(text, pipeline) -import weakref _MODEL_ID_2_REF: dict[int, weakref.ReferenceType] = {} @@ -63,58 +63,59 @@ def store_utterance_cache(cache_path, utterance, label, model_pipeline=None): Returns: The hash key of the stored utterance """ - # Open the database - db = Rdict(cache_path) + db = UtteranceCacheStore(cache_path) try: # Generate hash for utterance using mmh3 utterance_hash = str(mmh3.hash(utterance)) - # Get the cache or initialize - cache = db.get("cache", {}) - # Get current timestamp for feedback date current_time = datetime.now().isoformat() # Compute embedding if model_pipeline provided embedding = None if model_pipeline is not None: - embedding = get_embedding(utterance, model_pipeline)[0].tolist() - - if utterance_hash in cache: - # Update existing entry + embedding = get_embedding(utterance, model_pipeline)[0] + + existing = db.get(utterance_hash) + if existing is not None: + command_mapping = existing["command_mapping"] + stored_embedding = existing["embedding"] + stored_utterance = existing["utterance"] + if embedding is not None: - cache[utterance_hash]["embedding"] = embedding - - if label in cache[utterance_hash]["command_mapping"]: - # Increment frequency for this label - cache[utterance_hash]["command_mapping"][label]["frequency"] += 1 - cache[utterance_hash]["command_mapping"][label]["feedback_date"] = current_time + stored_embedding = embedding + + if label in command_mapping: + command_mapping[label]["frequency"] += 1 + command_mapping[label]["feedback_date"] = current_time else: - # Add new label mapping - cache[utterance_hash]["command_mapping"][label] = { + command_mapping[label] = { "frequency": 1, "feedback_date": current_time } + + db.upsert( + utterance_hash, + utterance=stored_utterance or utterance, + command_mapping=command_mapping, + embedding=stored_embedding, + ) else: - # Create new entry - cache[utterance_hash] = { - "embedding": embedding if embedding is not None else [], - "utterance": utterance, # Store original utterance for reference - "command_mapping": { + db.upsert( + utterance_hash, + utterance=utterance, + command_mapping={ label: { "frequency": 1, "feedback_date": current_time } - } - } - - # Save updated cache to database - db["cache"] = cache + }, + embedding=embedding if embedding is not None else None, + ) return utterance_hash finally: - # Always close the database db.close() def get_embedding(text: str, model_pipeline): @@ -144,14 +145,10 @@ def cache_match(cache_path, utterance, model_pipeline, threshold=0.90, return_de If match found: true_label or (true_label, similarity) if return_details=True If no match: None """ - # Open the database - db = Rdict(cache_path) + db = UtteranceCacheStore(cache_path) try: - # Get the cache dictionary - cache = db.get("cache", {}) - - # If no entries, return None - if not cache: + entries = list(db.iter_entries()) + if not entries: return None # Get embedding for the query utterance @@ -162,25 +159,25 @@ def cache_match(cache_path, utterance, model_pipeline, threshold=0.90, return_de # Check cache for similar utterances best_similarity = 0 - cache_match = None + best_key = None + best_mapping = None - # Find the best matching cached utterance - for hash_key, entry in cache.items(): - # Skip entries without embeddings - if not entry.get("embedding"): + for hash_key, entry in entries: + cached_embedding = entry.get("embedding") + if cached_embedding is None or cached_embedding.size == 0: continue - # Reshape cached embedding for cosine_similarity - cached_embedding = np.array(entry["embedding"]).reshape(1, -1) + cached_embedding = np.asarray(cached_embedding, dtype=np.float32).reshape(1, -1) similarity = cosine_similarity(query_embedding, cached_embedding)[0][0] if similarity > best_similarity: best_similarity = similarity - cache_match = hash_key + best_key = hash_key + best_mapping = entry["command_mapping"] # If good cache match found, determine the best label - if best_similarity >= threshold and cache_match is not None: - command_mapping = cache[cache_match]["command_mapping"] + if best_similarity >= threshold and best_key is not None and best_mapping is not None: + command_mapping = best_mapping # If only one label, return it directly if len(command_mapping) == 1: @@ -211,5 +208,4 @@ def cache_match(cache_path, utterance, model_pipeline, threshold=0.90, return_de # No good match found return None finally: - # Always close the database - db.close() \ No newline at end of file + db.close() diff --git a/fastworkflow/kvstore.py b/fastworkflow/kvstore.py new file mode 100644 index 0000000..9bc73d1 --- /dev/null +++ b/fastworkflow/kvstore.py @@ -0,0 +1,187 @@ +"""SQLite-backed key-value store. + +Replaces speedict/RocksDB. RocksDB took a process-exclusive lock, which forced an +open/close cycle around every operation and still raced across processes; SQLite in +WAL mode supports concurrent readers alongside a writer. Values are JSON, not pickle, +so a writable store directory is not an arbitrary-code-execution primitive. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +from typing import Any, Iterator, Optional + +import numpy as np + + +def _key_str(key: Any) -> str: + """Coerce mapping keys to TEXT. Call sites historically used int keys with Rdict.""" + return key if isinstance(key, str) else str(key) + + +class KVStore: + """A durable dict[str, Any]. Values must be JSON-serialisable.""" + + def __init__(self, path: str, *, timeout: float = 30.0) -> None: + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + # check_same_thread=False is safe here because sqlite3 serialises access + # internally and every method below is a single self-contained statement. + self._conn = sqlite3.connect(path, timeout=timeout, check_same_thread=False) + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self._conn.execute(f"PRAGMA busy_timeout={int(timeout * 1000)}") + self._conn.execute( + "CREATE TABLE IF NOT EXISTS kv (k TEXT PRIMARY KEY, v TEXT NOT NULL)" + ) + self._conn.commit() + + def __setitem__(self, key: Any, value: Any) -> None: + self._conn.execute( + "INSERT INTO kv (k, v) VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v=excluded.v", + (_key_str(key), json.dumps(value)), + ) + self._conn.commit() + + def __getitem__(self, key: Any) -> Any: + row = self._conn.execute( + "SELECT v FROM kv WHERE k=?", (_key_str(key),) + ).fetchone() + if row is None: + raise KeyError(key) + return json.loads(row[0]) + + def __delitem__(self, key: Any) -> None: + cur = self._conn.execute("DELETE FROM kv WHERE k=?", (_key_str(key),)) + self._conn.commit() + if cur.rowcount == 0: + raise KeyError(key) + + def __contains__(self, key: Any) -> bool: + return ( + self._conn.execute( + "SELECT 1 FROM kv WHERE k=?", (_key_str(key),) + ).fetchone() + is not None + ) + + def get(self, key: Any, default: Any = None) -> Any: + row = self._conn.execute( + "SELECT v FROM kv WHERE k=?", (_key_str(key),) + ).fetchone() + return default if row is None else json.loads(row[0]) + + def keys(self) -> Iterator[str]: + # Materialise first so callers may mutate the store while iterating + # (conversation_store historically did this against Rdict). + return (r[0] for r in self._conn.execute("SELECT k FROM kv").fetchall()) + + def close(self) -> None: + self._conn.close() + + def __enter__(self) -> "KVStore": + return self + + def __exit__(self, *exc: object) -> None: + self.close() + + +class UtteranceCacheStore: + """Per-utterance embedding cache with float32 BLOB vectors. + + The mechanical JSON-in-KVStore swap for this workload is 3–6x slower than + speedict; one row per hash with a raw float32 column is ~132x faster. + Shares a SQLite file safely with :class:`KVStore` (separate tables). + """ + + def __init__(self, path: str, *, timeout: float = 30.0) -> None: + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + self._conn = sqlite3.connect(path, timeout=timeout, check_same_thread=False) + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self._conn.execute(f"PRAGMA busy_timeout={int(timeout * 1000)}") + self._conn.execute( + """ + CREATE TABLE IF NOT EXISTS utterance_cache ( + k TEXT PRIMARY KEY, + meta TEXT NOT NULL, + vec BLOB NOT NULL + ) + """ + ) + self._conn.commit() + + @staticmethod + def _pack_vec(embedding: Optional[Any]) -> bytes: + if embedding is None: + return b"" + arr = np.asarray(embedding, dtype=np.float32) + if arr.size == 0: + return b"" + return arr.tobytes() + + @staticmethod + def _unpack_vec(blob: bytes) -> Optional[np.ndarray]: + if not blob: + return None + return np.frombuffer(blob, dtype=np.float32) + + def get(self, key: str) -> Optional[dict[str, Any]]: + row = self._conn.execute( + "SELECT meta, vec FROM utterance_cache WHERE k=?", (key,) + ).fetchone() + if row is None: + return None + meta = json.loads(row[0]) + embedding = self._unpack_vec(row[1]) + return { + "utterance": meta.get("utterance", ""), + "command_mapping": meta.get("command_mapping", {}), + "embedding": embedding, + } + + def upsert( + self, + key: str, + *, + utterance: str, + command_mapping: dict[str, Any], + embedding: Optional[Any], + ) -> None: + meta = json.dumps( + {"utterance": utterance, "command_mapping": command_mapping} + ) + self._conn.execute( + """ + INSERT INTO utterance_cache (k, meta, vec) VALUES (?, ?, ?) + ON CONFLICT(k) DO UPDATE SET meta=excluded.meta, vec=excluded.vec + """, + (key, meta, self._pack_vec(embedding)), + ) + self._conn.commit() + + def iter_entries(self) -> Iterator[tuple[str, dict[str, Any]]]: + rows = self._conn.execute( + "SELECT k, meta, vec FROM utterance_cache" + ).fetchall() + for key, meta_json, vec in rows: + meta = json.loads(meta_json) + yield key, { + "utterance": meta.get("utterance", ""), + "command_mapping": meta.get("command_mapping", {}), + "embedding": self._unpack_vec(vec), + } + + def close(self) -> None: + self._conn.close() + + def __enter__(self) -> "UtteranceCacheStore": + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/fastworkflow/run_fastapi_mcp/__main__.py b/fastworkflow/run_fastapi_mcp/__main__.py index 8fdeef7..d2f0176 100644 --- a/fastworkflow/run_fastapi_mcp/__main__.py +++ b/fastworkflow/run_fastapi_mcp/__main__.py @@ -5,7 +5,7 @@ Implementation Status: - ✅ All endpoints implemented per spec - ✅ Session management and concurrency control -- ✅ Rdict-backed conversation persistence +- ✅ SQLite-backed conversation persistence - ✅ Agent trace collection and inclusion in responses - ✅ SSE streaming for real-time trace events (/invoke_agent_stream) - ✅ Error handling with proper HTTP status codes @@ -1738,7 +1738,7 @@ async def activate_conversation( async def dump_all_conversations(request: DumpConversationsRequest) -> dict[str, str]: """ Admin endpoint: dump all conversations from all sessions to a JSONL file. - Scans all .rdb files in the base folder, not just active sessions. + Scans all .sqlite3 conversation stores in the base folder, not just active sessions. """ try: os.makedirs(request.output_folder, exist_ok=True) @@ -1751,12 +1751,12 @@ async def dump_all_conversations(request: DumpConversationsRequest) -> dict[str, all_conversations = [] session_count = 0 - # Scan the base folder for all .rdb files (all users, active or not) + # Scan the base folder for all .sqlite3 files (all users, active or not) if os.path.isdir(base_folder): for filename in os.listdir(base_folder): - if filename.endswith('.rdb'): - # Extract channel_id from filename (format: .rdb) - channel_id = filename[:-4] # Remove .rdb extension + if filename.endswith('.sqlite3'): + # Extract channel_id from filename (format: .sqlite3) + channel_id = filename[:-8] # Remove .sqlite3 extension # Create temporary ConversationStore for this user store = ConversationStore(channel_id, base_folder) diff --git a/fastworkflow/run_fastapi_mcp/conversation_store.py b/fastworkflow/run_fastapi_mcp/conversation_store.py index 9f491f1..5ca1e31 100644 --- a/fastworkflow/run_fastapi_mcp/conversation_store.py +++ b/fastworkflow/run_fastapi_mcp/conversation_store.py @@ -1,6 +1,6 @@ """ Conversation persistence layer for FastWorkflow -Provides Rdict-backed storage for multi-turn conversations with AI-generated topics/summaries +Provides SQLite-backed storage for multi-turn conversations with AI-generated topics/summaries """ import json @@ -11,8 +11,8 @@ import dspy from pydantic import BaseModel -from speedict import Rdict +from fastworkflow.kvstore import KVStore from fastworkflow.utils.logging import logger from fastworkflow.utils.dspy_utils import get_lm @@ -33,57 +33,46 @@ class ConversationSummary(BaseModel): class ConversationStore: - """Rdict-backed conversation persistence per user. + """SQLite-backed conversation persistence per user. - Turns are stored one Rdict entry per turn (``conv:{id}:turn:{index}``) so an + Turns are stored one key per turn (``conv:{id}:turn:{index}``) so an incremental save writes only the new turns instead of rewriting the whole conversation. The ``conv:{id}`` record keeps the metadata plus ``appended_turn_count``; reads rehydrate ``turns`` so callers see the same - record shape as before. Records written by earlier versions keep their turns - inline under ``turns`` and are migrated to per-turn entries on first append. + record shape as before. Pre-migration ``.rdb`` (RocksDB) files are abandoned; + this store uses ``{channel_id}.sqlite3`` only. """ def __init__(self, channel_id: str, base_folder: str): self.channel_id = channel_id - self.db_path = os.path.join(base_folder, f"{channel_id}.rdb") + self.db_path = os.path.join(base_folder, f"{channel_id}.sqlite3") os.makedirs(base_folder, exist_ok=True) - def _get_db(self) -> Rdict: - """Get Rdict instance""" - return Rdict(self.db_path) + def _get_db(self) -> KVStore: + """Get KVStore instance""" + return KVStore(self.db_path) @staticmethod def _turn_key(conversation_id: int, index: int) -> str: return f"conv:{conversation_id}:turn:{index}" def _iter_turn_records( - self, db: Rdict, conversation_id: int, conv: dict[str, Any] + self, db: KVStore, conversation_id: int, conv: dict[str, Any] ): - """Yield a conversation's turns in order, one at a time. - - An inline ``turns`` list wins outright. Only a writer that rewrites the - whole list produces one, and every writer here empties it, so a record - that still has one was written by an older version of this store — which - makes it the authoritative list and any leftover per-turn entries stale. - Concatenating the two instead would duplicate and reorder turns after a - version rollback. - """ - if inline_turns := conv.get("turns") or []: - yield from inline_turns - return + """Yield a conversation's turns in order, one at a time.""" for index in range(int(conv.get("appended_turn_count") or 0)): turn_key = self._turn_key(conversation_id, index) if turn_key in db: yield db[turn_key] def _read_turns( - self, db: Rdict, conversation_id: int, conv: dict[str, Any] + self, db: KVStore, conversation_id: int, conv: dict[str, Any] ) -> list[dict[str, Any]]: """Rehydrate a conversation's turns.""" return list(self._iter_turn_records(db, conversation_id, conv)) def _hydrated( - self, db: Rdict, conversation_id: int, conv: dict[str, Any] + self, db: KVStore, conversation_id: int, conv: dict[str, Any] ) -> dict[str, Any]: """The stored record as callers expect it: turns rehydrated, bookkeeping hidden.""" hydrated = {k: v for k, v in conv.items() if k != "appended_turn_count"} @@ -92,7 +81,7 @@ def _hydrated( def _replace_turns( self, - db: Rdict, + db: KVStore, conversation_id: int, conv: dict[str, Any], turns: list[dict[str, Any]], @@ -122,7 +111,7 @@ def get_last_conversation_id(self) -> Optional[int]: finally: db.close() - def _increment_conversation_id(self, db: Rdict) -> int: + def _increment_conversation_id(self, db: KVStore) -> int: """Increment and return new conversation ID""" meta = db.get("meta", {"last_conversation_id": 0}) new_id = meta["last_conversation_id"] + 1 @@ -138,7 +127,7 @@ def reserve_next_conversation_id(self) -> int: finally: db.close() - def _ensure_unique_topic(self, db: Rdict, candidate_topic: str) -> str: + def _ensure_unique_topic(self, db: KVStore, candidate_topic: str) -> str: """Ensure topic is unique per user with case/whitespace insensitive comparison""" # Normalize for comparison normalized_candidate = candidate_topic.lower().strip() @@ -395,14 +384,6 @@ def append_conversation_turns( } next_index = int(conv.get("appended_turn_count") or 0) - # A record written by an earlier version keeps its turns inline. - # Move them out once, so this and every later append stays O(1) in - # bytes written instead of rewriting the inline list every turn. - # Inline is the authoritative list (see _iter_turn_records), so any - # per-turn entries it coexists with are stale and get replaced. - if inline_turns := list(conv.get("turns") or []): - self._replace_turns(db, conversation_id, conv, inline_turns) - next_index = len(inline_turns) for offset, turn in enumerate(new_turns): db[self._turn_key(conversation_id, next_index + offset)] = turn @@ -422,8 +403,6 @@ def count_conversation_turns(self, conversation_id: int) -> int: conv = db.get(f"conv:{conversation_id}") if conv is None: return 0 - if inline_turns := conv.get("turns") or []: - return len(inline_turns) return int(conv.get("appended_turn_count") or 0) finally: db.close() @@ -472,14 +451,10 @@ def update_last_conversation_turn( conv = db[conv_key] appended = int(conv.get("appended_turn_count") or 0) - if appended: - db[self._turn_key(conversation_id, appended - 1)] = turn - elif inline_turns := list(conv.get("turns") or []): - inline_turns[-1] = turn - conv["turns"] = inline_turns - else: + if not appended: return False + db[self._turn_key(conversation_id, appended - 1)] = turn conv["updated_at"] = int(time.time() * 1000) db[conv_key] = conv return True @@ -533,4 +508,3 @@ class TopicSummarySignature(dspy.Signature): generator = dspy.ChainOfThought(TopicSummarySignature) result = generator(conversation_turns=turns_str) return result.topic, result.summary - diff --git a/fastworkflow/workflow.py b/fastworkflow/workflow.py index e5c9c08..befb43f 100644 --- a/fastworkflow/workflow.py +++ b/fastworkflow/workflow.py @@ -5,9 +5,8 @@ from functools import wraps from typing import Optional -from speedict import Rdict - import fastworkflow +from fastworkflow.kvstore import KVStore from fastworkflow.utils.logging import logger @@ -34,9 +33,10 @@ # SessionStateStore + ConversationStore; the CLI no longer resumes workflow # context across process restarts (accepted trade-off). # -# speedict is still used elsewhere (the enablecache decorator below, -# ConversationStore, and the NLU clarification cache) and is intentionally -# left in place there. +# Remaining on-disk caches (enablecache below, ConversationStore, the NLU +# clarification cache, and utterance/embedding matching) use stdlib sqlite3 +# via :class:`fastworkflow.kvstore.KVStore` / ``UtteranceCacheStore`` — JSON +# values, WAL mode, no process-exclusive LOCK. # ---------------------------------------------------------------------- _STATE_LOCK = threading.RLock() # workflow_id -> live Workflow (weak, so abandoned sessions auto-evict) @@ -52,14 +52,21 @@ def wrapper(self, *args, **kwargs): # Create a cache key based on the function arguments key = str(args) + str(kwargs) - # Get the cache database - cache_db_path = self.get_cachedb_folderpath(func.__name__) - cache_db = Rdict(cache_db_path) + # Get the cache database (folder path historically fed RocksDB; SQLite + # needs a file inside that folder). + cache_db_folder = self.get_cachedb_folderpath(func.__name__) + cache_db = KVStore(os.path.join(cache_db_folder, "cache.sqlite3")) if key not in cache_db: # If the result is not in the cache, call the function and store the result result = func(self, *args, **kwargs) - cache_db[key] = result + try: + cache_db[key] = result + except TypeError as exc: + raise TypeError( + f"@enablecache requires a JSON-serialisable return value; " + f"{func.__qualname__} returned {type(result).__name__}" + ) from exc else: result = cache_db[key] diff --git a/poetry.lock b/poetry.lock index 0e1a2e9..c22fec4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -146,7 +146,6 @@ attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" propcache = ">=0.2.0" -typing_extensions = {version = ">=4.4", markers = "python_version < \"3.13\""} yarl = ">=1.17.0,<2.0" [package.extras] @@ -165,7 +164,6 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" -typing-extensions = {version = ">=4.2", markers = "python_version < \"3.13\""} [[package]] name = "annotated-doc" @@ -202,7 +200,6 @@ files = [ [package.dependencies] idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] trio = ["trio (>=0.32.0)"] @@ -1417,7 +1414,6 @@ anyio = ">=4.10" httpcore2 = "2.9.1" idna = ">=3.18" truststore = ">=0.10" -typing-extensions = {version = ">=4.5.0", markers = "python_version < \"3.13\""} [package.extras] brotli = ["brotli", "brotlicffi"] @@ -1881,7 +1877,7 @@ files = [ ] [package.dependencies] -pyyaml = {version = ">=5.2", markers = "python_version < \"3.13\""} +pyyaml = {version = ">=6.0.3", markers = "python_version >= \"3.14\""} pyyaml-ft = {version = ">=8.0.0", markers = "python_version == \"3.13\""} [[package]] @@ -2232,13 +2228,22 @@ anyio = ">=4.5" httpx = ">=0.27.1,<1.0.0" httpx-sse = ">=0.4" jsonschema = ">=4.20.0" -pydantic = {version = ">=2.11.0,<3.0.0", markers = "python_version < \"3.14\""} +pydantic = [ + {version = ">=2.11.0,<3.0.0", markers = "python_version < \"3.14\""}, + {version = ">=2.12.0,<3.0.0", markers = "python_version >= \"3.14\""}, +] pydantic-settings = ">=2.5.2" pyjwt = {version = ">=2.10.1", extras = ["crypto"]} python-multipart = ">=0.0.9" -pywin32 = {version = ">=310", markers = "sys_platform == \"win32\" and python_version < \"3.14\""} +pywin32 = [ + {version = ">=310", markers = "sys_platform == \"win32\" and python_version < \"3.14\""}, + {version = ">=311", markers = "sys_platform == \"win32\" and python_version >= \"3.14\""}, +] sse-starlette = ">=1.6.1" -starlette = {version = ">=0.27", markers = "python_version < \"3.14\""} +starlette = [ + {version = ">=0.27", markers = "python_version < \"3.14\""}, + {version = ">=0.48.0", markers = "python_version >= \"3.14\""}, +] typing-extensions = ">=4.9.0" typing-inspection = ">=0.4.1" uvicorn = {version = ">=0.31.1", markers = "sys_platform != \"emscripten\""} @@ -2688,13 +2693,13 @@ sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] [[package]] name = "networkx" -version = "3.6.1" +version = "3.6" description = "Python package for creating and manipulating graphs and networks" optional = false -python-versions = "!=3.14.1,>=3.11" +python-versions = ">=3.11" files = [ - {file = "networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762"}, - {file = "networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509"}, + {file = "networkx-3.6-py3-none-any.whl", hash = "sha256:cdb395b105806062473d3be36458d8f1459a4e4b98e236a66c3a48996e07684f"}, + {file = "networkx-3.6.tar.gz", hash = "sha256:285276002ad1f7f7da0f7b42f004bcba70d381e936559166363707fdad3d72ad"}, ] [package.extras] @@ -2964,7 +2969,10 @@ files = [ ] [package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} +numpy = [ + {version = ">=1.26.0", markers = "python_version < \"3.14\""}, + {version = ">=2.3.3", markers = "python_version >= \"3.14\""}, +] python-dateutil = ">=2.8.2" tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} @@ -3497,10 +3505,7 @@ files = [ [package.dependencies] astroid = ">=3.3.8,<=3.4.0.dev0" colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -dill = [ - {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, - {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, -] +dill = {version = ">=0.3.7", markers = "python_version >= \"3.12\""} isort = ">=4.2.5,<5.13 || >5.13,<7" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2" @@ -3904,7 +3909,6 @@ files = [ [package.dependencies] attrs = ">=22.2.0" rpds-py = ">=0.7.0" -typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "regex" @@ -4526,51 +4530,6 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] -[[package]] -name = "speedict" -version = "0.3.12" -description = "Speedb Python Binding" -optional = false -python-versions = "*" -files = [ - {file = "speedict-0.3.12-cp310-cp310-macosx_10_14_x86_64.macosx_11_0_arm64.macosx_10_14_universal2.whl", hash = "sha256:8a8b9bbcd2bae9dcf6e233b79adccae253adfb0ac30c908aaf059eb5eb4c1ba4"}, - {file = "speedict-0.3.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ed7c5d713683dfaa49736363c647f2a627422efbad890f8531be2014e816666"}, - {file = "speedict-0.3.12-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:14c3f5618f131ac786a5e58dab7fc89e7c96e011eeef01fc191f3c9f08c08b2a"}, - {file = "speedict-0.3.12-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:7ad77167e30e1974a190ff388fb4b2cd90913bc3fc58f55371484e5d8870f5a5"}, - {file = "speedict-0.3.12-cp310-none-win32.whl", hash = "sha256:9d1c119e4624fb11557647007a875dac99cdce7af441c5539f9ca64434d53cf2"}, - {file = "speedict-0.3.12-cp310-none-win_amd64.whl", hash = "sha256:447f8177ea9b05f33f4928d8f87303cc285b49273c4e9bb3b7c24401a495e0ba"}, - {file = "speedict-0.3.12-cp311-cp311-macosx_10_14_x86_64.macosx_11_0_arm64.macosx_10_14_universal2.whl", hash = "sha256:e7bdbb14d1ada7a9980d2ef93c2fe7c23ccbff449042d37f2aab59b73a775a12"}, - {file = "speedict-0.3.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:820a856f94ed7fae6cf0df9a4e1d9273134e4f574bf81946a5f7ab8de6a43897"}, - {file = "speedict-0.3.12-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e530421a1156d6648acfe3831f79f7a9956c06c9039d973cf8d205b8fc570208"}, - {file = "speedict-0.3.12-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cdfa2cce2755d1bcfb955bf70adae1066565a4c3930d52c7dfede6fb07e37a53"}, - {file = "speedict-0.3.12-cp311-none-win32.whl", hash = "sha256:fbf5cc085d9cd6eed7de55311970b6003367ccf476beb806fe5b82d9f0e2e123"}, - {file = "speedict-0.3.12-cp311-none-win_amd64.whl", hash = "sha256:cf04e816ac9106fe48d5a8420a694accc25fa6afda0161c851ca8f62e107067f"}, - {file = "speedict-0.3.12-cp312-cp312-macosx_10_14_x86_64.macosx_11_0_arm64.macosx_10_14_universal2.whl", hash = "sha256:349558a553c32ac3fd8d8c8f3783f1029ffff67036b27857c1be54e1dc99f94f"}, - {file = "speedict-0.3.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10ed33d39fb247a879c13ed472fbe347cbf08109e8d96716455beff0d6200b69"}, - {file = "speedict-0.3.12-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:aaf83944fee9dc115506d5fb2bc24049f91a088da32272ab8e7e28684a6abfac"}, - {file = "speedict-0.3.12-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5ec1a786ed17cefc0c3205b6ce29e4acebef9a3af61777eb0fb22b4ad9ac13ea"}, - {file = "speedict-0.3.12-cp312-none-win32.whl", hash = "sha256:e6e05472e7eea5e4ead607831a62ea0d8617bd5a3debe85b8bb9c615b85fd9c4"}, - {file = "speedict-0.3.12-cp312-none-win_amd64.whl", hash = "sha256:532aedcf448007d293debfbcb3b95f0a89f5837faf0e0481e785f5cdeca38ab1"}, - {file = "speedict-0.3.12-cp37-cp37m-macosx_10_14_x86_64.macosx_11_0_arm64.macosx_10_14_universal2.whl", hash = "sha256:e9404b51e351a2357051bf7145463d95ee1ecfb6e8f9ce08c4e6b2f2be2e9282"}, - {file = "speedict-0.3.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d12160a1d8ba132fe64ecda42a1930c9db003b62413bef07f2dc71ba4abba977"}, - {file = "speedict-0.3.12-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:f7f785898e286498936ac5b61bb2d53603986e45f5917f67f2a07d2ee5704f20"}, - {file = "speedict-0.3.12-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:228e852d1e5359f6412829ad54fbb9a1eecee1d94753d4d348041a0b85ede187"}, - {file = "speedict-0.3.12-cp37-none-win32.whl", hash = "sha256:7f12ef6f26cb23bba799ece8113c78a03dfbe9eb02f3112406d9be13789fb929"}, - {file = "speedict-0.3.12-cp37-none-win_amd64.whl", hash = "sha256:540d40c9e1806f96048218f33b201553758f233d057a51dd40d2937a892dcfd1"}, - {file = "speedict-0.3.12-cp38-cp38-macosx_10_14_x86_64.macosx_11_0_arm64.macosx_10_14_universal2.whl", hash = "sha256:4321e211c32b0f982a8a83e0d3416835d1a6b5d5b249bf497e0b93d27b972e19"}, - {file = "speedict-0.3.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:220ea77095fe2f0b40fa371c0cdec8c68ec0d3bd7989bf3b142b450bdea1fd2a"}, - {file = "speedict-0.3.12-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:086547048c06875cc0c87f4eb940ed64e34b1d001df460a95524d4460706656b"}, - {file = "speedict-0.3.12-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:1e1f603efd634fa584cbb1566e3be30755ad59aec687ff8510775c0c46cd8b48"}, - {file = "speedict-0.3.12-cp38-none-win32.whl", hash = "sha256:8537c4a95e9e66159c47cc3c731d9c0ba14e0154990cc9435d20609014d81e05"}, - {file = "speedict-0.3.12-cp38-none-win_amd64.whl", hash = "sha256:b8d2d589256e8ddf8e4bdba4c8197190d6666775165769622ca3949c7f13b447"}, - {file = "speedict-0.3.12-cp39-cp39-macosx_10_14_x86_64.macosx_11_0_arm64.macosx_10_14_universal2.whl", hash = "sha256:73c8011367bb3367ad96117a0bf96889283a8fa6514247f5f34ae10cf5cb8ec2"}, - {file = "speedict-0.3.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcca71dbe3b7a368511ffd72bdef3ee7ad44c22f7c808133aa0b87e02608774e"}, - {file = "speedict-0.3.12-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:7ab409d6ab915097ebaf42f3b581ab8efd56685d9854a23ade8a60ec2189e352"}, - {file = "speedict-0.3.12-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:70984623badd07fef80d076bc8c24c083d73e9288262899ba79bf822067740b9"}, - {file = "speedict-0.3.12-cp39-none-win32.whl", hash = "sha256:5331a5b9640d61ac76951f8ce3cc58313d3e20fbc338ba55aca32c82d592fe89"}, - {file = "speedict-0.3.12-cp39-none-win_amd64.whl", hash = "sha256:52556bf1b8222dc1a06b8861fcf8ee3d437673ad2d25a535f2cf0f11ee1e0b5d"}, -] - [[package]] name = "sse-starlette" version = "3.4.8" @@ -4606,7 +4565,6 @@ files = [ [package.dependencies] anyio = ">=3.6.2,<5" -typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\""} [package.extras] full = ["httpx (>=0.27.0,<0.29.0)", "httpx2 (>=2.0.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] @@ -5428,5 +5386,5 @@ training = ["datasets"] [metadata] lock-version = "2.0" -python-versions = ">=3.11,<3.14" -content-hash = "493ab610110ac16f696c2e42cefc676aff799c7ec699468776e937bbce93a53c" +python-versions = ">=3.13,<3.15" +content-hash = "09f008d499247be415ec765258e6badf661113c585573aa9433eab3154492b47" diff --git a/pyproject.toml b/pyproject.toml index 5bfbe03..8bb0fa2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ repository = "https://github.com/radiantlogicinc/fastworkflow" [tool.poetry] name = "fastworkflow" -version = "2.30.1" +version = "2.31.0" description = "A framework for rapidly building large-scale, deterministic, interactive workflows with a fault-tolerant, conversational UX" authors = ["Dhar Rawal "] license = "Apache-2.0" @@ -31,9 +31,8 @@ exclude = [ fastworkflow = "fastworkflow.cli:main" [tool.poetry.dependencies] -python = ">=3.11,<3.14" +python = ">=3.13,<3.15" pydantic = "^2.9.2" -speedict = "^0.3.12" python-dotenv = "^1.2.2" # CVE-2026-28684 (set_key/unset_key symlink overwrite) scikit-learn = "^1.6.1" # Allow transformers 5.x (downstream apps pin >=5,<6). The intent-detection diff --git a/tests/soak/memory_soak.py b/tests/soak/memory_soak.py index 5461c72..76e036b 100644 --- a/tests/soak/memory_soak.py +++ b/tests/soak/memory_soak.py @@ -859,7 +859,9 @@ def durable_store_metrics(self) -> dict[str, Any]: conversations_dir = os.path.join(self.speeddict_dir, "channel_conversations") records = 0 if os.path.isdir(conversations_dir): - records = sum(name.endswith(".rdb") for name in os.listdir(conversations_dir)) + records = sum( + name.endswith(".sqlite3") for name in os.listdir(conversations_dir) + ) return { "records": records, "conversation_bytes": _dir_bytes(conversations_dir), diff --git a/tests/test_cache_matching_sqlite.py b/tests/test_cache_matching_sqlite.py new file mode 100644 index 0000000..48c790e --- /dev/null +++ b/tests/test_cache_matching_sqlite.py @@ -0,0 +1,51 @@ +"""Utterance cache matching uses float32 BLOB rows, not a JSON blob.""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +from fastworkflow.cache_matching import cache_match +from fastworkflow.kvstore import UtteranceCacheStore + + +class _FakePipeline: + """Minimal stand-in: get_embedding path is bypassed via direct upsert.""" + + +def test_store_and_match_via_blob_rows(tmp_path: Path, monkeypatch): + path = str(tmp_path / "cache.db") + vec_a = np.ones(8, dtype=np.float32) + vec_b = np.zeros(8, dtype=np.float32) + vec_b[0] = 1.0 + + with UtteranceCacheStore(path) as store: + store.upsert( + "1", + utterance="alpha", + command_mapping={"cmd_a": {"frequency": 2, "feedback_date": "t1"}}, + embedding=vec_a, + ) + store.upsert( + "2", + utterance="beta", + command_mapping={"cmd_b": {"frequency": 1, "feedback_date": "t2"}}, + embedding=vec_b, + ) + + def fake_get_embedding(text, model_pipeline): + # Near-identical to vec_a + return np.ones(8, dtype=np.float32).reshape(1, -1) + + monkeypatch.setattr( + "fastworkflow.cache_matching.get_embedding", fake_get_embedding + ) + label = cache_match(path, "query", _FakePipeline(), threshold=0.5) + assert label == "cmd_a" + + # Mechanical JSON whole-cache key must not exist + from fastworkflow.kvstore import KVStore + + with KVStore(path) as kv: + assert kv.get("cache") is None diff --git a/tests/test_conversation_store_concurrency.py b/tests/test_conversation_store_concurrency.py new file mode 100644 index 0000000..47ff689 --- /dev/null +++ b/tests/test_conversation_store_concurrency.py @@ -0,0 +1,60 @@ +"""Cross-process ConversationStore concurrency (WAL).""" + +from __future__ import annotations + +import multiprocessing +from pathlib import Path + +from fastworkflow.run_fastapi_mcp.conversation_store import ConversationStore + + +def _worker(base: str, channel: str, conv_id: int, n: int, result_path: str) -> None: + """Each process owns one conversation id — exercises concurrent writers on one DB.""" + try: + store = ConversationStore(channel, base) + for i in range(n): + store.append_conversation_turns( + conv_id, + [{"conversation summary": f"c{conv_id}-{i}", "i": i}], + ) + # Interleave reads while other processes write. + assert store.count_conversation_turns(conv_id) == i + 1 + Path(result_path).write_text("ok", encoding="utf-8") + except Exception as exc: # noqa: BLE001 + Path(result_path).write_text(f"fail:{exc!r}", encoding="utf-8") + raise SystemExit(1) from exc + + +def test_conversation_store_four_process_wal(tmp_path: Path): + base = str(tmp_path / "conversations") + channel = "shared" + n_workers = 4 + n_ops = 40 + + seed = ConversationStore(channel, base) + conv_ids = [seed.reserve_next_conversation_id() for _ in range(n_workers)] + + result_paths = [str(tmp_path / f"r{i}.txt") for i in range(n_workers)] + procs = [ + multiprocessing.Process( + target=_worker, args=(base, channel, conv_ids[i], n_ops, result_paths[i]) + ) + for i in range(n_workers) + ] + for p in procs: + p.start() + for p in procs: + p.join(timeout=60) + assert p.exitcode == 0, Path(result_paths[procs.index(p)]).read_text(encoding="utf-8") + + for rp in result_paths: + assert Path(rp).read_text(encoding="utf-8") == "ok" + + store = ConversationStore(channel, base) + assert Path(store.db_path).suffix == ".sqlite3" + for conv_id in conv_ids: + assert store.count_conversation_turns(conv_id) == n_ops + summaries = [ + t["conversation summary"] for t in store.get_conversation(conv_id)["turns"] + ] + assert summaries == [f"c{conv_id}-{i}" for i in range(n_ops)] diff --git a/tests/test_enablecache_kvstore.py b/tests/test_enablecache_kvstore.py new file mode 100644 index 0000000..39acc53 --- /dev/null +++ b/tests/test_enablecache_kvstore.py @@ -0,0 +1,45 @@ +"""enablecache decorator against KVStore (JSON-only values).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import fastworkflow +from fastworkflow.workflow import enablecache + + +class _Host: + def __init__(self, folderpath: str): + self._folderpath = folderpath + + def get_cachedb_folderpath(self, function_name: str) -> str: + speedict_foldername = fastworkflow.get_env_var("SPEEDDICT_FOLDERNAME") + return str( + Path(self._folderpath) / speedict_foldername / f"function_cache/{function_name}" + ) + + @enablecache + def add(self, a: int, b: int) -> int: + return a + b + + @enablecache + def bad(self) -> object: + return object() + + +def test_enablecache_round_trip(tmp_path: Path, monkeypatch): + monkeypatch.setenv("SPEEDDICT_FOLDERNAME", "___workflow_contexts") + fastworkflow.init({"SPEEDDICT_FOLDERNAME": "___workflow_contexts"}) + host = _Host(str(tmp_path)) + assert host.add(1, 2) == 3 + assert host.add(1, 2) == 3 # cache hit + + +def test_enablecache_rejects_non_json(tmp_path: Path, monkeypatch): + monkeypatch.setenv("SPEEDDICT_FOLDERNAME", "___workflow_contexts") + fastworkflow.init({"SPEEDDICT_FOLDERNAME": "___workflow_contexts"}) + host = _Host(str(tmp_path)) + with pytest.raises(TypeError, match="JSON-serialisable"): + host.bad() diff --git a/tests/test_fastapi_memory_bounds.py b/tests/test_fastapi_memory_bounds.py index 69a4178..ec92de5 100644 --- a/tests/test_fastapi_memory_bounds.py +++ b/tests/test_fastapi_memory_bounds.py @@ -11,7 +11,7 @@ whole point: a turn is durably recorded before it can be dropped from memory, so windowing memory never shortens the durable record. -Everything runs against real runtimes, a real Rdict-backed conversation store and +Everything runs against real runtimes, a real SQLite-backed conversation store and the real turn engine. Turn bodies are plain callables rather than trained commands, so no model or LLM call is required. """ @@ -23,7 +23,6 @@ import importlib import json import os -import pickle import sys import time import uuid @@ -409,15 +408,15 @@ async def body(): ] -class _CountingRdict: - """Forwards to a real Rdict and tallies the bytes handed to it.""" +class _CountingKVStore: + """Forwards to a real KVStore and tallies the JSON bytes handed to it.""" def __init__(self, db, tally: dict): self._db = db self._tally = tally def __setitem__(self, key, value): - self._tally["bytes_written"] += len(pickle.dumps(value)) + self._tally["bytes_written"] += len(json.dumps(value).encode("utf-8")) self._tally["writes"] += 1 self._db[key] = value @@ -445,7 +444,7 @@ def __init__(self, channel_id: str, base_folder: str): self.tally = {"bytes_written": 0, "writes": 0} def _get_db(self): - return _CountingRdict(super()._get_db(), self.tally) + return _CountingKVStore(super()._get_db(), self.tally) def test_incremental_save_writes_only_the_new_turns(app_module, tmp_path): @@ -458,7 +457,7 @@ def test_incremental_save_writes_only_the_new_turns(app_module, tmp_path): """ turn_count = 40 turns = [_payload_turn(i) for i in range(turn_count)] - payload_bytes = sum(len(pickle.dumps(t)) for t in turns) + payload_bytes = sum(len(json.dumps(t).encode("utf-8")) for t in turns) appending = _CountingConversationStore("appending", str(tmp_path)) for turn in turns: @@ -616,33 +615,33 @@ async def check(): asyncio.run(check()) -def test_a_record_rewritten_by_an_older_version_is_read_intact(app_module, tmp_path): - """Turns moved out of the record; a downgrade-then-upgrade must not duplicate them. +def test_a_stale_inline_turns_field_is_ignored(app_module, tmp_path): + """After the sqlite migration, only per-turn keys are authoritative. - An older binary rewrites the inline list while leaving the per-turn - bookkeeping in place. Concatenating both would return each turn twice and out - of order. + Pre-migration RocksDB stores could keep an inline ``turns`` list. New + ``.sqlite3`` stores ignore that field so a poisoned inline list cannot + duplicate or reorder the durable per-turn entries. """ - from speedict import Rdict + from fastworkflow.kvstore import KVStore store = ConversationStore("rollback", str(tmp_path)) conv_id = store.reserve_next_conversation_id() for i in range(3): store.append_conversation_turns(conv_id, [_payload_turn(i, size_bytes=32)]) - db = Rdict(store.db_path) + db = KVStore(store.db_path) conv = db[f"conv:{conv_id}"] conv["turns"] = [_payload_turn(i, size_bytes=32) for i in range(4)] db[f"conv:{conv_id}"] = conv db.close() summaries = [t["conversation summary"] for t in store.get_conversation(conv_id)["turns"]] - assert summaries == [f"turn-{i}" for i in range(4)] - assert store.count_conversation_turns(conv_id) == 4 + assert summaries == [f"turn-{i}" for i in range(3)] + assert store.count_conversation_turns(conv_id) == 3 - store.append_conversation_turns(conv_id, [_payload_turn(4, size_bytes=32)]) + store.append_conversation_turns(conv_id, [_payload_turn(3, size_bytes=32)]) summaries = [t["conversation summary"] for t in store.get_conversation(conv_id)["turns"]] - assert summaries == [f"turn-{i}" for i in range(5)] + assert summaries == [f"turn-{i}" for i in range(4)] def test_summary_read_never_materializes_turn_payloads(app_module, tmp_path): diff --git a/tests/test_kvstore.py b/tests/test_kvstore.py new file mode 100644 index 0000000..9b94e87 --- /dev/null +++ b/tests/test_kvstore.py @@ -0,0 +1,137 @@ +"""Integration tests for the SQLite KVStore replacing speedict.""" + +from __future__ import annotations + +import multiprocessing +import os +from pathlib import Path + +import numpy as np +import pytest + +from fastworkflow.kvstore import KVStore, UtteranceCacheStore + + +def test_kvstore_round_trip_and_keyerror(tmp_path: Path): + path = str(tmp_path / "store.sqlite3") + with KVStore(path) as db: + db["meta"] = {"last": 1} + db["list"] = ["a", "b"] + assert db["meta"] == {"last": 1} + assert "list" in db + assert db.get("missing", 42) == 42 + del db["list"] + assert "list" not in db + with pytest.raises(KeyError): + del db["list"] + with pytest.raises(KeyError): + _ = db["list"] + + +def test_kvstore_coerces_int_keys(tmp_path: Path): + path = str(tmp_path / "intkeys.sqlite3") + with KVStore(path) as db: + db[0] = {"utterance": "hi", "label": "x"} + assert db.get(0)["utterance"] == "hi" + assert 0 in db + + +def test_kvstore_keys_materialised_for_concurrent_mutation(tmp_path: Path): + path = str(tmp_path / "keys.sqlite3") + with KVStore(path) as db: + for i in range(5): + db[f"k{i}"] = i + seen = [] + for key in db.keys(): + seen.append(key) + if key == "k0": + db["k_new"] = 99 + del db["k4"] + assert "k0" in seen + assert "k_new" in db + assert "k4" not in db + + +def test_kvstore_values_are_json_not_pickle(tmp_path: Path): + path = tmp_path / "nopickle.sqlite3" + with KVStore(str(path)) as db: + db["obj"] = {"class_name_marker": "DefinitelyNotPickle"} + raw = path.read_bytes() + assert b"\x80\x05" not in raw + assert b"DefinitelyNotPickle" in raw # plain JSON text + + +def test_utterance_cache_float32_blob_round_trip(tmp_path: Path): + path = str(tmp_path / "utt.sqlite3") + vec = np.arange(8, dtype=np.float32) + with UtteranceCacheStore(path) as store: + store.upsert( + "123", + utterance="hello", + command_mapping={"cmd": {"frequency": 1, "feedback_date": "t"}}, + embedding=vec, + ) + entry = store.get("123") + assert entry is not None + assert entry["utterance"] == "hello" + assert entry["command_mapping"]["cmd"]["frequency"] == 1 + np.testing.assert_array_equal(entry["embedding"], vec) + + entries = list(store.iter_entries()) + assert len(entries) == 1 + assert entries[0][0] == "123" + + +def test_kvstore_and_utterance_cache_share_file(tmp_path: Path): + path = str(tmp_path / "shared.sqlite3") + with KVStore(path) as kv: + kv["suggested_commands"] = ["a", "b"] + with UtteranceCacheStore(path) as utt: + utt.upsert( + "h1", + utterance="x", + command_mapping={"c": {"frequency": 1, "feedback_date": "t"}}, + embedding=np.ones(4, dtype=np.float32), + ) + with KVStore(path) as kv: + assert kv["suggested_commands"] == ["a", "b"] + with UtteranceCacheStore(path) as utt: + assert utt.get("h1")["utterance"] == "x" + + +def _mp_writer(path: str, worker_id: int, n: int, result_path: str) -> None: + try: + with KVStore(path, timeout=30.0) as db: + for i in range(n): + key = f"w{worker_id}:{i}" + db[key] = {"worker": worker_id, "i": i} + assert db[key]["worker"] == worker_id + Path(result_path).write_text("ok", encoding="utf-8") + except Exception as exc: # noqa: BLE001 — report failure to parent via file + Path(result_path).write_text(f"fail:{exc!r}", encoding="utf-8") + raise SystemExit(1) from exc + + +def test_kvstore_four_process_concurrency(tmp_path: Path): + path = str(tmp_path / "concurrent.sqlite3") + n_workers = 4 + n_ops = 50 + result_paths = [str(tmp_path / f"result_{i}.txt") for i in range(n_workers)] + procs = [ + multiprocessing.Process( + target=_mp_writer, args=(path, i, n_ops, result_paths[i]) + ) + for i in range(n_workers) + ] + for p in procs: + p.start() + for p in procs: + p.join(timeout=60) + assert p.exitcode == 0, f"worker exit {p.exitcode}" + + for rp in result_paths: + assert Path(rp).read_text(encoding="utf-8") == "ok" + + with KVStore(path) as db: + keys = list(db.keys()) + assert len(keys) == n_workers * n_ops From 5eea8f930cb742a54f02143f1887c4fb11689382 Mon Sep 17 00:00:00 2001 From: Dhar Rawal Date: Sat, 8 Aug 2026 18:55:11 -0500 Subject: [PATCH 2/2] fix: address sqlite KVStore review feedback Stream utterance_cache rows instead of fetchall, prefer context managers at call sites, rely on sqlite3.connect(timeout=) instead of interpolating PRAGMA busy_timeout, and harden concurrency/empty-embedding tests. Co-authored-by: Cursor --- .../intent_detection.py | 34 ++------- fastworkflow/cache_matching.py | 24 +++---- fastworkflow/kvstore.py | 47 +++++++------ .../run_fastapi_mcp/conversation_store.py | 70 ++++--------------- fastworkflow/workflow.py | 28 ++++---- tests/test_cache_matching_sqlite.py | 59 ++++++++++++++-- tests/test_fastapi_memory_bounds.py | 6 ++ tests/test_kvstore.py | 55 ++++++++++++++- 8 files changed, 183 insertions(+), 140 deletions(-) diff --git a/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py b/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py index daf2d87..e84e194 100644 --- a/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py +++ b/fastworkflow/_workflows/command_metadata_extraction/intent_detection.py @@ -217,13 +217,10 @@ def _store_suggested_commands(cache_path, command_list, flag_type): command_list: List of suggested commands flag_type: Type of constraint (1=ambiguous, 2=misclassified) """ - db = KVStore(cache_path) - try: + with KVStore(cache_path) as db: # predict() returns a numpy ndarray of labels; JSON needs plain strs. db["suggested_commands"] = [str(c) for c in list(command_list)] db["flag_type"] = int(flag_type) - finally: - db.close() # Get the suggested commands @staticmethod @@ -231,29 +228,20 @@ def _get_suggested_commands(cache_path): """ Get the list of suggested commands for the constrained selection """ - db = KVStore(cache_path) - try: + with KVStore(cache_path) as db: return db.get("suggested_commands", []) - finally: - db.close() @staticmethod def _get_count(cache_path): - db = KVStore(cache_path) - try: + with KVStore(cache_path) as db: return db.get("utterance_count", 0) # Default to 0 if key doesn't exist - finally: - db.close() @staticmethod def _print_db_contents(cache_path): - db = KVStore(cache_path) - try: + with KVStore(cache_path) as db: print("All keys in database:", list(db.keys())) for key in db.keys(): print(f"Key: {key}, Value: {db[key]}") - finally: - db.close() @staticmethod def _store_utterance(cache_path, utterance, label): @@ -261,10 +249,7 @@ def _store_utterance(cache_path, utterance, label): Store utterance in existing or new database Returns: The utterance count used """ - # Open the database (creates if doesn't exist) - db = KVStore(cache_path) - - try: + with KVStore(cache_path) as db: # Get existing counter or initialize to 0 utterance_count = db.get("utterance_count", 0) @@ -282,21 +267,14 @@ def _store_utterance(cache_path, utterance, label): return utterance_count - 1 # Return the count used for this utterance - finally: - # Always close the database - db.close() - # Function to read from database @staticmethod def _read_utterance(cache_path, utterance_id): """ Read a specific utterance from the database """ - db = KVStore(cache_path) - try: + with KVStore(cache_path) as db: return db.get(utterance_id)['utterance'] - finally: - db.close() @staticmethod def resolve_fully_qualified_command_name( command_name: Optional[str], command_name_dict: dict[str, str]) -> Optional[str]: diff --git a/fastworkflow/cache_matching.py b/fastworkflow/cache_matching.py index 4a58def..854e0da 100644 --- a/fastworkflow/cache_matching.py +++ b/fastworkflow/cache_matching.py @@ -63,8 +63,7 @@ def store_utterance_cache(cache_path, utterance, label, model_pipeline=None): Returns: The hash key of the stored utterance """ - db = UtteranceCacheStore(cache_path) - try: + with UtteranceCacheStore(cache_path) as db: # Generate hash for utterance using mmh3 utterance_hash = str(mmh3.hash(utterance)) @@ -114,9 +113,6 @@ def store_utterance_cache(cache_path, utterance, label, model_pipeline=None): ) return utterance_hash - - finally: - db.close() def get_embedding(text: str, model_pipeline): """Return (possibly cached) embedding for *text* using *model_pipeline*.""" @@ -145,24 +141,21 @@ def cache_match(cache_path, utterance, model_pipeline, threshold=0.90, return_de If match found: true_label or (true_label, similarity) if return_details=True If no match: None """ - db = UtteranceCacheStore(cache_path) - try: - entries = list(db.iter_entries()) - if not entries: - return None - + with UtteranceCacheStore(cache_path) as db: # Get embedding for the query utterance query_embedding = get_embedding(utterance, model_pipeline) # Reshape query embedding for cosine_similarity query_embedding = query_embedding.reshape(1, -1) - # Check cache for similar utterances + # Check cache for similar utterances (stream; do not materialise all rows) best_similarity = 0 best_key = None best_mapping = None + saw_any = False - for hash_key, entry in entries: + for hash_key, entry in db.iter_entries(): + saw_any = True cached_embedding = entry.get("embedding") if cached_embedding is None or cached_embedding.size == 0: continue @@ -175,6 +168,9 @@ def cache_match(cache_path, utterance, model_pipeline, threshold=0.90, return_de best_key = hash_key best_mapping = entry["command_mapping"] + if not saw_any: + return None + # If good cache match found, determine the best label if best_similarity >= threshold and best_key is not None and best_mapping is not None: command_mapping = best_mapping @@ -207,5 +203,3 @@ def cache_match(cache_path, utterance, model_pipeline, threshold=0.90, return_de return (true_label, best_similarity) if return_details else true_label # No good match found return None - finally: - db.close() diff --git a/fastworkflow/kvstore.py b/fastworkflow/kvstore.py index 9bc73d1..92096bd 100644 --- a/fastworkflow/kvstore.py +++ b/fastworkflow/kvstore.py @@ -21,19 +21,31 @@ def _key_str(key: Any) -> str: return key if isinstance(key, str) else str(key) +def _open_sqlite(path: str, *, timeout: float) -> sqlite3.Connection: + """Open a WAL connection. ``timeout`` is enforced by sqlite3.connect (seconds).""" + parent = os.path.dirname(path) + if parent: + os.makedirs(parent, exist_ok=True) + # check_same_thread=False is safe here because sqlite3 serialises access + # internally and every method below is a single self-contained statement. + # Busy waiting uses connect(timeout=...); do not interpolate into PRAGMA SQL. + conn = sqlite3.connect(path, timeout=timeout, check_same_thread=False) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + return conn + + class KVStore: - """A durable dict[str, Any]. Values must be JSON-serialisable.""" + """A durable dict[str, Any]. Values must be JSON-serialisable. + + Values are stored as JSON TEXT. That matches today's call sites (scalars, + small dicts, conversation turns). Large binary or high-cardinality payloads + should use a dedicated table with typed columns / BLOBs (see + :class:`UtteranceCacheStore`) rather than stuffing them into ``v``. + """ def __init__(self, path: str, *, timeout: float = 30.0) -> None: - parent = os.path.dirname(path) - if parent: - os.makedirs(parent, exist_ok=True) - # check_same_thread=False is safe here because sqlite3 serialises access - # internally and every method below is a single self-contained statement. - self._conn = sqlite3.connect(path, timeout=timeout, check_same_thread=False) - self._conn.execute("PRAGMA journal_mode=WAL") - self._conn.execute("PRAGMA synchronous=NORMAL") - self._conn.execute(f"PRAGMA busy_timeout={int(timeout * 1000)}") + self._conn = _open_sqlite(path, timeout=timeout) self._conn.execute( "CREATE TABLE IF NOT EXISTS kv (k TEXT PRIMARY KEY, v TEXT NOT NULL)" ) @@ -98,13 +110,7 @@ class UtteranceCacheStore: """ def __init__(self, path: str, *, timeout: float = 30.0) -> None: - parent = os.path.dirname(path) - if parent: - os.makedirs(parent, exist_ok=True) - self._conn = sqlite3.connect(path, timeout=timeout, check_same_thread=False) - self._conn.execute("PRAGMA journal_mode=WAL") - self._conn.execute("PRAGMA synchronous=NORMAL") - self._conn.execute(f"PRAGMA busy_timeout={int(timeout * 1000)}") + self._conn = _open_sqlite(path, timeout=timeout) self._conn.execute( """ CREATE TABLE IF NOT EXISTS utterance_cache ( @@ -166,10 +172,11 @@ def upsert( self._conn.commit() def iter_entries(self) -> Iterator[tuple[str, dict[str, Any]]]: - rows = self._conn.execute( + # Stream rows from the cursor — do not fetchall(); cache_match only + # needs the best match and must not hold every embedding in memory. + for key, meta_json, vec in self._conn.execute( "SELECT k, meta, vec FROM utterance_cache" - ).fetchall() - for key, meta_json, vec in rows: + ): meta = json.loads(meta_json) yield key, { "utterance": meta.get("utterance", ""), diff --git a/fastworkflow/run_fastapi_mcp/conversation_store.py b/fastworkflow/run_fastapi_mcp/conversation_store.py index 5ca1e31..8f70776 100644 --- a/fastworkflow/run_fastapi_mcp/conversation_store.py +++ b/fastworkflow/run_fastapi_mcp/conversation_store.py @@ -104,12 +104,9 @@ def _replace_turns( def get_last_conversation_id(self) -> Optional[int]: """Get the last conversation ID for this user""" - try: - db = self._get_db() + with self._get_db() as db: meta = db.get("meta", {}) return meta.get("last_conversation_id") - finally: - db.close() def _increment_conversation_id(self, db: KVStore) -> int: """Increment and return new conversation ID""" @@ -121,11 +118,8 @@ def _increment_conversation_id(self, db: KVStore) -> int: def reserve_next_conversation_id(self) -> int: """Reserve the next conversation ID by incrementing the counter without creating a conversation""" - db = self._get_db() - try: + with self._get_db() as db: return self._increment_conversation_id(db) - finally: - db.close() def _ensure_unique_topic(self, db: KVStore, candidate_topic: str) -> str: """Ensure topic is unique per user with case/whitespace insensitive comparison""" @@ -169,8 +163,7 @@ def save_conversation( Returns: The conversation ID used """ - db = self._get_db() - try: + with self._get_db() as db: if conversation_id is not None: # Use the specified ID (assumes it's valid and reserved) conv_id = conversation_id @@ -190,22 +183,16 @@ def save_conversation( self._replace_turns(db, conv_id, conversation, turns) db[f"conv:{conv_id}"] = conversation return conv_id - finally: - db.close() def get_conversation(self, conv_id: int) -> Optional[dict[str, Any]]: """Get a conversation by ID""" - db = self._get_db() - try: + with self._get_db() as db: conv = db.get(f"conv:{conv_id}") return None if conv is None else self._hydrated(db, conv_id, conv) - finally: - db.close() def get_conversation_by_topic(self, topic: str) -> Optional[tuple[int, dict[str, Any]]]: """Get conversation ID and data by topic (case/whitespace insensitive)""" - db = self._get_db() - try: + with self._get_db() as db: meta = db.get("meta", {"last_conversation_id": 0}) normalized_topic = topic.lower().strip() @@ -216,13 +203,10 @@ def get_conversation_by_topic(self, topic: str) -> Optional[tuple[int, dict[str, if conv.get("topic", "").lower().strip() == normalized_topic: return i, self._hydrated(db, i, conv) return None - finally: - db.close() def list_conversations(self, limit: int) -> list[ConversationSummary]: """List conversations ordered by updated_at desc, up to limit""" - db = self._get_db() - try: + with self._get_db() as db: meta = db.get("meta", {"last_conversation_id": 0}) conversations = [] @@ -243,8 +227,6 @@ def list_conversations(self, limit: int) -> list[ConversationSummary]: # Sort by updated_at desc and limit conversations.sort(key=lambda c: c.updated_at, reverse=True) return conversations[:limit] - finally: - db.close() def update_conversation( self, @@ -254,8 +236,7 @@ def update_conversation( turns: list[dict[str, Any]] ) -> None: """Update an existing conversation with new topic, summary, and turns""" - db = self._get_db() - try: + with self._get_db() as db: conv_key = f"conv:{conv_id}" if conv_key not in db: raise ValueError(f"Conversation {conv_id} not found") @@ -270,8 +251,6 @@ def update_conversation( self._replace_turns(db, conv_id, conv, turns) db[conv_key] = conv - finally: - db.close() def update_conversation_topic_summary( self, @@ -283,8 +262,7 @@ def update_conversation_topic_summary( Update only the topic and summary of an existing conversation. Used when finalizing a conversation (turns already saved incrementally). """ - db = self._get_db() - try: + with self._get_db() as db: conv_key = f"conv:{conv_id}" if conv_key not in db: raise ValueError(f"Conversation {conv_id} not found") @@ -298,8 +276,6 @@ def update_conversation_topic_summary( conv["updated_at"] = int(time.time() * 1000) db[conv_key] = conv - finally: - db.close() def save_conversation_turns( self, @@ -320,8 +296,7 @@ def save_conversation_turns( Returns: The conversation ID used """ - db = self._get_db() - try: + with self._get_db() as db: conv_key = f"conv:{conversation_id}" if conv_key in db: @@ -341,8 +316,6 @@ def save_conversation_turns( db[conv_key] = conv return conversation_id - finally: - db.close() def append_conversation_turns( self, @@ -367,8 +340,7 @@ def append_conversation_turns( if not new_turns: return conversation_id - db = self._get_db() - try: + with self._get_db() as db: conv_key = f"conv:{conversation_id}" now = int(time.time() * 1000) @@ -393,19 +365,14 @@ def append_conversation_turns( db[conv_key] = conv return conversation_id - finally: - db.close() def count_conversation_turns(self, conversation_id: int) -> int: """Number of turns durably recorded for a conversation (0 if absent).""" - db = self._get_db() - try: + with self._get_db() as db: conv = db.get(f"conv:{conversation_id}") if conv is None: return 0 return int(conv.get("appended_turn_count") or 0) - finally: - db.close() def get_conversation_summaries(self, conversation_id: int) -> list[dict[str, Any]]: """Each turn's summary, without holding whole turns in memory. @@ -415,8 +382,7 @@ def get_conversation_summaries(self, conversation_id: int) -> list[dict[str, Any size of the whole conversation, which is the growth this store exists to keep out of the process. """ - db = self._get_db() - try: + with self._get_db() as db: conv = db.get(f"conv:{conversation_id}") if conv is None: return [] @@ -424,8 +390,6 @@ def get_conversation_summaries(self, conversation_id: int) -> list[dict[str, Any {"conversation summary": turn.get("conversation summary")} for turn in self._iter_turn_records(db, conversation_id, conv) ] - finally: - db.close() # NOTE: update_turn_feedback() removed - feedback is saved from the incremental # save flow after modifying conversation_history in memory, via the append path @@ -443,8 +407,7 @@ def update_last_conversation_turn( Used when a turn that is already recorded is edited (feedback), which the append path cannot express. Returns False if there is no turn to rewrite. """ - db = self._get_db() - try: + with self._get_db() as db: conv_key = f"conv:{conversation_id}" if conv_key not in db: return False @@ -458,13 +421,10 @@ def update_last_conversation_turn( conv["updated_at"] = int(time.time() * 1000) db[conv_key] = conv return True - finally: - db.close() def get_all_conversations_for_dump(self) -> list[dict[str, Any]]: """Get all conversations for admin dump""" - db = self._get_db() - try: + with self._get_db() as db: meta = db.get("meta", {"last_conversation_id": 0}) conversations = [] @@ -478,8 +438,6 @@ def get_all_conversations_for_dump(self) -> list[dict[str, Any]]: }) return conversations - finally: - db.close() def generate_topic_and_summary(turns: list[dict[str, Any]]) -> tuple[str, str]: diff --git a/fastworkflow/workflow.py b/fastworkflow/workflow.py index befb43f..757f63a 100644 --- a/fastworkflow/workflow.py +++ b/fastworkflow/workflow.py @@ -55,22 +55,20 @@ def wrapper(self, *args, **kwargs): # Get the cache database (folder path historically fed RocksDB; SQLite # needs a file inside that folder). cache_db_folder = self.get_cachedb_folderpath(func.__name__) - cache_db = KVStore(os.path.join(cache_db_folder, "cache.sqlite3")) - - if key not in cache_db: - # If the result is not in the cache, call the function and store the result - result = func(self, *args, **kwargs) - try: - cache_db[key] = result - except TypeError as exc: - raise TypeError( - f"@enablecache requires a JSON-serialisable return value; " - f"{func.__qualname__} returned {type(result).__name__}" - ) from exc - else: - result = cache_db[key] + with KVStore(os.path.join(cache_db_folder, "cache.sqlite3")) as cache_db: + if key not in cache_db: + # If the result is not in the cache, call the function and store the result + result = func(self, *args, **kwargs) + try: + cache_db[key] = result + except TypeError as exc: + raise TypeError( + f"@enablecache requires a JSON-serialisable return value; " + f"{func.__qualname__} returned {type(result).__name__}" + ) from exc + else: + result = cache_db[key] - cache_db.close() return result return wrapper diff --git a/tests/test_cache_matching_sqlite.py b/tests/test_cache_matching_sqlite.py index 48c790e..89dd486 100644 --- a/tests/test_cache_matching_sqlite.py +++ b/tests/test_cache_matching_sqlite.py @@ -7,7 +7,7 @@ import numpy as np from fastworkflow.cache_matching import cache_match -from fastworkflow.kvstore import UtteranceCacheStore +from fastworkflow.kvstore import KVStore, UtteranceCacheStore class _FakePipeline: @@ -15,7 +15,7 @@ class _FakePipeline: def test_store_and_match_via_blob_rows(tmp_path: Path, monkeypatch): - path = str(tmp_path / "cache.db") + path = str(tmp_path / "cache.sqlite3") vec_a = np.ones(8, dtype=np.float32) vec_b = np.zeros(8, dtype=np.float32) vec_b[0] = 1.0 @@ -45,7 +45,58 @@ def fake_get_embedding(text, model_pipeline): assert label == "cmd_a" # Mechanical JSON whole-cache key must not exist - from fastworkflow.kvstore import KVStore - with KVStore(path) as kv: assert kv.get("cache") is None + + +def test_cache_match_skips_none_and_empty_embeddings(tmp_path: Path, monkeypatch): + path = str(tmp_path / "cache_skip.sqlite3") + with UtteranceCacheStore(path) as store: + store.upsert( + "none", + utterance="no-vec", + command_mapping={"cmd_none": {"frequency": 1, "feedback_date": "t"}}, + embedding=None, + ) + store.upsert( + "empty", + utterance="empty-vec", + command_mapping={"cmd_empty": {"frequency": 1, "feedback_date": "t"}}, + embedding=np.array([], dtype=np.float32), + ) + + def fake_get_embedding(text, model_pipeline): + return np.ones(8, dtype=np.float32).reshape(1, -1) + + monkeypatch.setattr( + "fastworkflow.cache_matching.get_embedding", fake_get_embedding + ) + # Entries without usable embeddings must be skipped: no match, no raise. + assert cache_match(path, "query", _FakePipeline(), threshold=0.5) is None + + +def test_cache_match_ignores_missing_vecs_when_a_real_match_exists( + tmp_path: Path, monkeypatch +): + path = str(tmp_path / "cache_mixed.sqlite3") + with UtteranceCacheStore(path) as store: + store.upsert( + "none", + utterance="no-vec", + command_mapping={"cmd_none": {"frequency": 99, "feedback_date": "t"}}, + embedding=None, + ) + store.upsert( + "real", + utterance="alpha", + command_mapping={"cmd_a": {"frequency": 1, "feedback_date": "t"}}, + embedding=np.ones(8, dtype=np.float32), + ) + + def fake_get_embedding(text, model_pipeline): + return np.ones(8, dtype=np.float32).reshape(1, -1) + + monkeypatch.setattr( + "fastworkflow.cache_matching.get_embedding", fake_get_embedding + ) + assert cache_match(path, "query", _FakePipeline(), threshold=0.5) == "cmd_a" diff --git a/tests/test_fastapi_memory_bounds.py b/tests/test_fastapi_memory_bounds.py index ec92de5..ea0abc0 100644 --- a/tests/test_fastapi_memory_bounds.py +++ b/tests/test_fastapi_memory_bounds.py @@ -435,6 +435,12 @@ def get(self, key, default=None): def close(self): self._db.close() + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + class _CountingConversationStore(ConversationStore): """A real store whose write volume can be measured.""" diff --git a/tests/test_kvstore.py b/tests/test_kvstore.py index 9b94e87..945c811 100644 --- a/tests/test_kvstore.py +++ b/tests/test_kvstore.py @@ -3,7 +3,6 @@ from __future__ import annotations import multiprocessing -import os from pathlib import Path import numpy as np @@ -82,6 +81,50 @@ def test_utterance_cache_float32_blob_round_trip(tmp_path: Path): assert entries[0][0] == "123" +def test_utterance_cache_none_embedding_round_trip(tmp_path: Path): + path = str(tmp_path / "utt_none.sqlite3") + with UtteranceCacheStore(path) as store: + store.upsert( + "none-emb", + utterance="hello-none", + command_mapping={"cmd": {"frequency": 1, "feedback_date": "t"}}, + embedding=None, + ) + entry = store.get("none-emb") + assert entry is not None + assert entry["utterance"] == "hello-none" + assert entry["command_mapping"]["cmd"]["frequency"] == 1 + assert entry["embedding"] is None + + entries = list(store.iter_entries()) + assert len(entries) == 1 + assert entries[0][0] == "none-emb" + assert entries[0][1]["embedding"] is None + + +def test_utterance_cache_empty_embedding_round_trip(tmp_path: Path): + path = str(tmp_path / "utt_empty.sqlite3") + empty_vec = np.array([], dtype=np.float32) + with UtteranceCacheStore(path) as store: + store.upsert( + "empty-emb", + utterance="hello-empty", + command_mapping={"cmd": {"frequency": 1, "feedback_date": "t"}}, + embedding=empty_vec, + ) + entry = store.get("empty-emb") + assert entry is not None + assert entry["utterance"] == "hello-empty" + assert entry["command_mapping"]["cmd"]["frequency"] == 1 + # Empty vectors should round-trip to a None embedding, not an empty ndarray + assert entry["embedding"] is None + + entries = list(store.iter_entries()) + assert len(entries) == 1 + assert entries[0][0] == "empty-emb" + assert entries[0][1]["embedding"] is None + + def test_kvstore_and_utterance_cache_share_file(tmp_path: Path): path = str(tmp_path / "shared.sqlite3") with KVStore(path) as kv: @@ -134,4 +177,12 @@ def test_kvstore_four_process_concurrency(tmp_path: Path): with KVStore(path) as db: keys = list(db.keys()) - assert len(keys) == n_workers * n_ops + assert len(keys) == n_workers * n_ops + for key in keys: + worker_s, i_s = key.split(":", 1) + worker_id = int(worker_s[1:]) + i = int(i_s) + assert db[key] == {"worker": worker_id, "i": i} + for worker_id in range(n_workers): + for i in range(n_ops): + assert db[f"w{worker_id}:{i}"] == {"worker": worker_id, "i": i}