diff --git a/src/terse/fluency.py b/src/terse/fluency.py index 49b3e9a..ed57b9f 100644 --- a/src/terse/fluency.py +++ b/src/terse/fluency.py @@ -41,7 +41,7 @@ from . import text_diff from .capture import LONG_TEXT, OTHER, classify_shape, extract_records from .tokenize import count_cl100k -from .transforms import compress, compress_structure, dict_encode, diff_wire, minify +from .transforms import compress, compress_structure, dict_encode, diff_wire, minify, _uniform_dict_list # Loopback hosts where cleartext http is safe (never leaves the machine), so a Bearer # key over http to one of these is fine — a local LiteLLM/CCR gateway is a common setup. @@ -144,12 +144,125 @@ def _pick_numeric_col(records: list[dict], cols: list[str], exclude: str | None return fallback +def _intersection_cols(records: list[dict]) -> list[str]: + """Keys present in EVERY record, sorted for determinism. For a non-uniform record + list (e.g. structure symbols, where only some carry line/hash) these are the only + columns safe to index across all records.""" + common = set(records[0].keys()) + for r in records[1:]: + common &= set(r.keys()) + return sorted(common) + + +def _nested_record_group(obj: Any) -> tuple[str, list[dict], list[str]] | None: + """Reach a record list that terse's STRICT uniform extractor skips: a dict-map of + parent records each holding a child list of dicts (runecho.structure's + `files{path: {symbols: [...]}}`), where the child list is non-uniform (symbol kinds + carry different keys). Returns (label, records, common_cols) deterministically — first + match in source order, first parent in map order — else None. + + Fluency-local by design: it does NOT touch `extract_records`/`_uniform_dict_list`, + which the tabularizer, probe, and drop-path logic (#47) share — widening their notion + of a record would change what the codec folds. This only widens what the fluency + harness can ASK about, so `proxy --diff` gets exercised on structure-shaped output + (issue #71). + + Preferred OVER the uniform extractor for the dict-map case: an unscoped "how many + records" is ambiguous when the payload holds many groups, and `extract_records` would + otherwise return whichever group's child list happens to be uniform — a valid list but + the wrong (unlabelled) question. So group-scoped questions win when a dict-map is + present, regardless of any single group's uniformity.""" + if not isinstance(obj, dict): + return None + + def _records_of(lst: Any) -> list[str] | None: + if not isinstance(lst, list) or len(lst) < 2: + return None + if not all(isinstance(x, dict) for x in lst): + return None + return _intersection_cols(lst) or None + + for k, v in obj.items(): + # dict-map of parent records -> the first parent (map order) with a child record list + if isinstance(v, dict) and v and all(isinstance(pv, dict) for pv in v.values()): + for pkey, parent in v.items(): + for ck, cv in parent.items(): + if _records_of(cv): + return f"{k}[{json.dumps(pkey, ensure_ascii=False)}]", cv, _intersection_cols(cv) + # a NON-uniform top-level list of dicts (uniform ones are extract_records' domain) + if isinstance(v, list) and _records_of(v) and not _uniform_dict_list(v): + return k, v, _intersection_cols(v) + return None + + +def _nested_questions(obj: Any) -> list[Question]: + """Questions for structure-shaped payloads (dict-map of records with non-uniform child + lists) that `gen_questions`' uniform path can't reach — count/enumerate/lookup over the + columns shared by every record, plus aggregate if a shared numeric column exists. Same + deterministic, programmatically-checkable contract as `gen_questions` (issue #71).""" + grp = _nested_record_group(obj) + if grp is None: + return [] + label, records, cols = grp + n = len(records) + qs: list[Question] = [Question( + "count", "count", "nested", + f"How many records are listed under {label}?", + "Reply with only the integer count.", n)] + + # enumerate lists a column in order — duplicates are fine (order/count is the check). + # Use the most-distinct string column (most informative); deterministic — ties resolve + # to sorted-column order via max()'s stable first-max. + str_cols = [c for c in cols if all(isinstance(r[c], str) for r in records)] + if str_cols: + enum_col = max(str_cols, key=lambda c: len({r[c] for r in records})) + qs.append(Question( + "enumerate", "enumerate", "nested", + f"List the {enum_col!r} of every record under {label}, in order.", + "Reply with a JSON array of the values and nothing else.", + [r[enum_col] for r in records])) + + # lookup needs a column that UNIQUELY addresses one record — otherwise the prompt is + # ambiguous and a truthful answer about a different matching record scores wrong. Reuse + # the uniform path's uniqueness rule (`_pick_id_col`); skip lookup when none is unique + # (common in structure: `kind` and even overloaded `name` repeat within a file). + idcol = _pick_id_col(records, cols) + if idcol is not None: + tgt = next((c for c in cols if c != idcol), None) + if tgt is not None: + ri = n // 2 # idcol is unique, so any index gives an unambiguous prompt + qs.append(Question( + "lookup", "lookup", "nested", + f"Under {label}, for the record whose {idcol!r} is " + f"{json.dumps(records[ri][idcol], ensure_ascii=False)}, " + f"what is the value of {tgt!r}?", + "Reply with only the value, with no quotes and no extra words.", + records[ri][tgt])) + + numcol = next((c for c in cols if all(_is_number(r[c]) for r in records)), None) + if numcol is not None: + qs.append(Question( + "aggregate", "aggregate", "nested", + f"What is the maximum value of {numcol!r} across the records under {label}?", + "Reply with only the number.", + max(r[numcol] for r in records))) + return qs + + def gen_questions(obj: Any) -> list[Question]: """Generate deterministic, programmatically-checkable questions for a payload. - Only record-shaped payloads (what terse transforms) yield questions; everything - else returns []. Selection is fully deterministic so a re-run is reproducible. + Uniform record-shaped payloads (what terse tabularizes) yield questions directly; + payloads whose records the strict extractor skips (structure's dict-map of non-uniform + symbol lists) fall through to `_nested_questions` (#71); everything else returns []. + Selection is fully deterministic so a re-run is reproducible. """ + # Structure-shaped payloads (a dict-map of records) need GROUP-SCOPED questions; prefer + # them over the uniform extractor, which would otherwise emit an unscoped, ambiguous + # question from whichever group's child list happens to be uniform (#71). + nested = _nested_questions(obj) + if nested: + return nested records = extract_records(obj) if not records: return [] diff --git a/tests/test_fluency.py b/tests/test_fluency.py index fc6d4cc..d06db2d 100644 --- a/tests/test_fluency.py +++ b/tests/test_fluency.py @@ -5,6 +5,8 @@ from __future__ import annotations +import json + import pytest from terse import fluency @@ -82,6 +84,109 @@ def test_no_questions_for_non_record_payloads(): assert fluency.gen_questions("a string") == [] +# A runecho.structure-shaped payload (#71): `files` is a dict-map of file records, each +# holding a NON-UNIFORM `symbols` list — imports carry only name/kind, functions also carry +# line/hash. terse's strict identical-keyset extractor skips this, so gen_questions must +# fall through to _nested_questions, scoping to the first file and its intersection columns. +STRUCTURE = { + "detail": "symbols", + "file_count": 2, + "files": { + "a/first.py": { + "hash": "h0", + "symbols": [ + {"name": "alpha", "kind": "function", "line": 10, "hash": "x1"}, + {"name": "beta", "kind": "function", "line": 20, "hash": "x2"}, + {"name": "os", "kind": "import"}, # non-uniform: no line/hash + ], + }, + "b/second.py": { + "hash": "h1", + "symbols": [ + {"name": "gamma", "kind": "class", "line": 5, "hash": "y1"}, + {"name": "delta", "kind": "function", "line": 8, "hash": "y2"}, + ], + }, + }, + "repo": "demo", +} + + +def test_structure_uses_group_scoped_nested_questions(): + # Even though b/second.py's symbols ARE uniform (so extract_records finds a list), + # a dict-map payload must use GROUP-SCOPED questions — an unscoped count would be + # ambiguous across files. Group-scoping wins over the uniform extractor (#71). + qs = {q.qtype: q for q in fluency.gen_questions(STRUCTURE)} + assert set(qs) == {"count", "enumerate", "lookup"} + # scoped to the first file (map order), its 3 non-uniform symbols + assert qs["count"].expected == 3 + assert qs["enumerate"].expected == ["alpha", "beta", "os"] # 'name' = most-distinct id col + assert qs["lookup"].expected == "function" # alpha's kind + # no aggregate: 'line' is absent from the import symbol, so it isn't an intersection col + assert "aggregate" not in qs + for q in qs.values(): + assert 'files["a/first.py"]' in q.prompt + + +def test_nested_group_uses_intersection_columns_only(): + grp = fluency._nested_record_group(STRUCTURE) + assert grp is not None + label, records, cols = grp + assert label == 'files["a/first.py"]' + assert cols == ["kind", "name"] # sorted intersection; line/hash excluded (non-uniform) + assert len(records) == 3 + + +def test_nested_questions_are_self_consistent(): + for q in fluency.gen_questions(STRUCTURE): + reply = q.expected if isinstance(q.expected, str) else json.dumps(q.expected) + assert fluency.score(q.qtype, q.expected, reply) + + +def test_nested_lookup_skipped_when_no_column_uniquely_addresses_a_record(): + # Both `name` and `kind` repeat within the file, so no column uniquely identifies a + # record — a lookup prompt would be ambiguous and a truthful answer about a different + # matching record would score as a false-negative regression. lookup must be OMITTED; + # count + enumerate (which tolerate duplicates) still fire. + obj = {"files": {"x.py": {"symbols": [ + {"name": "X", "kind": "function"}, + {"name": "Y", "kind": "function"}, + {"name": "X", "kind": "class"}, + {"name": "Y", "kind": "class"}]}}} + qs = {q.qtype: q for q in fluency.gen_questions(obj)} + assert "lookup" not in qs + assert qs["count"].expected == 4 + assert "enumerate" in qs + # the enumerate ground truth stays exactly checkable even with repeated values + assert fluency.score("enumerate", qs["enumerate"].expected, + json.dumps(qs["enumerate"].expected)) + + +def test_nested_aggregate_appears_when_a_numeric_col_is_shared(): + # every symbol carries `line` (only `hash` varies) -> line is an intersection col -> aggregate + obj = {"files": {"f": {"symbols": [ + {"name": "a", "kind": "fn", "line": 3, "hash": "h"}, + {"name": "b", "kind": "fn", "line": 9}, # no hash -> still non-uniform + {"name": "c", "kind": "var", "line": 1, "hash": "h2"}]}}} + from terse.capture import extract_records + assert extract_records(obj) is None + qs = {q.qtype: q for q in fluency.gen_questions(obj)} + assert qs["aggregate"].expected == 9 + + +def test_run_diff_payload_now_exercises_structure_pairs(): + # the #71 payoff: a structure diff yields the same questions in both forms, so + # `terse fluency --diff` can finally measure structure comprehension. + curr = json.loads(json.dumps(STRUCTURE)) + curr["files"]["a/first.py"]["symbols"].append( + {"name": "epsilon", "kind": "function", "line": 40, "hash": "x9"}) + rows = fluency.run_diff_payload(STRUCTURE, curr, lambda s, u: "", + tool="runecho.structure", trials=1) + assert rows # non-empty: structure now generates questions -> diff is testable + assert {r["qid"] for r in rows} >= {"count", "enumerate"} + assert all("terse_ok" in r and "diff_ok" in r for r in rows) + + def test_score_count_and_aggregate_tolerate_prose_and_check_value(): assert fluency.score("count", 4, "There are 4 records.") assert fluency.score("count", 4, "4")