fluency: cover nested-record tools (structure) so proxy --diff is validated (#71)

src/terse/fluency.py

@@ -41,7 +41,7 @@
 from . import text_diff
 from .capture import LONG_TEXT, OTHER, classify_shape, extract_records
 from .tokenize import count_cl100k
-from .transforms import compress, compress_structure, dict_encode, diff_wire, minify
+from .transforms import compress, compress_structure, dict_encode, diff_wire, minify, _uniform_dict_list
 
 # Loopback hosts where cleartext http is safe (never leaves the machine), so a Bearer
 # key over http to one of these is fine — a local LiteLLM/CCR gateway is a common setup.
@@ -144,12 +144,125 @@ def _pick_numeric_col(records: list[dict], cols: list[str], exclude: str | None
     return fallback
 
 
+def _intersection_cols(records: list[dict]) -> list[str]:
+    """Keys present in EVERY record, sorted for determinism. For a non-uniform record
+    list (e.g. structure symbols, where only some carry line/hash) these are the only
+    columns safe to index across all records."""
+    common = set(records[0].keys())
+    for r in records[1:]:
+        common &= set(r.keys())
+    return sorted(common)
+
+
+def _nested_record_group(obj: Any) -> tuple[str, list[dict], list[str]] | None:
+    """Reach a record list that terse's STRICT uniform extractor skips: a dict-map of
+    parent records each holding a child list of dicts (runecho.structure's
+    `files{path: {symbols: [...]}}`), where the child list is non-uniform (symbol kinds
+    carry different keys). Returns (label, records, common_cols) deterministically — first
+    match in source order, first parent in map order — else None.
+
+    Fluency-local by design: it does NOT touch `extract_records`/`_uniform_dict_list`,
+    which the tabularizer, probe, and drop-path logic (#47) share — widening their notion
+    of a record would change what the codec folds. This only widens what the fluency
+    harness can ASK about, so `proxy --diff` gets exercised on structure-shaped output
+    (issue #71).
+
+    Preferred OVER the uniform extractor for the dict-map case: an unscoped "how many
+    records" is ambiguous when the payload holds many groups, and `extract_records` would
+    otherwise return whichever group's child list happens to be uniform — a valid list but
+    the wrong (unlabelled) question. So group-scoped questions win when a dict-map is
+    present, regardless of any single group's uniformity."""
+    if not isinstance(obj, dict):
+        return None
+
+    def _records_of(lst: Any) -> list[str] | None:
+        if not isinstance(lst, list) or len(lst) < 2:
+            return None
+        if not all(isinstance(x, dict) for x in lst):
+            return None
+        return _intersection_cols(lst) or None
+
+    for k, v in obj.items():
+        # dict-map of parent records -> the first parent (map order) with a child record list
+        if isinstance(v, dict) and v and all(isinstance(pv, dict) for pv in v.values()):
+            for pkey, parent in v.items():
+                for ck, cv in parent.items():
+                    if _records_of(cv):
+                        return f"{k}[{json.dumps(pkey, ensure_ascii=False)}]", cv, _intersection_cols(cv)
+        # a NON-uniform top-level list of dicts (uniform ones are extract_records' domain)
+        if isinstance(v, list) and _records_of(v) and not _uniform_dict_list(v):
+            return k, v, _intersection_cols(v)
+    return None
+
+
+def _nested_questions(obj: Any) -> list[Question]:
+    """Questions for structure-shaped payloads (dict-map of records with non-uniform child
+    lists) that `gen_questions`' uniform path can't reach — count/enumerate/lookup over the
+    columns shared by every record, plus aggregate if a shared numeric column exists. Same
+    deterministic, programmatically-checkable contract as `gen_questions` (issue #71)."""
+    grp = _nested_record_group(obj)
+    if grp is None:
+        return []
+    label, records, cols = grp
+    n = len(records)
+    qs: list[Question] = [Question(
+        "count", "count", "nested",
+        f"How many records are listed under {label}?",
+        "Reply with only the integer count.", n)]
+
+    # enumerate lists a column in order — duplicates are fine (order/count is the check).
+    # Use the most-distinct string column (most informative); deterministic — ties resolve
+    # to sorted-column order via max()'s stable first-max.
+    str_cols = [c for c in cols if all(isinstance(r[c], str) for r in records)]
+    if str_cols:
+        enum_col = max(str_cols, key=lambda c: len({r[c] for r in records}))
+        qs.append(Question(
+            "enumerate", "enumerate", "nested",
+            f"List the {enum_col!r} of every record under {label}, in order.",
+            "Reply with a JSON array of the values and nothing else.",
+            [r[enum_col] for r in records]))
+
+    # lookup needs a column that UNIQUELY addresses one record — otherwise the prompt is
+    # ambiguous and a truthful answer about a different matching record scores wrong. Reuse
+    # the uniform path's uniqueness rule (`_pick_id_col`); skip lookup when none is unique
+    # (common in structure: `kind` and even overloaded `name` repeat within a file).
+    idcol = _pick_id_col(records, cols)
+    if idcol is not None:
+        tgt = next((c for c in cols if c != idcol), None)
+        if tgt is not None:
+            ri = n // 2  # idcol is unique, so any index gives an unambiguous prompt
+            qs.append(Question(
+                "lookup", "lookup", "nested",
+                f"Under {label}, for the record whose {idcol!r} is "
+                f"{json.dumps(records[ri][idcol], ensure_ascii=False)}, "
+                f"what is the value of {tgt!r}?",
+                "Reply with only the value, with no quotes and no extra words.",
+                records[ri][tgt]))
+
+    numcol = next((c for c in cols if all(_is_number(r[c]) for r in records)), None)
+    if numcol is not None:
+        qs.append(Question(
+            "aggregate", "aggregate", "nested",
+            f"What is the maximum value of {numcol!r} across the records under {label}?",
+            "Reply with only the number.",
+            max(r[numcol] for r in records)))
+    return qs
+
+
 def gen_questions(obj: Any) -> list[Question]:
     """Generate deterministic, programmatically-checkable questions for a payload.
 
-    Only record-shaped payloads (what terse transforms) yield questions; everything
-    else returns []. Selection is fully deterministic so a re-run is reproducible.
+    Uniform record-shaped payloads (what terse tabularizes) yield questions directly;
+    payloads whose records the strict extractor skips (structure's dict-map of non-uniform
+    symbol lists) fall through to `_nested_questions` (#71); everything else returns [].
+    Selection is fully deterministic so a re-run is reproducible.
     """
+    # Structure-shaped payloads (a dict-map of records) need GROUP-SCOPED questions; prefer
+    # them over the uniform extractor, which would otherwise emit an unscoped, ambiguous
+    # question from whichever group's child list happens to be uniform (#71).
+    nested = _nested_questions(obj)
+    if nested:
+        return nested
     records = extract_records(obj)
     if not records:
         return []

tests/test_fluency.py

@@ -5,6 +5,8 @@
 
 from __future__ import annotations
 
+import json
+
 import pytest
 
 from terse import fluency
@@ -82,6 +84,109 @@ def test_no_questions_for_non_record_payloads():
     assert fluency.gen_questions("a string") == []
 
 
+# A runecho.structure-shaped payload (#71): `files` is a dict-map of file records, each
+# holding a NON-UNIFORM `symbols` list — imports carry only name/kind, functions also carry
+# line/hash. terse's strict identical-keyset extractor skips this, so gen_questions must
+# fall through to _nested_questions, scoping to the first file and its intersection columns.
+STRUCTURE = {
+    "detail": "symbols",
+    "file_count": 2,
+    "files": {
+        "a/first.py": {
+            "hash": "h0",
+            "symbols": [
+                {"name": "alpha", "kind": "function", "line": 10, "hash": "x1"},
+                {"name": "beta", "kind": "function", "line": 20, "hash": "x2"},
+                {"name": "os", "kind": "import"},  # non-uniform: no line/hash
+            ],
+        },
+        "b/second.py": {
+            "hash": "h1",
+            "symbols": [
+                {"name": "gamma", "kind": "class", "line": 5, "hash": "y1"},
+                {"name": "delta", "kind": "function", "line": 8, "hash": "y2"},
+            ],
+        },
+    },
+    "repo": "demo",
+}
+
+
+def test_structure_uses_group_scoped_nested_questions():
+    # Even though b/second.py's symbols ARE uniform (so extract_records finds a list),
+    # a dict-map payload must use GROUP-SCOPED questions — an unscoped count would be
+    # ambiguous across files. Group-scoping wins over the uniform extractor (#71).
+    qs = {q.qtype: q for q in fluency.gen_questions(STRUCTURE)}
+    assert set(qs) == {"count", "enumerate", "lookup"}
+    # scoped to the first file (map order), its 3 non-uniform symbols
+    assert qs["count"].expected == 3
+    assert qs["enumerate"].expected == ["alpha", "beta", "os"]  # 'name' = most-distinct id col
+    assert qs["lookup"].expected == "function"                  # alpha's kind
+    # no aggregate: 'line' is absent from the import symbol, so it isn't an intersection col
+    assert "aggregate" not in qs
+    for q in qs.values():
+        assert 'files["a/first.py"]' in q.prompt
+
+
+def test_nested_group_uses_intersection_columns_only():
+    grp = fluency._nested_record_group(STRUCTURE)
+    assert grp is not None
+    label, records, cols = grp
+    assert label == 'files["a/first.py"]'
+    assert cols == ["kind", "name"]  # sorted intersection; line/hash excluded (non-uniform)
+    assert len(records) == 3
+
+
+def test_nested_questions_are_self_consistent():
+    for q in fluency.gen_questions(STRUCTURE):
+        reply = q.expected if isinstance(q.expected, str) else json.dumps(q.expected)
+        assert fluency.score(q.qtype, q.expected, reply)
+
+
+def test_nested_lookup_skipped_when_no_column_uniquely_addresses_a_record():
+    # Both `name` and `kind` repeat within the file, so no column uniquely identifies a
+    # record — a lookup prompt would be ambiguous and a truthful answer about a different
+    # matching record would score as a false-negative regression. lookup must be OMITTED;
+    # count + enumerate (which tolerate duplicates) still fire.
+    obj = {"files": {"x.py": {"symbols": [
+        {"name": "X", "kind": "function"},
+        {"name": "Y", "kind": "function"},
+        {"name": "X", "kind": "class"},
+        {"name": "Y", "kind": "class"}]}}}
+    qs = {q.qtype: q for q in fluency.gen_questions(obj)}
+    assert "lookup" not in qs
+    assert qs["count"].expected == 4
+    assert "enumerate" in qs
+    # the enumerate ground truth stays exactly checkable even with repeated values
+    assert fluency.score("enumerate", qs["enumerate"].expected,
+                         json.dumps(qs["enumerate"].expected))
+
+
+def test_nested_aggregate_appears_when_a_numeric_col_is_shared():
+    # every symbol carries `line` (only `hash` varies) -> line is an intersection col -> aggregate
+    obj = {"files": {"f": {"symbols": [
+        {"name": "a", "kind": "fn", "line": 3, "hash": "h"},
+        {"name": "b", "kind": "fn", "line": 9},          # no hash -> still non-uniform
+        {"name": "c", "kind": "var", "line": 1, "hash": "h2"}]}}}
+    from terse.capture import extract_records
+    assert extract_records(obj) is None
+    qs = {q.qtype: q for q in fluency.gen_questions(obj)}
+    assert qs["aggregate"].expected == 9
+
+
+def test_run_diff_payload_now_exercises_structure_pairs():
+    # the #71 payoff: a structure diff yields the same questions in both forms, so
+    # `terse fluency --diff` can finally measure structure comprehension.
+    curr = json.loads(json.dumps(STRUCTURE))
+    curr["files"]["a/first.py"]["symbols"].append(
+        {"name": "epsilon", "kind": "function", "line": 40, "hash": "x9"})
+    rows = fluency.run_diff_payload(STRUCTURE, curr, lambda s, u: "",
+                                    tool="runecho.structure", trials=1)
+    assert rows  # non-empty: structure now generates questions -> diff is testable
+    assert {r["qid"] for r in rows} >= {"count", "enumerate"}
+    assert all("terse_ok" in r and "diff_ok" in r for r in rows)
+
+
 def test_score_count_and_aggregate_tolerate_prose_and_check_value():
     assert fluency.score("count", 4, "There are 4 records.")
     assert fluency.score("count", 4, "4")