From f52c9e24e16c63c0a47486a15319be65822ab401 Mon Sep 17 00:00:00 2001 From: Marc Shade <167566032+marc-shade@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:40:13 -0400 Subject: [PATCH 1/4] Add test asserting every query validator rejects a refusal Nothing currently checks that the 104 per-query validators can fail. A validator that accepted any input would mark every agent correct on its query and nothing would notice. Standard library only, no new dependencies. --- tests/test_validators_reject_refusals.py | 81 ++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tests/test_validators_reject_refusals.py diff --git a/tests/test_validators_reject_refusals.py b/tests/test_validators_reject_refusals.py new file mode 100644 index 000000000..f46057977 --- /dev/null +++ b/tests/test_validators_reject_refusals.py @@ -0,0 +1,81 @@ +"""Assert that every query validator rejects a refusal. + +Each query ships its own ``validate.py`` exposing ``validate(llm_output) -> (bool, str)``. +Nothing currently checks that those validators can fail, so a validator that accepts +any input would score every agent as correct on its query and nothing would notice. + +This test feeds each validator several refusals that contain no answer and no digits, +and asserts each is rejected. + +Scope and limits: this is a necessary check, not a sufficient one. It cannot detect a +validator that is too strict, and a validator that always returned False would pass it. +A companion positive test (validator accepts its own ground truth) would close that +gap and is a natural follow-up. + +Run: + python -m unittest discover -s tests -v +""" + +import importlib.util +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# Refusals a real agent emits when it gives up. Deliberately free of digits and of +# any state or entity name, so no validator should legitimately accept one. +REFUSALS = [ + "I could not determine the answer.", + "I'm sorry, I cannot answer this question.", + "Unable to complete the task with the available data.", + "", +] + + +def discover_validators(): + """Yield (query_id, path) for every query validator in the repo.""" + for path in sorted(REPO_ROOT.glob("query_*/query*/validate.py")): + query_id = f"{path.parent.parent.name}/{path.parent.name}" + yield query_id, path + + +def load_validate(query_id, path): + """Import a validator module in isolation and return its validate callable.""" + spec = importlib.util.spec_from_file_location( + f"dab_validator_{query_id.replace('/', '_')}", path + ) + if spec is None or spec.loader is None: + raise ImportError(f"could not build an import spec for {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.validate + + +def is_accepted(result): + """Validators return (bool, reason); tolerate a bare bool.""" + if isinstance(result, tuple): + return bool(result[0]) + return bool(result) + + +class TestValidatorsRejectRefusals(unittest.TestCase): + def test_validators_are_discoverable(self): + self.assertGreater( + len(list(discover_validators())), 0, + "no query validators found; check REPO_ROOT resolution", + ) + + def test_every_validator_rejects_refusals(self): + for query_id, path in discover_validators(): + validate = load_validate(query_id, path) + for refusal in REFUSALS: + with self.subTest(query=query_id, refusal=refusal or ""): + self.assertFalse( + is_accepted(validate(refusal)), + f"{query_id} accepted a refusal, so it cannot distinguish a " + f"correct answer from no answer at all. Input was: {refusal!r}", + ) + + +if __name__ == "__main__": + unittest.main() From 9a232326c5853984f5436d55f73c9306a681effd Mon Sep 17 00:00:00 2001 From: Marc Shade <167566032+marc-shade@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:40:14 -0400 Subject: [PATCH 2/4] Run the validator refusal test in CI Triggers on changes to any validate.py, to tests, or to the workflow. --- .github/workflows/validators.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/validators.yml diff --git a/.github/workflows/validators.yml b/.github/workflows/validators.yml new file mode 100644 index 000000000..77317a281 --- /dev/null +++ b/.github/workflows/validators.yml @@ -0,0 +1,32 @@ +name: Validators + +on: + pull_request: + paths: + - "query_*/**/validate.py" + - "tests/**" + - ".github/workflows/validators.yml" + push: + branches: + - main + paths: + - "query_*/**/validate.py" + - "tests/**" + - ".github/workflows/validators.yml" + workflow_dispatch: + +jobs: + reject-refusals: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: false + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # The test uses only the standard library, so there is nothing to install. + - name: Assert every validator rejects a refusal + run: python -m unittest discover -s tests -v From 90e7899e63590aded6f26463375afedfa5ed1c8d Mon Sep 17 00:00:00 2001 From: Marc Shade <167566032+marc-shade@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:05:37 -0400 Subject: [PATCH 3/4] Add positive control: every validator must accept its own ground truth The refusal check alone would be satisfied by a validator that always returned False. Feeding each validator its own ground_truth.csv bounds it from the other side: it must be able to say yes as well as no. Renames the module to test_validators.py since it now covers both directions. Refs #81 --- tests/test_validators.py | 119 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/test_validators.py diff --git a/tests/test_validators.py b/tests/test_validators.py new file mode 100644 index 000000000..c1d2b40e7 --- /dev/null +++ b/tests/test_validators.py @@ -0,0 +1,119 @@ +"""Sanity checks that every query validator can both accept and reject. + +Each query ships its own ``validate.py`` exposing ``validate(llm_output) -> (bool, str)``. +Nothing currently checks those validators, so a validator that accepted any input would +score every agent correct on its query, and one that rejected every input would score +every agent wrong. Neither would be noticed. + +Two complementary checks: + +* **Negative control.** Every validator must reject refusals that contain no answer and + no digits. A validator that accepts one cannot tell a correct answer from no answer. +* **Positive control.** Every validator must accept its own ``ground_truth.csv``, rendered + as an answer-like string. A validator that rejects its own ground truth makes its query + unpassable, and it also stops the negative control from being satisfiable trivially by + a validator that always returns False. + +Together these bound the validator: it must be able to say yes, and it must be able to +say no. They do not check that it says yes and no in exactly the right places, which +would require labelled agent answers this repo does not ship. + +Run: + python -m unittest discover -s tests -v +""" + +import csv +import importlib.util +import io +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# Refusals a real agent emits when it gives up. Deliberately free of digits and of any +# entity name, so no validator should legitimately accept one. +REFUSALS = [ + "I could not determine the answer.", + "I'm sorry, I cannot answer this question.", + "Unable to complete the task with the available data.", + "", +] + + +def discover_validators(): + """Yield (query_id, validate_path) for every query validator in the repo.""" + for path in sorted(REPO_ROOT.glob("query_*/query*/validate.py")): + query_id = f"{path.parent.parent.name}/{path.parent.name}" + yield query_id, path + + +def load_validate(query_id, path): + """Import a validator module in isolation and return its validate callable.""" + spec = importlib.util.spec_from_file_location( + f"dab_validator_{query_id.replace('/', '_')}", path + ) + if spec is None or spec.loader is None: + raise ImportError(f"could not build an import spec for {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.validate + + +def is_accepted(result): + """Validators return (bool, reason); tolerate a bare bool.""" + if isinstance(result, tuple): + return bool(result[0]) + return bool(result) + + +def render_ground_truth(query_dir): + """Render ground_truth.csv as an answer-like string: field values, space separated. + + Fields are split rather than passed as a raw CSV line because a bare line such as + ``2023,64171000`` reads as one fused number to validators that strip thousands + separators before parsing. + """ + text = (query_dir / "ground_truth.csv").read_text(errors="ignore").strip() + cells = [ + cell.strip() + for row in csv.reader(io.StringIO(text)) + for cell in row + if cell.strip() + ] + return " ".join(cells) if cells else text + + +class TestValidators(unittest.TestCase): + def test_validators_are_discoverable(self): + self.assertGreater( + len(list(discover_validators())), 0, + "no query validators found; check REPO_ROOT resolution", + ) + + def test_every_validator_rejects_refusals(self): + """Negative control: a validator must be able to say no.""" + for query_id, path in discover_validators(): + validate = load_validate(query_id, path) + for refusal in REFUSALS: + with self.subTest(query=query_id, refusal=refusal or ""): + self.assertFalse( + is_accepted(validate(refusal)), + f"{query_id} accepted a refusal, so it cannot distinguish a " + f"correct answer from no answer at all. Input was: {refusal!r}", + ) + + def test_every_validator_accepts_its_ground_truth(self): + """Positive control: a validator must be able to say yes.""" + for query_id, path in discover_validators(): + validate = load_validate(query_id, path) + answer = render_ground_truth(path.parent) + with self.subTest(query=query_id): + self.assertTrue( + is_accepted(validate(answer)), + f"{query_id} rejected its own ground truth, so the query cannot be " + f"passed by reporting the recorded answer. Input was: {answer!r}", + ) + + +if __name__ == "__main__": + unittest.main() From db8fabda3803313dd286bfc4afebc36ae0231a91 Mon Sep 17 00:00:00 2001 From: Marc Shade <167566032+marc-shade@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:05:38 -0400 Subject: [PATCH 4/4] Remove superseded test module (renamed to test_validators.py) --- tests/test_validators_reject_refusals.py | 81 ------------------------ 1 file changed, 81 deletions(-) delete mode 100644 tests/test_validators_reject_refusals.py diff --git a/tests/test_validators_reject_refusals.py b/tests/test_validators_reject_refusals.py deleted file mode 100644 index f46057977..000000000 --- a/tests/test_validators_reject_refusals.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Assert that every query validator rejects a refusal. - -Each query ships its own ``validate.py`` exposing ``validate(llm_output) -> (bool, str)``. -Nothing currently checks that those validators can fail, so a validator that accepts -any input would score every agent as correct on its query and nothing would notice. - -This test feeds each validator several refusals that contain no answer and no digits, -and asserts each is rejected. - -Scope and limits: this is a necessary check, not a sufficient one. It cannot detect a -validator that is too strict, and a validator that always returned False would pass it. -A companion positive test (validator accepts its own ground truth) would close that -gap and is a natural follow-up. - -Run: - python -m unittest discover -s tests -v -""" - -import importlib.util -import unittest -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parent.parent - -# Refusals a real agent emits when it gives up. Deliberately free of digits and of -# any state or entity name, so no validator should legitimately accept one. -REFUSALS = [ - "I could not determine the answer.", - "I'm sorry, I cannot answer this question.", - "Unable to complete the task with the available data.", - "", -] - - -def discover_validators(): - """Yield (query_id, path) for every query validator in the repo.""" - for path in sorted(REPO_ROOT.glob("query_*/query*/validate.py")): - query_id = f"{path.parent.parent.name}/{path.parent.name}" - yield query_id, path - - -def load_validate(query_id, path): - """Import a validator module in isolation and return its validate callable.""" - spec = importlib.util.spec_from_file_location( - f"dab_validator_{query_id.replace('/', '_')}", path - ) - if spec is None or spec.loader is None: - raise ImportError(f"could not build an import spec for {path}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module.validate - - -def is_accepted(result): - """Validators return (bool, reason); tolerate a bare bool.""" - if isinstance(result, tuple): - return bool(result[0]) - return bool(result) - - -class TestValidatorsRejectRefusals(unittest.TestCase): - def test_validators_are_discoverable(self): - self.assertGreater( - len(list(discover_validators())), 0, - "no query validators found; check REPO_ROOT resolution", - ) - - def test_every_validator_rejects_refusals(self): - for query_id, path in discover_validators(): - validate = load_validate(query_id, path) - for refusal in REFUSALS: - with self.subTest(query=query_id, refusal=refusal or ""): - self.assertFalse( - is_accepted(validate(refusal)), - f"{query_id} accepted a refusal, so it cannot distinguish a " - f"correct answer from no answer at all. Input was: {refusal!r}", - ) - - -if __name__ == "__main__": - unittest.main()