From 3b3148272a69dca83104ea5f3325c505903a08ce Mon Sep 17 00:00:00 2001 From: Gadi Evron Date: Thu, 28 May 2026 11:47:56 -0700 Subject: [PATCH 1/2] fix(parsers/c): exclude on path components in extract_all, not a substring of the abspath c/function_extractor.py extract_all skipped files via `any(excl in str(file_path) for excl in ['.git','build','test','node_modules'])` -- an unanchored substring test against the absolute path. So files whose path merely contained a token were wrongly skipped ('src/latest/main.c' and 'contest/sol.c' contain 'test'), and an ancestor of repo_path containing a token poisoned the whole scan (a checkout under '/home/tester/' excluded every file). Match on path COMPONENTS relative to repo_path instead, using c's own token set. Scope: c's member of the cross-parser extract_all substring family. The python/php/ruby extract_all siblings carry DIFFERENT token sets (vendor*/tmp*/venv*) and are not widened here. Tests: tests/test_c_extract_all_path_components.py (spies process_file: files with a token in a path *segment* are processed; real test/ and .git dirs stay excluded). RED 1 failed (all excluded via ancestor-poison) -> GREEN 1 passed; full suite 177 passed / 63 skipped. The test loads the C function_extractor under a unique module name via importlib so it does not pollute sys.modules['function_extractor'] for the sibling python parser tests. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../parsers/c/function_extractor.py | 7 ++- .../test_c_extract_all_path_components.py | 53 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 libs/openant-core/tests/test_c_extract_all_path_components.py diff --git a/libs/openant-core/parsers/c/function_extractor.py b/libs/openant-core/parsers/c/function_extractor.py index 8e5b1cf1..cae58601 100644 --- a/libs/openant-core/parsers/c/function_extractor.py +++ b/libs/openant-core/parsers/c/function_extractor.py @@ -530,8 +530,11 @@ def extract_all(self, files: Optional[List[str]] = None) -> Dict: all_extensions = C_EXTENSIONS | CPP_EXTENSIONS for ext in all_extensions: for file_path in self.repo_path.rglob(f'*{ext}'): - path_str = str(file_path) - if any(excl in path_str for excl in ['.git', 'build', 'test', 'node_modules']): + # Exclude on path COMPONENTS relative to repo_path, not a substring of the absolute + # path. A substring test wrongly skips files whose path merely contains a token + # ('src/latest/main.c', 'contest/sol.c' contain 'test') and is poisoned when an + # ancestor of repo_path contains one (a checkout under '/home/tester/' excludes all). + if {'.git', 'build', 'test', 'node_modules'} & set(file_path.relative_to(self.repo_path).parts): continue self.process_file(file_path) diff --git a/libs/openant-core/tests/test_c_extract_all_path_components.py b/libs/openant-core/tests/test_c_extract_all_path_components.py new file mode 100644 index 00000000..31262fa9 --- /dev/null +++ b/libs/openant-core/tests/test_c_extract_all_path_components.py @@ -0,0 +1,53 @@ +"""Regression: c/function_extractor extract_all over-excludes by substring. + +extract_all filters discovered files with `any(excl in str(file_path) for excl in [.git,build,test, +node_modules])` — an unanchored SUBSTRING test against the absolute path. So a file whose path merely +*contains* a token is wrongly skipped ('src/latest/main.c' contains 'test'; 'contest/sol.c' too), and an +ANCESTOR dir of repo_path that contains a token poisons the whole scan (a checkout under '/home/tester/' +excludes everything — pytest's own tmp_path, which contains 'test', reproduces this). Fix: match on path +COMPONENTS relative to repo_path, using c's own token set. +""" +import importlib.util +import sys +from pathlib import Path + +CORE = Path(__file__).resolve().parents[1] # libs/openant-core +if str(CORE) not in sys.path: + sys.path.insert(0, str(CORE)) # for utilities.file_io + +# Load the C parser's function_extractor under a UNIQUE module name (not the bare +# 'function_extractor') so we do NOT pollute sys.modules for sibling parser tests: +# parsers/python also ships a 'function_extractor' module, and a bare import here would +# shadow it for the whole pytest session. The C module imports only stdlib + tree_sitter_c + +# utilities.file_io, so no parsers/c entry on sys.path is required. +_spec = importlib.util.spec_from_file_location( + "c_function_extractor_isolated", str(CORE / "parsers" / "c" / "function_extractor.py")) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +FunctionExtractor = _mod.FunctionExtractor + + +def test_extract_all_excludes_on_path_components_not_substring(tmp_path, monkeypatch): + repo = tmp_path / "repo" + for d in ("src/latest", "contest", "src", "test", ".git"): + (repo / d).mkdir(parents=True, exist_ok=True) + (repo / "src" / "latest" / "main.c").write_text("int latest_fn(void){return 1;}\n") + (repo / "contest" / "sol.c").write_text("int contest_fn(void){return 2;}\n") + (repo / "src" / "clean.c").write_text("int clean_fn(void){return 3;}\n") + (repo / "test" / "helper.c").write_text("int test_helper(void){return 4;}\n") # real test/ dir + (repo / ".git" / "hook.c").write_text("int git_fn(void){return 5;}\n") # .git + + ex = FunctionExtractor(str(repo)) + processed = [] + monkeypatch.setattr(ex, "process_file", + lambda fp: processed.append(str(Path(fp).relative_to(ex.repo_path)))) + ex.extract_all() + procset = set(processed) + + # path merely CONTAINS a token -> must still be processed (substring match was the bug) + assert "src/latest/main.c" in procset, f"'latest' wrongly excluded by 'test' substring: {sorted(procset)}" + assert "contest/sol.c" in procset, f"'contest' wrongly excluded: {sorted(procset)}" + assert "src/clean.c" in procset + # genuine excluded directory NAMES stay excluded + assert "test/helper.c" not in procset, "a real test/ directory should stay excluded" + assert ".git/hook.c" not in procset, ".git should stay excluded" From ce5ca98d16453dc840f6c7180376d73734495d07 Mon Sep 17 00:00:00 2001 From: Gadi Evron Date: Thu, 28 May 2026 17:25:47 -0700 Subject: [PATCH 2/2] test(parsers/c): normalize recorded path to forward slashes for Windows CI The extract_all over-exclusion regression test recorded the processed path via str(Path.relative_to(...)), which yields backslash separators on Windows and fails the forward-slash 'in procset' assertions. Use .as_posix() so the comparison is OS-independent. The substring-over-exclusion assertions are unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- libs/openant-core/tests/test_c_extract_all_path_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/openant-core/tests/test_c_extract_all_path_components.py b/libs/openant-core/tests/test_c_extract_all_path_components.py index 31262fa9..68a7df24 100644 --- a/libs/openant-core/tests/test_c_extract_all_path_components.py +++ b/libs/openant-core/tests/test_c_extract_all_path_components.py @@ -40,7 +40,7 @@ def test_extract_all_excludes_on_path_components_not_substring(tmp_path, monkeyp ex = FunctionExtractor(str(repo)) processed = [] monkeypatch.setattr(ex, "process_file", - lambda fp: processed.append(str(Path(fp).relative_to(ex.repo_path)))) + lambda fp: processed.append(Path(fp).relative_to(ex.repo_path).as_posix())) ex.extract_all() procset = set(processed)