From 2742e6ff8a99a7845f1360e80055e4256fa58383 Mon Sep 17 00:00:00 2001 From: Gadi Evron Date: Thu, 28 May 2026 12:23:58 -0700 Subject: [PATCH 1/2] fix(parsers/php): extract procedural top-level + closure units; seed PHP entry points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two extraction/seeding defects on the PHP analysis path, plus the PHP entry-point-seeding gap they depend on. All changes are confined to the extraction layer (parsers/php/function_extractor.py) and the entry-point detector (utilities/agentic_enhancer/entry_point_detector.py); the PHP call-graph builder is untouched. 1. Procedural top-level blackout: _extract_functions_from_tree emitted units only for named definitions; top-level procedural statements (assignments, echo, add_action(...) hook registrations) fell through the catch-all else and produced NO unit, so a WordPress-style plugin.php was invisible to reachability seeding. The Python parser has a module-level synthesizer (extract_module_level_code -> unit_type='module_level'); PHP had none. Adds _extract_module_level_unit (called from process_file), synthesising a :__module__ unit from program-level statements. Handles braceless + braced namespaces; emits nothing for files with no file-scope code. 2. PHP entry-point seeding: entry_point_detector USER_INPUT_PATTERNS / MODULE_LEVEL_INPUT_PATTERNS were Python/JS-only, so a PHP handler reading $_POST was never an entry point (Check 3) and the module_level unit could not fire Check 4. Adds PHP superglobals ($_GET/$_POST/$_REQUEST/$_COOKIE/$_SERVER/$_FILES/$_ENV/$_SESSION), php://input / filter_input, and WordPress hook idioms (add_action/add_filter/do_action/ apply_filters) for the module-level path. 3. Anonymous closures + arrow functions as units: anonymous_function / arrow_function nodes fell through the same else and were never modeled; the named-definition walk also did not descend into function/method bodies, so nested closures were unreachable. Adds a closure dispatch branch (unit_type='closure', synthetic {closure@line:col} name) and makes function_definition / method_declaration recurse into their bodies. The closure-DISPATCH edge ($cb() -> closure) lives in call_graph_builder.py and is out of this file's scope; this fixes only the extraction half. Out of scope (not fixed here): - The use_declaration -> namespace_use_declaration node-type correction is already handled by the existing PHP import-extraction code in _extract_imports; re-touching it here would duplicate that change. Left untouched. - Aliased `use Foo\Bar as Baz` -> alias-to-FQN translation lives in call_graph_builder.py::_resolve_class_call (out of this file's scope). An alias-capture in function_extractor alone would be unobservable (the only consumer of the imports map is call_graph_builder.py) and would risk regressing import-matching. No no-op change made. Tests: tests/parsers/php/test_php_extractor.py (new; the package had no PHP extractor tests). Modules loaded under unique importlib names (function_extractor.py is a basename shared by every parser). Eight tests covering module_level synthesis, no-false-positive on class-only files, PHP superglobal entry-point seeding (Check 3 + Check 4), and closure/arrow-function unit extraction. 6 failed pre-fix (2 guard tests green at base by design) -> 8 passed after the fix. ruff clean; full suite 184 passed / 63 skipped (suite excluding the new file is exactly 176/63 — zero regression). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../parsers/php/function_extractor.py | 151 ++++++++++ .../tests/parsers/php/__init__.py | 0 .../tests/parsers/php/test_php_extractor.py | 265 ++++++++++++++++++ .../agentic_enhancer/entry_point_detector.py | 14 + 4 files changed, 430 insertions(+) create mode 100644 libs/openant-core/tests/parsers/php/__init__.py create mode 100644 libs/openant-core/tests/parsers/php/test_php_extractor.py diff --git a/libs/openant-core/parsers/php/function_extractor.py b/libs/openant-core/parsers/php/function_extractor.py index 2c9039ad..0df7043e 100644 --- a/libs/openant-core/parsers/php/function_extractor.py +++ b/libs/openant-core/parsers/php/function_extractor.py @@ -238,6 +238,13 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, node, source, relative_path, class_name, namespace_name, is_static=False ) + # Recurse into the body so nested closures/arrow functions are + # reached; named definitions cannot lexically nest other named + # functions/classes in PHP, so this only surfaces + # anonymous_function / arrow_function units. + for child in reversed(node.children): + stack.append((child, class_name, namespace_name)) + continue elif node.type == 'method_declaration': is_static = self._is_static_method(node, source) @@ -245,6 +252,24 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, node, source, relative_path, class_name, namespace_name, is_static=is_static ) + for child in reversed(node.children): + stack.append((child, class_name, namespace_name)) + continue + + elif node.type in ('anonymous_function', 'arrow_function'): + # Anonymous closures (`function ($x) {...}`) and arrow functions + # (`fn($z) => ...`) are unnamed; tree-sitter emits them as + # `anonymous_function` / `arrow_function`. Without this branch + # they fell through the catch-all else and were never modeled as + # units, so callback-heavy PHP was invisible. + self._process_closure_node( + node, source, relative_path, class_name, namespace_name + ) + # Still recurse so a closure declared inside another closure + # (or anything else nested in its body) is reached. + for child in reversed(node.children): + stack.append((child, class_name, namespace_name)) + continue elif node.type == 'class_declaration': # Extract class name @@ -462,6 +487,129 @@ def _process_function_node(self, node, source: bytes, relative_path: str, self.stats['by_type'][unit_type] = self.stats['by_type'].get(unit_type, 0) + 1 + def _process_closure_node(self, node, source: bytes, relative_path: str, + class_name: Optional[str], + namespace_name: Optional[str]) -> None: + """Process an anonymous_function or arrow_function node as a closure unit. + + Closures are unnamed, so a synthetic name keyed on the source position + keeps the func_id unique within a file. + """ + start_line = node.start_point[0] + 1 # tree-sitter is 0-indexed + end_line = node.end_point[0] + 1 + start_col = node.start_point[1] + + kind = 'arrow' if node.type == 'arrow_function' else 'closure' + name = f'{{{kind}@{start_line}:{start_col}}}' + + code = self._node_text(node, source) + parameters = self._get_parameters(node, source) + + # Qualify the synthetic name with the lexical owner so distinct closures + # in the same file do not collide. + if class_name: + qualified_name = f"{class_name}.{name}" + elif namespace_name: + qualified_name = f"{namespace_name}\\{name}" + else: + qualified_name = name + + func_id = f"{relative_path}:{qualified_name}" + + self.functions[func_id] = { + 'name': name, + 'qualified_name': qualified_name, + 'file_path': relative_path, + 'start_line': start_line, + 'end_line': end_line, + 'code': code, + 'class_name': class_name, + 'namespace_name': namespace_name, + 'parameters': parameters, + 'is_static': False, + 'unit_type': 'closure', + } + self.stats['total_functions'] += 1 + self.stats['standalone_functions'] += 1 + self.stats['by_type']['closure'] = self.stats['by_type'].get('closure', 0) + 1 + + def _extract_module_level_unit(self, tree, source: bytes, + relative_path: str) -> None: + """Synthesise a `module_level` unit for top-level procedural statements. + + PHP scripts (WordPress plugins, legacy procedural files) run top-to-bottom + and register hooks / read superglobals at file scope. The named-definition + walk in `_extract_functions_from_tree` never emits a unit for that + file-scope code, so it was invisible to reachability seeding. This + mirrors the Python parser's `extract_module_level_code`, + emitting a single synthetic `:__module__` unit whose code is the + concatenation of the program-level statements that are not themselves + function/class/interface/trait/namespace declarations. + """ + root = tree.root_node + + # Named definitions (each already emitted as its own unit) and pure + # structural/import tokens are NOT executable top-level code. + skip_types = { + 'function_definition', 'class_declaration', 'interface_declaration', + 'trait_declaration', 'enum_declaration', 'namespace_use_declaration', + 'use_declaration', 'php_tag', 'text_interpolation', 'text', ';', + 'comment', 'declare_statement', '{', '}', + } + + def program_statements(node): + """Yield file-scope statement nodes. + + A braceless `namespace App;` is a self-contained node whose following + statements are program-level SIBLINGS (tree-sitter-php does not nest + them), so we simply skip the namespace token-node. A braced + `namespace App { ... }` wraps its statements in a body, so we descend + into that body. + """ + for child in node.children: + if child.type == 'namespace_definition': + body = child.child_by_field_name('body') + if body is not None: + yield from program_statements(body) + # braceless: its real statements are program-level siblings, + # reached later in this same loop; the namespace node itself + # contributes no executable code. + continue + if child.type in skip_types: + continue + yield child + + statements = list(program_statements(root)) + if not statements: + return + + code = '\n'.join(self._node_text(s, source) for s in statements).strip() + if not code: + return + + start_line = statements[0].start_point[0] + 1 + end_line = statements[-1].end_point[0] + 1 + + func_id = f"{relative_path}:__module__" + self.functions[func_id] = { + 'name': '__module__', + 'qualified_name': '__module__', + 'file_path': relative_path, + 'start_line': start_line, + 'end_line': end_line, + 'code': code, + 'class_name': None, + 'namespace_name': None, + 'parameters': [], + 'is_static': False, + 'unit_type': 'module_level', + 'is_module_level': True, + } + self.stats['total_functions'] += 1 + self.stats['standalone_functions'] += 1 + self.stats['by_type']['module_level'] = \ + self.stats['by_type'].get('module_level', 0) + 1 + def process_file(self, file_path: Path) -> None: """Process a single PHP file.""" source = self.read_file(file_path) @@ -486,6 +634,9 @@ def process_file(self, file_path: Path) -> None: # Extract functions self._extract_functions_from_tree(tree, source, file_path, relative_path) + # Synthesise a module_level unit for any top-level procedural statements + self._extract_module_level_unit(tree, source, relative_path) + def extract_from_scan(self, scan_result: Dict) -> Dict: """Extract functions from files listed in a scan result.""" for file_info in scan_result.get('files', []): diff --git a/libs/openant-core/tests/parsers/php/__init__.py b/libs/openant-core/tests/parsers/php/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/openant-core/tests/parsers/php/test_php_extractor.py b/libs/openant-core/tests/parsers/php/test_php_extractor.py new file mode 100644 index 00000000..bab16979 --- /dev/null +++ b/libs/openant-core/tests/parsers/php/test_php_extractor.py @@ -0,0 +1,265 @@ +"""Regression tests for the PHP function extractor + entry-point detector. + +Covers three confirmed defects: + + * Procedural top-level blackout: + parsers/php/function_extractor.py only emits units for + function/method/class/interface/trait/namespace nodes; top-level + procedural statements (assignments, echo, hook registrations) fall + through the catch-all else branch and produce NO unit. The Python + parser has a module-level synthesizer (extract_module_level_code -> + unit_type='module_level'); PHP had none, so a WordPress-style + plugin.php is invisible to reachability seeding. + + * Entry-point seeding for PHP: + utilities/agentic_enhancer/entry_point_detector.py USER_INPUT_PATTERNS + and MODULE_LEVEL_INPUT_PATTERNS were Python/JS-only; no PHP superglobal + ($_GET/$_POST/$_REQUEST/...) was ever recognised, so a PHP handler that + reads $_POST was never flagged as an entry point. + + * Closures not modeled as units: + anonymous_function and arrow_function nodes fell through the same else + branch, so closures and arrow functions were never extracted as units. + +NOTE on import strategy: ``function_extractor.py`` is a basename shared by +every parser (php/python/go/...), so a bare ``import function_extractor`` +would collide. Each module is loaded under a UNIQUE name via +``importlib.util.spec_from_file_location``. + +The call-graph dispatch portions (closure dispatch edges, WordPress +do_action edges, XML-RPC dispatch, alias->FQN resolution) live in +parsers/php/call_graph_builder.py, which is out of this unit's file scope; +they are covered/tracked elsewhere. These tests assert only the +extraction-layer + entry-point-seeding behavior owned by this unit. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[3] +if str(_CORE_ROOT) not in sys.path: + sys.path.insert(0, str(_CORE_ROOT)) + + +def _load_unique(rel_path: str, unique_name: str): + """Load a module from libs/openant-core/ under a unique name. + + function_extractor.py recurs across parsers, so a bare import would + collide; spec_from_file_location with a unique name avoids that. + """ + abs_path = _CORE_ROOT / rel_path + spec = importlib.util.spec_from_file_location(unique_name, abs_path) + module = importlib.util.module_from_spec(spec) + sys.modules[unique_name] = module + spec.loader.exec_module(module) + return module + + +_php_fe = _load_unique("parsers/php/function_extractor.py", "isolated_php_function_extractor") +PhpFunctionExtractor = _php_fe.FunctionExtractor + +_epd = _load_unique( + "utilities/agentic_enhancer/entry_point_detector.py", "isolated_entry_point_detector" +) +EntryPointDetector = _epd.EntryPointDetector + + +def _extract(tmp_path: Path, filename: str, source: str): + """Write a PHP file and run the real extractor over the repo dir.""" + repo = tmp_path + (repo / filename).write_text(source) + extractor = PhpFunctionExtractor(str(repo)) + return extractor.extract_all() + + +# --------------------------------------------------------------------------- +# Procedural top-level code must become a unit +# --------------------------------------------------------------------------- + +PROCEDURAL_PLUGIN = """ transform($z); + $arrow(1); +} + +function helper($x) { + return $x; +} + +function transform($z) { + return $z; +} +""" + + +def test_anonymous_closure_extracted_as_unit(tmp_path): + """An anonymous_function must be emitted as a closure unit.""" + result = _extract(tmp_path, "closures.php", CLOSURE_SOURCE) + closure_units = [ + fid + for fid, fd in result["functions"].items() + if fd.get("unit_type") == "closure" + ] + assert closure_units, ( + "expected at least one closure unit for the anonymous_function / " + f"arrow_function; unit_types=" + f"{sorted({fd.get('unit_type') for fd in result['functions'].values()})}" + ) + + +def test_arrow_function_extracted_as_unit(tmp_path): + """Both the anonymous closure and the arrow function must be units (2 total).""" + result = _extract(tmp_path, "closures.php", CLOSURE_SOURCE) + closure_units = [ + fd + for fd in result["functions"].values() + if fd.get("unit_type") == "closure" + ] + assert len(closure_units) == 2, ( + "expected exactly two closure units (one anonymous_function + one " + f"arrow_function); got {len(closure_units)}" + ) + + +def test_closure_units_capture_their_body(tmp_path): + """Closure units must carry the closure body in their code field.""" + result = _extract(tmp_path, "closures.php", CLOSURE_SOURCE) + bodies = "\n".join( + fd["code"] + for fd in result["functions"].values() + if fd.get("unit_type") == "closure" + ) + assert "helper" in bodies # from the anonymous closure body + assert "transform" in bodies # from the arrow function body + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py b/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py index 16df5b5b..6f71a568 100644 --- a/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py +++ b/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py @@ -92,6 +92,12 @@ # WebSocket message handlers r'on_message|onmessage|message\.data', r'websocket\.receive', + # PHP superglobals (request/server/file/cookie input) + r'\$_(GET|POST|REQUEST|COOKIE|SERVER|FILES|ENV|SESSION)\b', + r'\$HTTP_RAW_POST_DATA\b', + r'php://input', + r'\bfile_get_contents\s*\(\s*["\']php://input', + r'\bfilter_input\s*\(', ] # Patterns that indicate module-level scripts with user input @@ -100,6 +106,14 @@ r'sys\.argv', r'\binput\s*\(', r'argparse\.', + # PHP file-scope scripts: superglobal reads and WordPress hook dispatch + # (procedural plugins/themes register handlers at the top level). + r'\$_(GET|POST|REQUEST|COOKIE|SERVER|FILES|ENV|SESSION)\b', + r'php://input', + r'\badd_action\s*\(', + r'\badd_filter\s*\(', + r'\bdo_action\s*\(', + r'\bapply_filters\s*\(', ] From 1931314b2d233cf7a8012d5aee9244c288580420 Mon Sep 17 00:00:00 2001 From: Gadi Evron Date: Thu, 28 May 2026 17:42:22 -0700 Subject: [PATCH 2/2] fix(parsers/php): anchor file-discovery exclusion to repo-relative path components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extract_all() skipped files whose ABSOLUTE path contained the substring 'tmp' (or 'vendor'/'node_modules'/'.git'/'.cache'). When the analyzed repo lives under such an ancestor directory — e.g. a Linux /tmp working dir (as on CI), or any path with a 'template'-like segment — every file was wrongly excluded and zero functions were extracted. Match the excluded names against the path's components RELATIVE to the repo root instead, so only the repo's own vendored/transient dirs are skipped, not ancestor directories. Co-Authored-By: Claude Opus 4.7 (1M context) --- libs/openant-core/parsers/php/function_extractor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/openant-core/parsers/php/function_extractor.py b/libs/openant-core/parsers/php/function_extractor.py index 0df7043e..004ea6d1 100644 --- a/libs/openant-core/parsers/php/function_extractor.py +++ b/libs/openant-core/parsers/php/function_extractor.py @@ -655,8 +655,11 @@ def extract_all(self, files: Optional[List[str]] = None) -> Dict: else: for ext in ('.php', '.phtml'): for file_path in self.repo_path.rglob(f'*{ext}'): - path_str = str(file_path) - if any(excl in path_str for excl in ['.git', 'vendor', 'node_modules', 'tmp', '.cache']): + # Exclude vendored/transient dirs by path COMPONENT relative to the repo, not by + # absolute-path substring -- an ancestor dir (e.g. a /tmp working dir, as on Linux CI) + # or a name like 'template' must not exclude the repo's own files. + rel_parts = file_path.relative_to(self.repo_path).parts + if any(excl in rel_parts for excl in ('.git', 'vendor', 'node_modules', 'tmp', '.cache')): continue self.process_file(file_path)