diff --git a/graphify/export.py b/graphify/export.py index 16271a730..84b5258cf 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -17,6 +17,8 @@ ] MAX_NODES_FOR_VIZ = 5_000 +SAMPLE_NODES_PER_COMMUNITY = 50 +SAMPLE_MAX_COMMUNITIES = 20 def _html_styles() -> str: @@ -323,23 +325,64 @@ def to_cypher(G: nx.Graph, output_path: str) -> None: f.write("\n".join(lines)) +def sample_for_viz( + G: nx.Graph, + communities: dict[int, list[str]], + max_communities: int = SAMPLE_MAX_COMMUNITIES, + nodes_per_community: int = SAMPLE_NODES_PER_COMMUNITY, +) -> tuple[nx.Graph, dict[int, list[str]]]: + """Sample the largest communities and highest-degree nodes for visualization. + + Returns a subgraph and filtered communities dict that fits within + MAX_NODES_FOR_VIZ. Useful for large-scale projects where the full graph + would overwhelm the browser. (See issue #52, sub-issue 5.) + """ + # Sort communities by size, keep the largest ones + sorted_cids = sorted(communities.keys(), key=lambda c: len(communities[c]), reverse=True) + top_cids = sorted_cids[:max_communities] + + sampled_nodes: set[str] = set() + sampled_communities: dict[int, list[str]] = {} + + for cid in top_cids: + members = communities[cid] + # Pick highest-degree nodes from this community + member_degrees = [(n, G.degree(n)) for n in members if n in G] + member_degrees.sort(key=lambda x: -x[1]) + selected = [n for n, _ in member_degrees[:nodes_per_community]] + sampled_nodes.update(selected) + sampled_communities[cid] = selected + + subgraph = G.subgraph(sampled_nodes).copy() + return subgraph, sampled_communities + + def to_html( G: nx.Graph, communities: dict[int, list[str]], output_path: str, community_labels: dict[int, str] | None = None, + sample: bool = False, ) -> None: """Generate an interactive vis.js HTML visualization of the graph. Features: node size by degree, click-to-inspect panel, search box, community filter, physics clustering by community, confidence-styled edges. - Raises ValueError if graph exceeds MAX_NODES_FOR_VIZ. + + Args: + sample: If True and the graph exceeds MAX_NODES_FOR_VIZ, automatically + sample the top communities and highest-degree nodes instead of + raising ValueError. The HTML title will note it's a sampled view. """ if G.number_of_nodes() > MAX_NODES_FOR_VIZ: - raise ValueError( - f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz. " - f"Use --no-viz or reduce input size." - ) + if sample: + original_count = G.number_of_nodes() + G, communities = sample_for_viz(G, communities) + else: + raise ValueError( + f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz. " + f"Use --sample to visualize top communities, or --no-viz to skip." + ) node_community = _node_community_map(communities) degree = dict(G.degree()) diff --git a/graphify/extract.py b/graphify/extract.py index c767e07f4..c51d6a0d1 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -2,6 +2,7 @@ from __future__ import annotations import importlib import json +import logging import os import re import sys @@ -10,6 +11,8 @@ from typing import Callable, Any from .cache import load_cached, save_cached +_log = logging.getLogger(__name__) + def _make_id(*parts: str) -> str: """Build a stable node ID from one or more name parts.""" @@ -632,6 +635,14 @@ def _extract_generic(path: Path, config: LanguageConfig) -> dict: language = Language(lang_fn()) except ImportError: return {"nodes": [], "edges": [], "error": f"{config.ts_module} not installed"} + except TypeError as e: + # tree-sitter version mismatch: old Language() expects (lib_path), + # new Language() expects (language_capsule, name). + hint = ( + f"tree-sitter version mismatch for {config.ts_module}: {e}. " + "Try: pip install --upgrade tree-sitter tree-sitter-languages" + ) + return {"nodes": [], "edges": [], "error": hint} except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} @@ -2625,23 +2636,39 @@ def extract(paths: list[Path]) -> dict: } total = len(paths) - _PROGRESS_INTERVAL = 100 + extracted = 0 + cached_count = 0 + skipped = 0 + errors = 0 + # Log progress every N files (at least every 10%, minimum every 100 files) + progress_interval = max(100, total // 10) if total > 0 else 1 + for i, path in enumerate(paths): - if total >= _PROGRESS_INTERVAL and i % _PROGRESS_INTERVAL == 0 and i > 0: - print(f" AST extraction: {i}/{total} files ({i * 100 // total}%)", flush=True) extractor = _DISPATCH.get(path.suffix) if extractor is None: + skipped += 1 continue cached = load_cached(path, root) if cached is not None: per_file.append(cached) - continue - result = extractor(path) - if "error" not in result: - save_cached(path, result, root) - per_file.append(result) - if total >= _PROGRESS_INTERVAL: - print(f" AST extraction: {total}/{total} files (100%)", flush=True) + cached_count += 1 + else: + result = extractor(path) + if "error" not in result: + save_cached(path, result, root) + else: + errors += 1 + per_file.append(result) + extracted += 1 + + # Periodic progress logging for large batches + processed = i + 1 + if total >= 100 and (processed % progress_interval == 0 or processed == total): + _log.info( + "Extraction progress: %d/%d files (%.0f%%) — %d extracted, %d cached, %d skipped, %d errors", + processed, total, 100 * processed / total, + extracted, cached_count, skipped, errors, + ) all_nodes: list[dict] = [] all_edges: list[dict] = [] @@ -2657,8 +2684,7 @@ def extract(paths: list[Path]) -> dict: cross_file_edges = _resolve_cross_file_imports(py_results, py_paths) all_edges.extend(cross_file_edges) except Exception as exc: - import logging - logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc) + _log.warning("Cross-file import resolution failed, skipping: %s", exc) return { "nodes": all_nodes, diff --git a/tests/test_issue52.py b/tests/test_issue52.py new file mode 100644 index 000000000..9f7927b95 --- /dev/null +++ b/tests/test_issue52.py @@ -0,0 +1,206 @@ +"""Tests for issue #52 fixes: large-scale project robustness. + +Sub-issue 1: tree-sitter version mismatch gives clear error message +Sub-issue 5: sample_for_viz reduces large graphs for HTML visualization +Sub-issue 8: extract() logs progress for large file batches +""" +from __future__ import annotations + +import logging +import sys +from pathlib import Path +from unittest.mock import patch + +import networkx as nx +import pytest + +from graphify.export import ( + MAX_NODES_FOR_VIZ, + sample_for_viz, + to_html, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_large_graph(n_communities: int = 25, nodes_per: int = 60) -> tuple[ + nx.Graph, dict[int, list[str]], dict[int, str] +]: + """Build a graph that exceeds MAX_NODES_FOR_VIZ when n_communities * nodes_per > 5000.""" + G = nx.Graph() + communities: dict[int, list[str]] = {} + labels: dict[int, str] = {} + for cid in range(n_communities): + members = [] + for j in range(nodes_per): + node_id = f"c{cid}_n{j}" + G.add_node(node_id, label=f"Node {cid}-{j}", file_type="code", source_file=f"f{cid}.py") + members.append(node_id) + # Connect nodes within community + for j in range(nodes_per - 1): + G.add_edge(members[j], members[j + 1], relation="calls", confidence="EXTRACTED") + communities[cid] = members + labels[cid] = f"Community {cid}" + return G, communities, labels + + +def _make_small_graph() -> tuple[nx.Graph, dict[int, list[str]], dict[int, str]]: + """Build a small graph well under the viz limit.""" + G = nx.Graph() + G.add_node("a", label="Alpha", file_type="code", source_file="a.py") + G.add_node("b", label="Beta", file_type="code", source_file="b.py") + G.add_node("c", label="Gamma", file_type="code", source_file="c.py") + G.add_edge("a", "b", relation="calls", confidence="EXTRACTED") + G.add_edge("b", "c", relation="imports", confidence="EXTRACTED") + communities = {0: ["a", "b"], 1: ["c"]} + labels = {0: "Core", 1: "Docs"} + return G, communities, labels + + +# --------------------------------------------------------------------------- +# Sub-issue 1: tree-sitter version mismatch error message +# --------------------------------------------------------------------------- + +def test_tree_sitter_version_mismatch_error_message(): + """When Language() raises TypeError (version mismatch), the error should + contain a helpful upgrade hint.""" + from graphify.extract import _extract_generic, LanguageConfig + + config = LanguageConfig( + ts_module="tree_sitter_python", + ts_language_fn="language", + ) + + # Simulate tree-sitter version mismatch: Language() raises TypeError + # Language is imported inside _extract_generic via `from tree_sitter import Language` + def raise_type_error(*args, **kwargs): + raise TypeError("missing 1 required positional argument: 'name'") + + import types + fake_ts = types.ModuleType("tree_sitter") + fake_ts.Language = raise_type_error + fake_ts.Parser = None + + with patch("graphify.extract.importlib.import_module") as mock_import: + mock_mod = type("FakeMod", (), {"language": lambda: None})() + mock_import.return_value = mock_mod + + with patch.dict(sys.modules, {"tree_sitter": fake_ts}): + result = _extract_generic(Path("test.py"), config) + + assert "error" in result + assert "tree-sitter version mismatch" in result["error"] + assert "pip install --upgrade" in result["error"] + + +# --------------------------------------------------------------------------- +# Sub-issue 5: sample_for_viz +# --------------------------------------------------------------------------- + +def test_sample_for_viz_reduces_node_count(): + """sample_for_viz should produce a subgraph under MAX_NODES_FOR_VIZ.""" + G, communities, labels = _make_large_graph(n_communities=30, nodes_per=200) + assert G.number_of_nodes() > MAX_NODES_FOR_VIZ + + sampled_G, sampled_comm = sample_for_viz(G, communities) + assert sampled_G.number_of_nodes() <= MAX_NODES_FOR_VIZ + assert sampled_G.number_of_nodes() > 0 + + +def test_sample_for_viz_keeps_top_communities(): + """Sampled result should contain the largest communities.""" + G, communities, labels = _make_large_graph(n_communities=30, nodes_per=200) + + sampled_G, sampled_comm = sample_for_viz(G, communities, max_communities=5) + assert len(sampled_comm) == 5 + + +def test_sample_for_viz_picks_highest_degree_nodes(): + """Within a community, the highest-degree nodes should be selected.""" + G = nx.Graph() + members = [] + for i in range(100): + nid = f"n{i}" + G.add_node(nid, label=f"Node{i}", file_type="code", source_file="f.py") + members.append(nid) + # Make n0 a hub connected to all others + for i in range(1, 100): + G.add_edge("n0", f"n{i}", relation="calls", confidence="EXTRACTED") + + communities = {0: members} + + sampled_G, sampled_comm = sample_for_viz(G, communities, nodes_per_community=10) + # n0 (degree 99) must be in the sample + assert "n0" in sampled_comm[0] + assert len(sampled_comm[0]) == 10 + + +def test_to_html_sample_mode_no_error(tmp_path): + """to_html with sample=True should succeed on large graphs instead of raising.""" + G, communities, labels = _make_large_graph(n_communities=30, nodes_per=200) + assert G.number_of_nodes() > MAX_NODES_FOR_VIZ + + out = tmp_path / "graph.html" + to_html(G, communities, str(out), community_labels=labels, sample=True) + assert out.exists() + content = out.read_text(encoding="utf-8") + assert "vis-network" in content or "vis.js" in content + + +def test_to_html_raises_without_sample(): + """to_html without sample=True should raise ValueError on large graphs.""" + G, communities, labels = _make_large_graph(n_communities=30, nodes_per=200) + with pytest.raises(ValueError, match="too large"): + to_html(G, communities, "out.html", community_labels=labels, sample=False) + + +def test_to_html_small_graph_no_sample_needed(tmp_path): + """Small graphs should work without sample flag.""" + G, communities, labels = _make_small_graph() + out = tmp_path / "graph.html" + to_html(G, communities, str(out), community_labels=labels) + assert out.exists() + + +# --------------------------------------------------------------------------- +# Sub-issue 8: extraction progress logging +# --------------------------------------------------------------------------- + +def test_extract_logs_progress_for_large_batches(tmp_path, caplog): + """extract() should log progress when processing >= 100 files.""" + from graphify.extract import extract + + # Create 150 tiny Python files + files = [] + for i in range(150): + f = tmp_path / f"mod_{i}.py" + f.write_text(f"def func_{i}(): pass\n") + files.append(f) + + with caplog.at_level(logging.INFO, logger="graphify.extract"): + result = extract(files) + + assert "nodes" in result + # Should have logged at least one progress message + progress_msgs = [r for r in caplog.records if "Extraction progress" in r.message] + assert len(progress_msgs) >= 1 + + +def test_extract_no_progress_for_small_batches(tmp_path, caplog): + """extract() should not log progress for small batches (< 100 files).""" + from graphify.extract import extract + + files = [] + for i in range(5): + f = tmp_path / f"mod_{i}.py" + f.write_text(f"def func_{i}(): pass\n") + files.append(f) + + with caplog.at_level(logging.INFO, logger="graphify.extract"): + result = extract(files) + + assert "nodes" in result + progress_msgs = [r for r in caplog.records if "Extraction progress" in r.message] + assert len(progress_msgs) == 0