Skip to content

fix: bolt optimization of graph exports #1012

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
n24q02m wants to merge 1 commit into from
Closed

fix: bolt optimization of graph exports #1012

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .jules/bolt.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ Every landed entry is anchored to a commit. If an entry cannot be located in

**Action:** To optimize querying aggregate graph statistics in `GraphStore` (e.g., `get_stats` in `src/better_code_review_graph/graph.py`), derive absolute totals (`total_nodes`, `total_edges`, `files_count`) in Python by summing the values of grouped queries (`sum(nodes_by_kind.values())`) rather than executing redundant `COUNT(*)` subqueries, reducing database roundtrips.

### 2026-09-04 - Avoid full object materialization in graph exports

**Learning:** When exporting the entire graph to formats like JSON-LD, GraphML, DOT, or Cypher, calling `get_all_nodes` and `get_all_edges` triggers `.fetchall()`-like behavior by materializing the entire SQLite table into heavy Python list representations of `GraphNode` and `GraphEdge` objects. This leads to massive peak memory usage for large graphs (e.g. 80MB vs 38MB in our benchmark for 50k nodes).

**Action:** In `exporter.py`, iterate directly over the database cursor (`for node in store._conn.execute("SELECT * FROM nodes")`) and access the SQLite `Row` dictionary directly (e.g., `node["qualified_name"]`) instead of instantiating `GraphNode`/`GraphEdge` wrappers, eliminating unnecessary memory overhead.

## Rejected

Proposals that were evaluated with measurements and declined. The reasoning is
Expand Down
103 changes: 53 additions & 50 deletions src/better_code_review_graph/exporter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,74 +104,77 @@ def export_graphml(store: GraphStore) -> str:
root, f"{{{GRAPHML_NS}}}graph", {"id": "G", "edgedefault": "directed"}
)

for node in store.get_all_nodes():
for node in store._conn.execute("SELECT * FROM nodes"):
n_el = standard_ET.SubElement(
graph, f"{{{GRAPHML_NS}}}node", {"id": node.qualified_name}
graph, f"{{{GRAPHML_NS}}}node", {"id": node["qualified_name"]}
)
for k, v in (
("kind", node.kind),
("name", node.name),
("qualified_name", node.qualified_name),
("file_path", node.file_path),
("language", node.language),
("kind", node["kind"]),
("name", node["name"]),
("qualified_name", node["qualified_name"]),
("file_path", node["file_path"]),
("language", node["language"]),
):
if v is None or v == "":
continue
d = standard_ET.SubElement(n_el, f"{{{GRAPHML_NS}}}data", {"key": k})
d.text = str(v)
for k, v in (("line_start", node.line_start), ("line_end", node.line_end)):
for k, v in (
("line_start", node["line_start"]),
("line_end", node["line_end"]),
):
if v is None:
continue
d = standard_ET.SubElement(n_el, f"{{{GRAPHML_NS}}}data", {"key": k})
d.text = str(v)

for edge in store.get_all_edges():
for edge in store._conn.execute("SELECT * FROM edges"):
e_el = standard_ET.SubElement(
graph,
f"{{{GRAPHML_NS}}}edge",
{"source": edge.source_qualified, "target": edge.target_qualified},
{"source": edge["source_qualified"], "target": edge["target_qualified"]},
)
for k, v in (("edge_kind", edge.kind), ("edge_file", edge.file_path)):
for k, v in (("edge_kind", edge["kind"]), ("edge_file", edge["file_path"])):
if v is None or v == "":
continue
d = standard_ET.SubElement(e_el, f"{{{GRAPHML_NS}}}data", {"key": k})
d.text = str(v)
if edge.line is not None:
if edge["line"] is not None:
d = standard_ET.SubElement(
e_el, f"{{{GRAPHML_NS}}}data", {"key": "edge_line"}
)
d.text = str(edge.line)
d.text = str(edge["line"])

return standard_ET.tostring(root, encoding="unicode", xml_declaration=True)


def export_jsonld(store: GraphStore) -> str:
"""Emit JSON-LD with @context + nodes + edges arrays."""
nodes = []
for node in store.get_all_nodes():
for node in store._conn.execute("SELECT * FROM nodes"):
n: dict[str, object] = {
"@id": node.qualified_name,
"@type": node.kind,
"name": node.name,
"filePath": node.file_path,
"language": node.language,
"@id": node["qualified_name"],
"@type": node["kind"],
"name": node["name"],
"filePath": node["file_path"],
"language": node["language"],
}
if node.line_start is not None:
n["lineStart"] = node.line_start
if node.line_end is not None:
n["lineEnd"] = node.line_end
if node["line_start"] is not None:
n["lineStart"] = node["line_start"]
if node["line_end"] is not None:
n["lineEnd"] = node["line_end"]
nodes.append(n)
edges = []
for edge in store.get_all_edges():
for edge in store._conn.execute("SELECT * FROM edges"):
e: dict[str, object] = {
"source": edge.source_qualified,
"target": edge.target_qualified,
"kind": edge.kind,
"source": edge["source_qualified"],
"target": edge["target_qualified"],
"kind": edge["kind"],
}
if edge.file_path:
e["filePath"] = edge.file_path
if edge.line is not None:
e["line"] = edge.line
if edge["file_path"]:
e["filePath"] = edge["file_path"]
if edge["line"] is not None:
e["line"] = edge["line"]
edges.append(e)
return json.dumps(
{"@context": JSONLD_CONTEXT, "nodes": nodes, "edges": edges}, indent=2
Expand All @@ -181,13 +184,13 @@ def export_jsonld(store: GraphStore) -> str:
def export_dot(store: GraphStore) -> str:
"""Emit Graphviz DOT format (digraph)."""
lines = ["digraph G {"]
for node in store.get_all_nodes():
label = _safe_label(node.name or node.qualified_name)
lines.append(f' "{node.qualified_name}" [label="{label}"];')
for edge in store.get_all_edges():
kind = _safe_label(edge.kind)
for node in store._conn.execute("SELECT * FROM nodes"):
label = _safe_label(node["name"] or node["qualified_name"])
lines.append(f' "{node["qualified_name"]}" [label="{label}"];')
for edge in store._conn.execute("SELECT * FROM edges"):
kind = _safe_label(edge["kind"])
lines.append(
f' "{edge.source_qualified}" -> "{edge.target_qualified}" [label="{kind}"];'
f' "{edge["source_qualified"]}" -> "{edge["target_qualified"]}" [label="{kind}"];'
)
lines.append("}")
return "\n".join(lines)
Expand All @@ -196,22 +199,22 @@ def export_dot(store: GraphStore) -> str:
def export_cypher(store: GraphStore) -> str:
"""Emit Neo4j Cypher CREATE statements that recreate the graph."""
parts = []
for node in store.get_all_nodes():
kind_label = node.kind or "Node"
var = _cypher_var(node.qualified_name)
for node in store._conn.execute("SELECT * FROM nodes"):
kind_label = node["kind"] or "Node"
var = _cypher_var(node["qualified_name"])
props: dict[str, object] = {
"id": node.qualified_name,
"name": node.name,
"file_path": node.file_path,
"language": node.language,
"line_start": node.line_start,
"line_end": node.line_end,
"id": node["qualified_name"],
"name": node["name"],
"file_path": node["file_path"],
"language": node["language"],
"line_start": node["line_start"],
"line_end": node["line_end"],
}
parts.append(f"CREATE ({var}:{kind_label} {{{_cypher_props(props)}}});")
for edge in store.get_all_edges():
kind = (edge.kind or "RELATED").upper().replace("-", "_")
src_escaped = edge.source_qualified.replace("'", "\\'")
tgt_escaped = edge.target_qualified.replace("'", "\\'")
for edge in store._conn.execute("SELECT * FROM edges"):
kind = (edge["kind"] or "RELATED").upper().replace("-", "_")
src_escaped = edge["source_qualified"].replace("'", "\\'")
tgt_escaped = edge["target_qualified"].replace("'", "\\'")
parts.append(
f"MATCH (a {{id: '{src_escaped}'}}), (b {{id: '{tgt_escaped}'}}) "
f"CREATE (a)-[:{kind}]->(b);"
Expand Down