diff --git a/.github/workflows/self-analyze.yml b/.github/workflows/self-analyze.yml new file mode 100644 index 00000000..de122072 --- /dev/null +++ b/.github/workflows/self-analyze.yml @@ -0,0 +1,177 @@ +# Grafema self-analyze regression guard +# +# WHY THIS EXISTS +# --------------- +# Grafema's CI runs CodeQL "Analyze" jobs — it does NOT run *Grafema's own* +# `grafema analyze` against this repo. That gap is exactly what let a whole class +# of self-scale bugs ship silently: a derive-phase SIGSEGV (compaction vs. +# parallel-derive heap corruption) plus several conservative planner/eval caps +# (max_intermediate_results, MAX_MATERIALIZED_FACTS) that only bite when grafema +# analyzes its OWN ~714k-node / 1.33M-edge monorepo. None of that is reachable by +# the unit/integration tests, which run on tiny fixture graphs. See +# fix/derive-batch-cap (Disentinel/grafema#469). +# +# WHAT THIS JOB DOES +# ------------------ +# 1. Builds grafema as it ships: pnpm build (TS) + cargo --release for the two +# native binaries the analyze pipeline spawns (rfdb-server + orchestrator). +# 2. Runs `grafema analyze` against this repo using the checked-in dogfooding +# config (.grafema/config.yaml), with the env that makes self-analyze pass. +# 3. Asserts the run reached exit 0 AND the JS enrichers actually fired — the +# graph must contain `mcp:tool` nodes (produced by the behavior enricher on +# packages/mcp/src/server.ts). A green exit with an empty graph is a silent +# failure we explicitly catch. +# +# RUNNER / SCOPE CHOICE +# --------------------- +# Runs the FULL monorepo self-analyze (not a scoped subset) so it exercises all +# ~40 derive packs — including the ones that triggered the SIGSEGV +# (@stdlib/js_local_refs) and the cap rejects (js_property_access_full). A scoped +# single-package analyze would NOT reproduce those cross-package derive paths, so +# scoping was rejected in favor of the full run. +# Runner: ubuntu-latest (GitHub-hosted: 16 GB RAM / 4 vCPU). A full self-analyze +# peaks ~7.5 GB RAM and ~14 min wall — it fits in 16 GB with headroom. The job +# timeout covers the Rust release build (~10-15 min, cached after the first run) +# plus the analyze. If a future graph outgrows 16 GB, switch `runs-on` to a larger +# GitHub-hosted runner label, or scope the analyze via `--service mcp --service +# cli` (note: --service currently takes a single name; would need a small CLI +# change or a per-service loop). +# +# NON-BLOCKING — READ BEFORE MAKING REQUIRED +# ------------------------------------------ +# This job is intentionally NON-BLOCKING (continue-on-error: true) and is NOT a +# required status check. As of fix/derive-batch-cap the derive phase completes +# with 0 SIGSEGV, but the run still fails LATE on an enricher RPC 60s client +# timeout (being fixed separately on fix/enricher-rpc-timeout). Until a full +# self-analyze is green end-to-end, this job RUNS and SURFACES the result without +# blocking PRs. +# +# TODO(make-required): once fix/enricher-rpc-timeout lands and a full self-analyze +# reaches exit 0 with mcp:tool > 0 reliably, remove `continue-on-error: true` +# below AND add `self-analyze` to the branch-protection required checks for `main` +# so this class of bug can never regress silently again. + +name: Self-Analyze + +on: + # Run on PRs so regressions surface in review, and on the fix branch itself. + # Deliberately NOT wired into every push to main yet (it's non-blocking and + # still fails late — see header). Flip this on when made required. + pull_request: + branches: [main, ai-dev] + push: + branches: [fix/derive-batch-cap, fix/ci-self-analyze] + workflow_dispatch: {} + +env: + NODE_VERSION: '22' + CARGO_TERM_COLOR: always + RUST_BACKTRACE: '1' + # Self-analyze env from fix/derive-batch-cap — lifts the interactive deadline + # and the planner fact-guard so the full 40-pack derive completes at self-scale. + RFDB_MATERIALIZE_DEADLINE_SECS: '3600' + RFDB_MAX_MATERIALIZED_FACTS: '200000000' + +jobs: + self-analyze: + name: grafema analyze (self, non-blocking) + runs-on: ubuntu-latest + # Non-blocking: surface the result, never block a PR — see header TODO. + continue-on-error: true + # Generous: Rust release build (~10-15 min cold) + ~14 min analyze. + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'pnpm' + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + packages/rfdb-server -> target + packages/grafema-orchestrator -> target + key: self-analyze + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Build TS packages + run: pnpm build + + # The analyze pipeline spawns two native binaries; the CLI auto-detects them + # from packages//target/release/ (findRfdbBinary / findOrchestratorBinary). + - name: Build rfdb-server (release) + working-directory: packages/rfdb-server + run: cargo build --release + + - name: Build grafema-orchestrator (release) + working-directory: packages/grafema-orchestrator + run: cargo build --release + + - name: Show free memory before analyze + run: free -h || true + + # The repo ships .grafema/config.yaml (dogfooding config, full monorepo). + # --clear rebuilds from scratch; --no-auto-start is NOT used so the CLI + # brings up the freshly-built rfdb-server itself. + - name: Run grafema self-analyze + id: analyze + run: | + set -o pipefail + node packages/cli/dist/cli.js analyze --clear --verbose 2>&1 | tee /tmp/self-analyze.log + echo "Analyze exited 0." + + # Assert the JS enrichers fired: the behavior enricher emits mcp:tool nodes + # for packages/mcp/src/server.ts. A clean exit with an empty graph is the + # silent-failure mode this guard exists to catch. + - name: Assert JS enrichers fired (mcp:tool nodes present) + run: | + set -o pipefail + node packages/cli/dist/cli.js query --type "mcp:tool" "" --json -l 100000 2>/dev/null > /tmp/mcp-tool.json + COUNT=$(node -e "const a=JSON.parse(require('fs').readFileSync('/tmp/mcp-tool.json','utf8')); console.log(Array.isArray(a)?a.length:0)") + echo "mcp:tool node count: $COUNT" + echo "mcp:tool nodes: $COUNT" >> "$GITHUB_STEP_SUMMARY" + if [ "$COUNT" -lt 1 ]; then + echo "::error::self-analyze produced 0 mcp:tool nodes — JS enrichers did not fire (graph is empty/incomplete)." + exit 1 + fi + echo "OK: JS enrichers fired ($COUNT mcp:tool nodes)." + + # Best-effort SIGSEGV / derive-reject canary. The derive phase must finish + # without a segfault or plan rejects; flag them loudly even though the job + # is non-blocking for now. + - name: Check derive phase for SIGSEGV / plan rejects + if: always() + run: | + if grep -qiE "SIGSEGV|signal: 11|segmentation fault" /tmp/self-analyze.log 2>/dev/null; then + echo "::error::SIGSEGV detected in self-analyze — derive-phase heap corruption regressed." + grep -iE "SIGSEGV|signal: 11|segmentation fault" /tmp/self-analyze.log | head >> "$GITHUB_STEP_SUMMARY" || true + exit 1 + fi + if grep -qiE "plan reject|rejected by planner|MAX_MATERIALIZED_FACTS guard" /tmp/self-analyze.log 2>/dev/null; then + echo "::warning::Derive plan rejects present in self-analyze log (planner cap hit)." + fi + echo "No SIGSEGV in derive phase." + + - name: Upload self-analyze log + if: always() + uses: actions/upload-artifact@v4 + with: + name: self-analyze-log + path: | + /tmp/self-analyze.log + /tmp/mcp-tool.json + retention-days: 14 diff --git a/packages/rfdb-server/src/bin/rfdb_server.rs b/packages/rfdb-server/src/bin/rfdb_server.rs index 5e126807..60477cd8 100644 --- a/packages/rfdb-server/src/bin/rfdb_server.rs +++ b/packages/rfdb-server/src/bin/rfdb_server.rs @@ -3045,11 +3045,32 @@ fn dispatch_materialize_datalog( limits.deadline = Some(std::time::Instant::now() + std::time::Duration::from_secs(materialize_deadline_secs)); + // Same E-EXEC-001 reasoning as the deadline above: the 100k `max_intermediate_results` is an + // interactive-query bound and is WRONG for a batch full materialize. Intermediate relations of + // legitimate stdlib packs (e.g. js_local_refs scope-walk: ref_scope/inner_of/chain_declares) + // scale ~linearly with reference count, so a cold materialize over a non-trivial graph routinely + // exceeds 100k rows in a single stratum (the ~21k-node mcp graph already produces ~141k). The + // batch path is build-step-like; its real guards are the deadline + cancellation + memory, NOT a + // fixed row count. Lift it (default: unbounded). Tunable via `RFDB_MATERIALIZE_MAX_INTERMEDIATE`. + let materialize_max_intermediate = std::env::var("RFDB_MATERIALIZE_MAX_INTERMEDIATE") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(usize::MAX); + limits.max_intermediate_results = materialize_max_intermediate; + // Empty source ⇒ the canonical bundled depends.dl; "@stdlib/" ⇒ a named bundled // pack (E-MAT-007 when unknown); anything else ⇒ explicit program text. The single // source of truth is `resolve_pack_source` — no drift between dispatchers. let program = resolve_pack_source(source)?; + // Suppress auto-compaction for the duration of this materialize: committing derived edges would + // otherwise auto-trigger compaction, which (rayon, memmap2) rewrites/unmaps segments while this + // same materialize's `par_join_rows` rayon tasks read them → use-after-unmap heap corruption → + // SIGSEGV (confirmed root cause of the parallel-derive crash, 2026-06-19). Deferred compaction + // runs at the natural barrier (end_bulk_load / explicit Compact) under the exclusive write lock. + let _compaction_guard = + rfdb::storage_v2::compaction::coordinator::AutoCompactionSuppressGuard::new(); + // Gate D2: the cached entry MAINTAINS the derived relations across calls against this // long-lived per-database engine (work-proportional on the 2nd+ run) and commits only the // edge delta — additions AND tombstones of stale edges, so a reanalysis supersedes obsolete diff --git a/packages/rfdb-server/src/derive/plan.rs b/packages/rfdb-server/src/derive/plan.rs index 4ed89edf..e2e9f568 100644 --- a/packages/rfdb-server/src/derive/plan.rs +++ b/packages/rfdb-server/src/derive/plan.rs @@ -40,8 +40,24 @@ use super::stratify::Stratification; /// Per-rule materialization ceiling (spec §3): a rule whose plan-time output estimate /// exceeds this is rejected with `E-PLAN-003`. Ten million facts. +/// +/// This is the DEFAULT — a conservative guard against cross-join blow-ups whose plan-time +/// estimate is also q-error-prone (often an OVERestimate). For a large batch self-analyze the +/// 10M default is too low (a legitimate per-rule output like js property-access READS_FROM on a +/// ~700k-node graph estimates ~11.7M), so the effective ceiling is env-overridable via +/// [`max_materialized_facts`] / `RFDB_MAX_MATERIALIZED_FACTS`. pub const MAX_MATERIALIZED_FACTS: u64 = 10_000_000; +/// The effective per-rule materialization ceiling: `RFDB_MAX_MATERIALIZED_FACTS` if set and +/// parseable, else [`MAX_MATERIALIZED_FACTS`]. Read per check (cheap); lets a batch analyze raise +/// the guard without recompiling. +pub fn max_materialized_facts() -> u64 { + std::env::var("RFDB_MAX_MATERIALIZED_FACTS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(MAX_MATERIALIZED_FACTS) +} + // ── Error taxonomy (invariant I5) ────────────────────────────────── @@ -361,13 +377,14 @@ fn plan_rule_with( } // §3 per-rule materialization guard. - if rule_estimate > MAX_MATERIALIZED_FACTS { + let guard = max_materialized_facts(); + if rule_estimate > guard { return Err(PlanError { code: PlanCode::GuardRejected, head: head.clone(), detail: format!( "per-rule output estimate {} exceeds max_materialized_facts {}", - rule_estimate, MAX_MATERIALIZED_FACTS + rule_estimate, guard ), }); } diff --git a/packages/rfdb-server/src/storage_v2/compaction/coordinator.rs b/packages/rfdb-server/src/storage_v2/compaction/coordinator.rs index f657c8dd..0a78130d 100644 --- a/packages/rfdb-server/src/storage_v2/compaction/coordinator.rs +++ b/packages/rfdb-server/src/storage_v2/compaction/coordinator.rs @@ -23,6 +23,38 @@ use crate::storage_v2::shard::{Shard, TombstoneSet}; use crate::storage_v2::types::{SegmentMeta, SegmentType}; use crate::storage_v2::writer::{EdgeSegmentWriter, NodeSegmentWriter}; +// ── Auto-compaction suppression during derive/materialize ─────────── +// +// A derive/materialize handler holds a `db.engine.read()` lock but internally commits derived +// edges across many rule packs; those commits would auto-trigger compaction. Compaction (rayon, +// memmap2) rewrites/unmaps segments while the SAME materialize's `par_join_rows` rayon tasks read +// them → use-after-unmap / heap corruption → SIGSEGV (confirmed by isolation test 2026-06-19). +// While a materialize is in progress we SUPPRESS auto-compaction; the deferred compaction runs at +// the natural barrier (end_bulk_load / explicit Compact) under the exclusive write lock. +static AUTO_COMPACTION_SUPPRESSED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Suppress (or re-enable) automatic compaction. Set while a derive/materialize is in progress. +pub fn set_auto_compaction_suppressed(suppressed: bool) { + AUTO_COMPACTION_SUPPRESSED.store(suppressed, std::sync::atomic::Ordering::SeqCst); +} + +/// RAII guard: suppresses auto-compaction for its lifetime, restoring on drop (even on panic). +pub struct AutoCompactionSuppressGuard; + +impl AutoCompactionSuppressGuard { + pub fn new() -> Self { + set_auto_compaction_suppressed(true); + AutoCompactionSuppressGuard + } +} + +impl Drop for AutoCompactionSuppressGuard { + fn drop(&mut self) { + set_auto_compaction_suppressed(false); + } +} + // ── Policy ────────────────────────────────────────────────────────── /// Check if a shard should be compacted based on L0 segment count. @@ -32,6 +64,14 @@ use crate::storage_v2::writer::{EdgeSegmentWriter, NodeSegmentWriter}; /// /// Complexity: O(1) pub fn should_compact(shard: &Shard, config: &CompactionConfig) -> bool { + // Suppressed while a derive/materialize is in progress (see AUTO_COMPACTION_SUPPRESSED) — the + // confirmed fix for the parallel-derive SIGSEGV (compaction unmapping segments under parallel + // reads). RFDB_DISABLE_COMPACTION=1 is a manual override for diagnostics. + if AUTO_COMPACTION_SUPPRESSED.load(std::sync::atomic::Ordering::SeqCst) + || std::env::var_os("RFDB_DISABLE_COMPACTION").is_some() + { + return false; + } let total_l0 = shard.l0_node_segment_count() + shard.l0_edge_segment_count(); total_l0 >= config.segment_threshold }