feat: prepare Entire Graph for Graphify advertised benchmark parity

README.md

@@ -250,6 +250,18 @@ configs) will be public soon.
 
 ## More
 
+### Compact snapshot artifact
+
+`entire graph snapshot --repo . --format ndjson` remains the interoperable default: it is the existing object-per-line stream. For a complete, local compact artifact, use:
+
+```sh
+entire graph snapshot --repo . --format compact-ndjson > graph.compact.ndjson
+entire graph snapshot-query --input graph.compact.ndjson --symbol Cache.Refresh --format ndjson
+entire graph snapshot-query --input graph.compact.ndjson --from '<stable-id>' --relation CALLS --format ndjson
+```
+
+Compact NDJSON v1 is full-snapshot-only; targeted `--to`, `--from`, and `--relation` output stays native NDJSON. Its first `h` line is the only version marker, dictionary `d` lines are part of the artifact and its raw byte count, and unknown versions are rejected. The compact and native streams must have the same decoded public projection and canonical semantic SHA-256; hash equality alone is not a losslessness proof. Compact cache entries use a separate namespace from native snapshot entries.
+
 - [AGENTS.md](AGENTS.md) — the agent operating guide (also: `entire graph agent-guide`)
 - [docs/DETAILS.md](docs/DETAILS.md) — full command reference, architecture, language support,
   performance and accuracy benchmarks, security model

cmd/graph-bench/main.go

@@ -20,10 +20,10 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime/pprof"
 	"sort"
 	"strconv"
 	"strings"
@@ -63,41 +63,32 @@ func (r repoSpec) dirName() string  { return strings.ReplaceAll(r.repoPath, "/",
 
 func main() {
 	var (
-		manifestPath = flag.String("manifest", "bench/repos.json", "path to the repo manifest")
-		cacheDir     = flag.String("cache", "bench/.cache", "directory for cloned repos (gitignored)")
-		outDir       = flag.String("out", "bench/results", "directory for the JSON report, or - for stdout")
-		lockPath     = flag.String("lock", "bench/repos.lock.json", "path to the commit lock file")
-		languages    = flag.String("languages", "", "comma-separated language filter (default: all)")
-		limit        = flag.Int("limit", 0, "max repos per language (0 = all)")
-		jobs         = flag.Int("jobs", 4, "concurrent clone jobs")
-		depth        = flag.Int("depth", 1, "git clone depth")
-		skipClone    = flag.Bool("skip-clone", false, "do not clone; measure repos already in cache")
-		updateLock   = flag.Bool("update-lock", false, "resolve current commits and rewrite the lock file")
-		providerVer  = flag.String("provider-version", "dev", "provider version label recorded in the report")
-		profile      = flag.String("profile", "full", "indexing profile to measure: full, fast, or syntax-only")
-		progress     = flag.Bool("progress", false, "print provider phase progress to stderr")
-		minLOCPerSec = flag.Float64("min-loc-per-sec", 0, "fail if successful aggregate LOC/s is below this floor")
-		maxRSSBytes  = flag.Uint64("max-rss-bytes", 0, "fail if process peak RSS bytes exceeds this ceiling")
-		exactOutput  = flag.Bool("exact-output-bytes", false, "marshal every streamed record for exact NDJSON output bytes; slower on large repos")
-		cpuProfile   = flag.String("cpuprofile", "", "write a Go CPU profile for the benchmark process")
+		manifestPath  = flag.String("manifest", "bench/repos.json", "path to the repo manifest")
+		cacheDir      = flag.String("cache", "bench/.cache", "directory for cloned repos (gitignored)")
+		outDir        = flag.String("out", "bench/results", "directory for the JSON report, or - for stdout")
+		lockPath      = flag.String("lock", "bench/repos.lock.json", "path to the commit lock file")
+		languages     = flag.String("languages", "", "comma-separated language filter (default: all)")
+		limit         = flag.Int("limit", 0, "max repos per language (0 = all)")
+		jobs          = flag.Int("jobs", 4, "concurrent clone jobs")
+		depth         = flag.Int("depth", 1, "git clone depth")
+		skipClone     = flag.Bool("skip-clone", false, "do not clone; measure repos already in cache")
+		updateLock    = flag.Bool("update-lock", false, "resolve current commits and rewrite the lock file")
+		providerVer   = flag.String("provider-version", "dev", "provider version label recorded in the report")
+		profile       = flag.String("profile", "full", "indexing profile to measure: full, fast, or syntax-only")
+		progress      = flag.Bool("progress", false, "print provider phase progress to stderr")
+		minLOCPerSec  = flag.Float64("min-loc-per-sec", 0, "fail if successful aggregate LOC/s is below this floor")
+		maxRSSBytes   = flag.Uint64("max-rss-bytes", 0, "fail if any repository cold peak RSS exceeds this ceiling")
+		exactOutput   = flag.Bool("exact-output-bytes", false, "marshal every streamed record for exact NDJSON output bytes; slower on large repos")
+		cpuProfile    = flag.String("cpuprofile", "", "unsupported with mandatory isolated measurement workers")
+		measureWorker = flag.Bool("measure-worker", false, "serve one isolated measurement request on stdin")
 	)
 	flag.Parse()
-
-	if *cpuProfile != "" {
-		f, err := os.Create(*cpuProfile)
-		if err != nil {
-			fmt.Fprintln(os.Stderr, "graph-bench:", err)
-			os.Exit(1)
-		}
-		if err := pprof.StartCPUProfile(f); err != nil {
-			_ = f.Close()
-			fmt.Fprintln(os.Stderr, "graph-bench:", err)
-			os.Exit(1)
-		}
-		defer func() {
-			pprof.StopCPUProfile()
-			_ = f.Close()
-		}()
+	if *measureWorker {
+		os.Exit(bench.RunMeasureWorker(context.Background(), os.Stdin, os.Stdout))
+	}
+	if err := validateExecutionMode(*cpuProfile); err != nil {
+		fmt.Fprintln(os.Stderr, "graph-bench:", err)
+		os.Exit(1)
 	}
 
 	if err := run(*manifestPath, *cacheDir, *outDir, *lockPath, *languages, *profile, *limit, *jobs, *depth, *skipClone, *updateLock, *providerVer, *progress, *minLOCPerSec, *maxRSSBytes, *exactOutput); err != nil {
@@ -106,7 +97,22 @@ func main() {
 	}
 }
 
+func validateExecutionMode(cpuProfile string) error {
+	if strings.TrimSpace(cpuProfile) != "" {
+		return fmt.Errorf("-cpuprofile is not supported with mandatory isolated measurement workers; parent-only profiles would omit provider work")
+	}
+	return nil
+}
+
 func run(manifestPath, cacheDir, outDir, lockPath, languages, profileName string, limit, jobs, depth int, skipClone, updateLock bool, providerVer string, progress bool, minLOCPerSec float64, maxRSSBytes uint64, exactOutputBytes bool) error {
+	executable, err := os.Executable()
+	if err != nil {
+		return fmt.Errorf("resolve graph-bench executable: %w", err)
+	}
+	return runWithWorkerCommand(manifestPath, cacheDir, outDir, lockPath, languages, profileName, limit, jobs, depth, skipClone, updateLock, providerVer, progress, minLOCPerSec, maxRSSBytes, exactOutputBytes, []string{executable, "-measure-worker"})
+}
+
+func runWithWorkerCommand(manifestPath, cacheDir, outDir, lockPath, languages, profileName string, limit, jobs, depth int, skipClone, updateLock bool, providerVer string, progress bool, minLOCPerSec float64, maxRSSBytes uint64, exactOutputBytes bool, workerCommand []string) error {
 	profile, err := parseProfile(profileName)
 	if err != nil {
 		return err
@@ -157,20 +163,10 @@ func run(manifestPath, cacheDir, outDir, lockPath, languages, profileName string
 		opts := bench.MeasureOptions{MaxRSSBytes: maxRSSBytes, ExactOutputBytes: exactOutputBytes}
 		if progress {
 			opts.Progress = func(event sem.ProgressEvent) {
-				fmt.Fprintf(os.Stderr, "  progress %-40s phase=%s files=%d/%d symbols=%d relations=%d heap=%d rss=%d elapsed=%s\n",
-					spec.repoPath,
-					event.Phase,
-					event.FilesDone,
-					event.FilesTotal,
-					event.Symbols,
-					event.Relations,
-					event.HeapAlloc,
-					event.MaxRSSBytes,
-					event.Elapsed.Round(time.Millisecond),
-				)
+				fmt.Fprint(os.Stderr, formatProgress(spec.repoPath, event))
 			}
 		}
-		m, measureErr := bench.MeasureRepoWithOptions(ctx, spec.repoPath, spec.language, dir, providerVer, profile, opts)
+		m, measureErr := bench.MeasureRepoIsolated(ctx, spec.repoPath, spec.language, dir, providerVer, profile, opts, workerCommand)
 		if measureErr != nil {
 			fmt.Fprintf(os.Stderr, "  FAIL %-40s %v\n", spec.repoPath, measureErr)
 		} else {
@@ -187,12 +183,33 @@ func run(manifestPath, cacheDir, outDir, lockPath, languages, profileName string
 	if minLOCPerSec > 0 && report.Totals.LOCPerSec < minLOCPerSec {
 		return fmt.Errorf("performance guardrail failed: total LOC/s %.2f below floor %.2f", report.Totals.LOCPerSec, minLOCPerSec)
 	}
-	if maxRSSBytes > 0 && report.MaxRSSBytes > maxRSSBytes {
-		return fmt.Errorf("memory guardrail failed: max RSS %d exceeds ceiling %d", report.MaxRSSBytes, maxRSSBytes)
+	maxObservedRSS := uint64(0)
+	for _, metric := range metrics {
+		if metric.MaxRSSBytes > maxObservedRSS {
+			maxObservedRSS = metric.MaxRSSBytes
+		}
+	}
+	if maxRSSBytes > 0 && maxObservedRSS > maxRSSBytes {
+		return fmt.Errorf("memory guardrail failed: max cold RSS %d exceeds ceiling %d", maxObservedRSS, maxRSSBytes)
 	}
 	return nil
 }
 
+func formatProgress(repoPath string, event sem.ProgressEvent) string {
+	return fmt.Sprintf("  progress %-40s phase=%s files=%d/%d symbols=%d relations=%d heap=%d rss=%d phase_elapsed=%s elapsed=%s\n",
+		repoPath,
+		event.Phase,
+		event.FilesDone,
+		event.FilesTotal,
+		event.Symbols,
+		event.Relations,
+		event.HeapAlloc,
+		event.MaxRSSBytes,
+		event.PhaseElapsed.Round(time.Millisecond),
+		event.Elapsed.Round(time.Millisecond),
+	)
+}
+
 func loadSpecs(manifestPath, languages string, limit int) ([]repoSpec, error) {
 	data, err := os.ReadFile(manifestPath)
 	if err != nil {
@@ -363,7 +380,11 @@ func emitReport(report bench.Report, outDir string) error {
 }
 
 func printSummary(report bench.Report) {
-	w := tabwriter.NewWriter(os.Stderr, 0, 2, 2, ' ', 0)
+	writeSummary(os.Stderr, report)
+}
+
+func writeSummary(output io.Writer, report bench.Report) {
+	w := tabwriter.NewWriter(output, 0, 2, 2, ' ', 0)
 	fmt.Fprintln(w, "\nLANGUAGE\tREPOS\tFILES\tLOC\tSYMBOLS\tRELATIONS\tLOC/S\tPARSE_FAIL")
 	languages := make([]string, 0, len(report.ByLanguage))
 	for language := range report.ByLanguage {
@@ -376,5 +397,26 @@ func printSummary(report bench.Report) {
 	}
 	t := report.Totals
 	fmt.Fprintf(w, "TOTAL\t%d\t%d\t%d\t%d\t%d\t%.0f\t%d\n", t.Repos, t.Files, t.LOC, t.Symbols, t.Relations, t.LOCPerSec, t.ParseFailures)
+	fmt.Fprintln(w, "\nPHASE\tMS\tSHARE")
+	phaseTotal := 0.0
+	for _, elapsed := range t.PhaseMS {
+		phaseTotal += elapsed
+	}
+	for _, phase := range []string{"inventory", "parse", "relations", "finalize"} {
+		elapsed := t.PhaseMS[phase]
+		share := 0.0
+		if phaseTotal > 0 {
+			share = elapsed * 100 / phaseTotal
+		}
+		fmt.Fprintf(w, "%s\t%.2f\t%.2f%%\n", phase, elapsed, share)
+	}
 	w.Flush()
+	fmt.Fprintf(output, "ARTIFACT native_raw=%d compact_raw=%d compact_dictionary=%d projected_facts=%d native_bytes/fact=%.2f compact_bytes/fact=%.2f\n",
+		t.NDJSONRawBytes,
+		t.CompactRawBytes,
+		t.CompactDictionaryBytes,
+		t.ProjectedFacts,
+		t.NDJSONBytesPerProjectedFact,
+		t.CompactBytesPerProjectedFact,
+	)
 }

cmd/graph-bench/main_test.go

@@ -1,15 +1,96 @@
 package main
 
 import (
+	"bytes"
+	"context"
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/entireio/entire-graph/internal/bench"
 	"github.com/entireio/entire-graph/internal/sem"
 )
 
+func TestFormatProgressIncludesTypedPhaseElapsedAndTotalElapsed(t *testing.T) {
+	got := formatProgress("owner/repo", sem.ProgressEvent{
+		Phase:        sem.BuildPhaseParse,
+		FilesDone:    3,
+		FilesTotal:   5,
+		PhaseElapsed: 12 * time.Millisecond,
+		Elapsed:      30 * time.Millisecond,
+	})
+	if !strings.Contains(got, "phase=parse") || !strings.Contains(got, "phase_elapsed=12ms") || !strings.Contains(got, "elapsed=30ms") {
+		t.Fatalf("progress = %q", got)
+	}
+}
+
+func TestWriteSummaryPrintsPhaseSharesAndArtifactMetrics(t *testing.T) {
+	report := bench.Report{Totals: bench.Aggregate{
+		Repos: 1, Files: 2, LOC: 10, Symbols: 3, Relations: 4, WallMS: 100,
+		PhaseMS:        map[string]float64{"inventory": 10, "parse": 50, "relations": 30, "finalize": 10},
+		NDJSONRawBytes: 1000, CompactRawBytes: 500, CompactDictionaryBytes: 100,
+		ProjectedFacts: 10, NDJSONBytesPerProjectedFact: 100, CompactBytesPerProjectedFact: 50,
+	}}
+	var out bytes.Buffer
+	writeSummary(&out, report)
+	got := out.String()
+	for _, want := range []string{"PHASE", "inventory", "50.00%", "native_raw=1000", "compact_raw=500", "native_bytes/fact=100.00", "compact_bytes/fact=50.00"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("summary missing %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestValidateExecutionModeRejectsParentOnlyCPUProfile(t *testing.T) {
+	err := validateExecutionMode("cpu.out")
+	if err == nil || !strings.Contains(err.Error(), "isolated") || !strings.Contains(err.Error(), "not supported") {
+		t.Fatalf("CPU profile validation error = %v", err)
+	}
+	if err := validateExecutionMode(""); err != nil {
+		t.Fatalf("empty CPU profile unexpectedly rejected: %v", err)
+	}
+}
+
+func TestMaxRSSGuardFailsRunEvenWhenViolatingRowIsExcludedFromAggregates(t *testing.T) {
+	dir := t.TempDir()
+	manifestPath := filepath.Join(dir, "manifest.json")
+	if err := os.WriteFile(manifestPath, []byte(`{"languages":{"Go":["owner/repo"]}}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	cacheDir := filepath.Join(dir, "cache")
+	repoDir := filepath.Join(cacheDir, "Go", "owner__repo")
+	if err := os.MkdirAll(repoDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(repoDir, "main.go"), []byte("package main\nfunc main() {}\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("ENTIRE_GRAPH_BENCH_MAIN_TEST_WORKER", "1")
+	worker := []string{os.Args[0], "-test.run=^TestGraphBenchMeasureWorker$"}
+	outDir := filepath.Join(dir, "out")
+	err := runWithWorkerCommand(manifestPath, cacheDir, outDir, filepath.Join(dir, "lock.json"), "", "fast", 0, 1, 1, true, false, "test", false, 0, 1, false, worker)
+	if err == nil || !strings.Contains(err.Error(), "memory guardrail failed") {
+		t.Fatalf("run guard error = %v", err)
+	}
+	report := readOnlyReport(t, outDir)
+	if len(report.Repos) != 1 || report.Repos[0].Error == "" || report.Repos[0].MaxRSSBytes <= 1 {
+		t.Fatalf("guard failure row lost from report: %#v", report.Repos)
+	}
+	if report.Totals.Repos != 0 {
+		t.Fatalf("guard failure row should remain excluded from aggregates: %#v", report.Totals)
+	}
+}
+
+func TestGraphBenchMeasureWorker(t *testing.T) {
+	if os.Getenv("ENTIRE_GRAPH_BENCH_MAIN_TEST_WORKER") != "1" {
+		return
+	}
+	os.Exit(bench.RunMeasureWorker(context.Background(), os.Stdin, os.Stdout))
+}
+
 func TestParseProfile(t *testing.T) {
 	cases := map[string]sem.Profile{
 		"":            sem.ProfileFull,

docs/DETAILS.md

@@ -15,6 +15,8 @@ It ships as an **Entire CLI plugin**, invoked as `entire graph ...`, and doubles
 ```sh
 entire graph search   --repo . --query "where is webhook retry handled?"   # 🔍 ranked code for a task
 entire graph snapshot --repo . --format ndjson                             # 🕸️  full symbol + relation graph
+entire graph snapshot --repo . --format compact-ndjson > graph.compact.ndjson # compact full graph artifact
+entire graph snapshot-query --input graph.compact.ndjson --symbol Cache.Refresh --format ndjson
 entire graph diff     --base main --head HEAD                               # 🧬 what changed, at the entity level
 entire graph capabilities --json                                           # 🧭 languages + relation types
 ```
@@ -292,6 +294,20 @@ Savings scale with symbol connectivity: single-digit for narrow symbols, 280x+ f
 - **Cached committed-tree search:** reuses a tree-keyed compressed index across invocations, in the platform's per-user cache directory unless `--cache-dir`/`ENTIRE_PLUGIN_DATA_DIR` redirect it, so repeated queries on an unchanged tree skip re-parsing. The working tree is never cached. A complete prepared index derives the exact query-selected view, so relation expansion cannot escape that file set.
 - **Explicit preindex:** `index --head` builds and verifies that query-independent artifact before latency-sensitive work; cached `search` and `neighbors` calls then report the hit directly.
 
+### Compact snapshot NDJSON v1
+
+Normal `snapshot --format ndjson` remains the interoperable default and retains its object-per-line schema. `snapshot --format compact-ndjson` is a separate, complete-snapshot-only native artifact: positional rows tagged `f`, `x`, `s`, and `r` reference deterministic first-seen dictionaries emitted as `d` rows; `h` is the required first header row and carries the sole v1 version marker; `m` is the required final summary. A decoder rejects unknown versions, malformed arity, duplicate headers, and missing summaries.
+
+Every header, dictionary, data, and summary line counts toward raw compact bytes—size claims never subtract dictionary overhead. Compact output uses a separate cache namespace. Consumers load it through `snapshot-query`, which returns deterministic native NDJSON symbol/relation records:
+
+```sh
+entire graph snapshot --repo . --format compact-ndjson > graph.compact.ndjson
+entire graph snapshot-query --input graph.compact.ndjson --symbol Cache.Refresh --format ndjson
+entire graph snapshot-query --input graph.compact.ndjson --from '<stable-id>' --relation CALLS --format ndjson
+```
+
+The canonical SHA-256 is calculated from normalized native records in record order, not compact bytes. A valid compact artifact must match native NDJSON in both that hash and decoded public projection; a matching hash alone is not a proof of losslessness.
+
 Absolute numbers are environment-sensitive (measured on Apple Silicon). Read them as relative signals and reproduce locally with the harness.
 
 ---