From ed2fa711e08fe159302cad2e12a64169e531bbc1 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Mon, 24 Aug 2026 08:42:46 -0500 Subject: [PATCH 1/8] Added ability to deploy a watcher without needing to use webworker. Added deploy with --fresh flag for a clean redeploy (all embeddings wiped) --- .env.example | 7 +++ DOCKER-README.md | 16 ++++-- Dockerfile | 6 ++- deploy-sources.sh | 60 +++++++++++++++++++++ deploy.sh | 84 ++++++++++++++++++++++++++--- docker-compose.yml | 8 +++ quickbeam/ingest/sources/fangorn.py | 16 ++++-- quickbeam/watcher.py | 36 ++++++++++--- tests/test_sources.py | 3 ++ tests/test_wildcard_cdn_domain.py | 70 ++++++++++++++++++++++++ 10 files changed, 286 insertions(+), 20 deletions(-) create mode 100755 deploy-sources.sh create mode 100644 tests/test_wildcard_cdn_domain.py diff --git a/.env.example b/.env.example index 364a308..5189371 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,13 @@ SOURCES_URL=https://quickbeam-registry..workers.dev/watchlist SOURCES_REFRESH=60 +# Replay from this block before going live. Only meaningful for a WILDCARD source +# (`APP::` / `*:*`), which cannot be seeded with `fangorn read` and so would otherwise +# see nothing published before the box started. 0 (the default) means no replay. +# Cost: 1000 blocks per eth_getLogs, so pick a block just before the commits you want — +# a million blocks back is a thousand sequential calls on the public RPC. +FROM_BLOCK=0 + # Fallback app for a watch-list entry that names none. Every entry the worker serves # carries its own app, so this only covers a hand-written list — but keep it equal to # the worker's DEFAULT_APP, which is what it stamps on a view created without one. diff --git a/DOCKER-README.md b/DOCKER-README.md index e061d19..978edab 100644 --- a/DOCKER-README.md +++ b/DOCKER-README.md @@ -119,7 +119,8 @@ silently reuses the old layer and reproduces bugs you already fixed. To rebuild starting anything, `docker compose build`. For a static list, write `sources.json` into the shared volume and set -`SOURCES_URL=file:///data/sources.json`: +`SOURCES_URL=file:///data/sources.json` (`./deploy-sources.sh` does both halves on +the box — it is `deploy.sh` plus a `docker compose cp` of the file into the volume): ```json [ {"app": "fangorn", "owner": "0x147c24c5Ea2f1EE1ac42AD16820De23bBba45Ef6", @@ -302,6 +303,14 @@ gcloud compute ssh quickbeam-1 --zone=$REGION-a --command='docker compose pull & Compose recreates only the services whose image actually changed, and the `qdrant` and `data` volumes are untouched, so the collection and the baked shards survive. +`./deploy.sh --fresh` does the opposite deliberately: it prefixes the remote step with +`docker compose down -v`, so the box comes back with no vectors, no ingest checkpoint +and no shards and re-embeds everything. All three have to go together — the checkpoint +alone would make the watcher skip every record it has already seen and report "no new +records" over an empty collection. It refuses to wipe unless the box's `FROM_BLOCK` is +set, because a wildcard (`*:*`) watch list has no seed read and history replay is the +only way anything comes back. + **Machine type.** `e2-medium` (4GB) is the working default above. `e2-small` (2GB) runs but leaves little headroom — if a seed OOMs there, stop the instance, change the type and start it again; the disk survives, so it costs a minute. `e2-micro` (1GB, free tier) is @@ -327,8 +336,9 @@ worker stores the view and its sources join the watchlist; this instance picks u A namespace another view already covers logs nothing at all — it is already embedded, and the new view queries the same points. That silence is the design working. -**Removing** is a founder action — `POST /admin/remove` with the view id, signed by a -wallet in `ADMIN_WALLETS`. See `webworker/quickbeam-registry/README.md`. A source stops +**Removing** is `POST /views/remove` with the view id, signed by the wallet that created +it (the website's "Stop watching" button), or `POST /admin/remove` for any view, signed +by a wallet in `ADMIN_WALLETS`. See `webworker/quickbeam-registry/README.md`. A source stops being watched only when the **last** view referencing it goes; nothing expires on its own, so a lapsed subscription keeps running until someone removes it. diff --git a/Dockerfile b/Dockerfile index 25b2504..d42212f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,11 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Provides the `fangorn` binary on PATH — the default --fangorn-bin. -RUN npm i -g @fangorn-network/sdk && npm cache clean --force +# PIN the version: the DataRegistry address rides inside the SDK's config.js, so an +# unpinned install baked into a cached layer silently keeps reading a retired registry +# (an image built 2026-08-13 was still on 0x9dfa…572c and saw none of the state +# published to 0x97d6…df91). Bump this when the registry moves. +RUN npm i -g @fangorn-network/sdk@2026.8.18-dev && npm cache clean --force WORKDIR /app COPY pyproject.toml ./ diff --git a/deploy-sources.sh b/deploy-sources.sh new file mode 100755 index 0000000..2b7afc2 --- /dev/null +++ b/deploy-sources.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Deploy quickbeam to the GCE box with a STATIC watch list: the JSON file in +# data/ instead of the registry worker's /watchlist. +# +# ./deploy-sources.sh [--env] [--dry-run] [path/to/sources.json] +# +# The image half is deploy.sh's job and is not duplicated here; this only adds +# the watch list. The file goes INTO the shared `data` volume via +# `docker compose cp`, not onto the box's disk — /data in the container IS that +# volume, so a copy left in the home directory is invisible to `watch`. +set -euo pipefail +cd "$(dirname "$0")" + +INSTANCE=${INSTANCE:-quickbeam-1} +ZONE=${ZONE:-us-east4-a} + +SOURCES=data/sources.json +args=() +for a in "$@"; do + case "$a" in + --env|--dry-run) args+=("$a") ;; + -*) echo "usage: $0 [--env] [--dry-run] [sources.json]" >&2; exit 2 ;; + *) SOURCES=$a ;; + esac +done +DRY=0; [[ " ${args[*]} " == *" --dry-run "* ]] && DRY=1 +run() { if [[ $DRY -eq 1 ]]; then echo " + $*"; else "$@"; fi; } + +[[ -f $SOURCES ]] || { echo "no watch list at $SOURCES" >&2; exit 1; } +# Validate before shipping: nothing downstream treats a bad list as an error. +# _fetch_sources silently skips an entry it cannot parse, and DROPS one that +# names no app — so a typo here comes back as a box that watches nothing and +# says so exactly once, in a log line nobody reads. +python3 - "$SOURCES" <<'PY' +import json, sys +items = json.load(open(sys.argv[1])) +items = items.get("sources", []) if isinstance(items, dict) else items +if not items: + sys.exit("watch list is empty") +bad = [i for i in items + if not ((isinstance(i, str) and len(i.split(":")) == 3) + or (isinstance(i, dict) and i.get("app")))] +if bad: + sys.exit(f"entries must name an app (APP:OWNER:NAMESPACE, `*` = any): {bad}") +print(f"==> {len(items)} source(s): " + ", ".join(map(str, items))) +PY + +# The image half, plus the box's SOURCES_URL — WATCHLIST_URL is how this script says +# "that mode, not yours": deploy.sh writes the line and drops any COMPOSE_FILE overlay, +# so a box that was on the worker converges here in one run instead of silently staying +# on the worker's list. +WATCHLIST_URL=file:///data/sources.json ./deploy.sh "${args[@]}" +run gcloud compute scp "$SOURCES" "$INSTANCE:~/sources.json" --zone="$ZONE" +# Into the volume, since /data in the container IS the volume. No restart: `watch` +# re-reads the file every SOURCES_REFRESH seconds and converges in place — it may log +# one "sources fetch failed" first, in the window between the container starting and +# this copy landing. +run gcloud compute ssh "$INSTANCE" --zone="$ZONE" --command=' + docker compose cp ~/sources.json watch:/data/sources.json && + docker compose exec -T watch cat /data/sources.json' diff --git a/deploy.sh b/deploy.sh index 1c2db3a..f4928cc 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -# Deploy the quickbeam image to the shared GCE instance. -# Run from repo root: ./deploy.sh [--env] [--dry-run] +# Deploy the quickbeam image to the shared GCE instance, watching the REGISTRY +# WORKER's /watchlist — views created from the website are the source of truth. +# For a static list on the box instead, use ./deploy-sources.sh. +# Run from repo root: ./deploy.sh [--env] [--fresh] [--dry-run] # # Wraps the three-command redeploy in DOCKER-README.md ("Redeploying a code # change"): build here, push to Artifact Registry, then tell the box to pull. @@ -13,6 +15,7 @@ INSTANCE=${INSTANCE:-quickbeam-1} ZONE=${ZONE:-us-east4-a} PUSH_ENV=0 +FRESH=0 DRY=0 for a in "$@"; do case "$a" in @@ -20,8 +23,13 @@ for a in "$@"; do # QDRANT_API_KEY, and the box's copy may have been edited in place. # Needed on a first deploy, or after rotating a key / changing a port. --env) PUSH_ENV=1 ;; + # Deploy as if the box had never run: drop the qdrant and data volumes, so it + # comes up with no collection, no ingest checkpoint and no baked CDN shards and + # re-embeds everything from chain. Slow (a full --from-block replay + one + # `fangorn read` and embed per pair), so it is opt-in, not the default. + --fresh) FRESH=1 ;; --dry-run) DRY=1 ;; - *) echo "usage: $0 [--env] [--dry-run]" >&2; exit 2 ;; + *) echo "usage: $0 [--env] [--fresh] [--dry-run]" >&2; exit 2 ;; esac done @@ -39,6 +47,49 @@ IMG=$(grep -E '^IMAGE=' .env | tail -1 | cut -d= -f2- || true) [[ -n "$IMG" ]] || { echo "IMAGE is unset in .env (compose would fall back to quickbeam:local)" >&2; exit 1; } [[ "$IMG" != *"quickbeam:local"* ]] || { echo "IMAGE points at a local build, not Artifact Registry: $IMG" >&2; exit 1; } +# The mode lives in ONE variable: the box's SOURCES_URL. It is rewritten on every +# deploy rather than assumed, because deploy-sources.sh points it at a file and a box +# left in that mode ignores the worker silently — it keeps serving whatever list it +# has and nothing anywhere says why. COMPOSE_FILE goes for the same reason: a +# standalone overlay pinned there re-applies the file mode under a bare +# `docker compose up -d`. +# WATCHLIST_URL, when set, is deploy-sources.sh calling in to reuse the image half of +# this script while owning the mode itself — so the .env guard below is skipped. +if [[ -n "${WATCHLIST_URL:-}" ]]; then + SOURCES_URL=$WATCHLIST_URL +else + SOURCES_URL=$(grep -E '^SOURCES_URL=' .env | tail -1 | cut -d= -f2- || true) + [[ -n "$SOURCES_URL" ]] || { echo "SOURCES_URL is unset in .env (needs the worker's /watchlist URL)" >&2; exit 1; } + case "$SOURCES_URL" in + file://*) echo "SOURCES_URL in .env is a file:// list — that is ./deploy-sources.sh's job, not this script's" >&2; exit 1 ;; + http://*|https://*) ;; + # A bare host is what you get from copying the worker's hostname out of the + # dashboard. urlopen() rejects it with "unknown url type" on every poll, which the + # watcher reports as a fetch failure and then keeps its old list forever. + *) SOURCES_URL="https://$SOURCES_URL" ;; + esac +fi + +# --fresh's remote half. Three separate stores hold ingest state and ALL of them have +# to go together: the qdrant volume (the vectors), /data/db/checkpoint.json (whose +# processed_track_ids + per-source vertex_cids make the watcher skip anything it has +# already embedded) and /data/cdn (whose manifests make append_domain skip anything +# already delivered). Wiping only qdrant leaves the other two claiming the work is +# done, and the watcher then logs "no new records" over an empty collection forever. +# `down -v` drops both named volumes in one go — see DOCKER-README.md's Teardown. +FRESH_CMD="" +if [[ $FRESH -eq 1 ]]; then + # Checked on the BOX, not here, because .env only ships with --env — the local copy + # is not what the watcher reads. Aborts before the wipe, never after. + # ponytail: FROM_BLOCK is the only rebuild path while the watch list is wildcard + # (`*:*`) sources — those get no startup seed read, so their pairs are only + # discovered by replaying StateCommitted history. Relax this to a warning if pinned + # `owner:namespace` entries ever come back, since those DO seed themselves. + FRESH_CMD="grep -qE '^FROM_BLOCK=[1-9]' .env \ + || { echo 'FROM_BLOCK is 0/unset on the box: a wiped instance would never rebuild, because wildcard sources have no seed read. Set it in .env and redeploy with --env.' >&2; exit 1; } \ + && docker compose down -v && " +fi + # Also tag the immutable git sha. `docker compose pull` on a moving :latest # cannot tell you which build a box is running; this leaves a tag you can roll # back to. -dirty when the tree has uncommitted changes, so the tag never lies. @@ -47,14 +98,25 @@ IMG=$(grep -E '^IMAGE=' .env | tail -1 | cut -d= -f2- || true) # --porcelain (rather than `git diff`) because it also catches an UNTRACKED file # under quickbeam/, which `COPY quickbeam ./quickbeam` would bake in regardless. SHA=$(git rev-parse --short HEAD) -[[ -z "$(git status --porcelain -- quickbeam pyproject.toml)" ]] || SHA="$SHA-dirty" +[[ -z "$(git status --porcelain -- quickbeam pyproject.toml Dockerfile)" ]] || SHA="$SHA-dirty" SHA_IMG="${IMG%:*}:$SHA" echo "==> $IMG" echo "==> $SHA_IMG" echo "==> $INSTANCE ($ZONE)" +echo "==> watch list: $SOURCES_URL" [[ $DRY -eq 1 ]] && echo "-- dry run, nothing will be built, pushed or restarted --" +# Asked BEFORE the build so a stray --fresh costs a keystroke, not five minutes and a +# re-embed. Skipped with no tty (CI) and on a dry run, which wipes nothing anyway. +if [[ $FRESH -eq 1 && $DRY -eq 0 ]]; then + echo "==> FRESH: drops the qdrant + data volumes (vectors, checkpoints, shards)" + if [[ -t 0 ]]; then + read -r -p " re-embed everything from chain? [y/N] " ans + [[ $ans == [yY] ]] || { echo "aborted"; exit 1; } + fi +fi + run docker build -t "$IMG" -t "$SHA_IMG" . run docker push "$IMG" run docker push "$SHA_IMG" @@ -66,13 +128,23 @@ files=(docker-compose.yml) run gcloud compute scp "${files[@]}" "$INSTANCE:~/" --zone="$ZONE" # Recreates only the services whose image actually changed; the qdrant and data -# volumes are untouched, so the collection and baked shards survive. +# volumes are untouched, so the collection and baked shards survive — unless --fresh +# put a `down -v` in front, which is exactly what drops them. +# +# The prune is not optional housekeeping. IMAGE is a moving :latest, so every pull +# leaves the previous ~2.8GB image untagged — invisible to plain `docker images`, +# which is why a 20GB box fills up while it still reports 3GB of images. Six of them +# had accumulated (14.8GB) by 2026-08-20. Dangling only: the image the stack now runs +# is tagged, and rollback images live in Artifact Registry, not here. run gcloud compute ssh "$INSTANCE" --zone="$ZONE" \ - --command='docker compose pull && docker compose up -d && docker compose ps' + --command="sed -i -e '/^SOURCES_URL=/d' -e '/^COMPOSE_FILE=/d' .env \ + && echo 'SOURCES_URL=$SOURCES_URL' >> .env \ + && ${FRESH_CMD}docker compose pull && docker compose up -d && docker image prune -f && docker compose ps" if [[ $DRY -eq 0 ]]; then echo echo "deployed $SHA. Roll back with:" echo " ssh the box and run: docker compose pull ${SHA_IMG%:*}:" echo "cdn restarts until watch bakes the first domain — see DOCKER-README.md." + [[ $FRESH -eq 1 ]] && echo "fresh: the collection rebuilds by replaying from FROM_BLOCK — follow it with 'docker compose logs -f watch'." fi diff --git a/docker-compose.yml b/docker-compose.yml index 35fd5c7..de02f84 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,14 @@ services: - --role-map-file=/data/db/role_map.json - --qdrant-url=http://qdrant:6333 - --qdrant-api-key=${QDRANT_API_KEY} + # Replay history before going live, which is the ONLY way a wildcard source + # (`*:*`) discovers namespaces published before this box existed — it has no + # single namespace to seed with `fangorn read`. Overrides the CLI's saved cursor, + # so it re-runs on every restart. 0 = off (the default), NOT genesis. + # Cost: the catch-up is 1000 blocks per eth_getLogs, so a million blocks back is a + # thousand sequential calls against the public RPC. Pick a block just before the + # commits you want, not a round number far in the past. + - --from-block=${FROM_BLOCK:-0} # Reconnect backoff for a dropped subscribe stream — NOT a monitoring interval. # `watch` is push-based off `fangorn subscribe` and reacts as commits land. - --poll-interval=${INTERVAL:-60} diff --git a/quickbeam/ingest/sources/fangorn.py b/quickbeam/ingest/sources/fangorn.py index 825bc7d..21da415 100644 --- a/quickbeam/ingest/sources/fangorn.py +++ b/quickbeam/ingest/sources/fangorn.py @@ -103,13 +103,21 @@ def subscribe_cmd(fangorn_bin: str, owner: str | None, namespace: str | None, `from_block` (or `from_start`, its genesis form) replays history before going live — how an app-level watch discovers namespaces published before it started, since a - wildcard source can't be seeded with `read` (which needs one exact namespace). Prefer - `from_block`: the catch-up is windowed, and genesis on a fast chain like Arbitrum is - hundreds of thousands of windows.""" + wildcard source can't be seeded with `read` (which needs one exact namespace). It + OVERRIDES the CLI's saved cursor (cli.js reads the flag before the cursor file), so + it replays on every reconnect, not only the first. + + Prefer `from_block` to `from_start`: the catch-up is windowed at 1000 blocks per + eth_getLogs (FANGORN_LOG_WINDOW in the SDK), so each million blocks is a thousand + sequential RPC calls and genesis on Arbitrum is hundreds of thousands of them. + + 0 means "no replay", not "from genesis" — that is `from_start`. The falsy check is + what lets docker-compose.yml pass `--from-block=${FROM_BLOCK:-0}` unconditionally; + compose cannot omit an argument, and an empty one dies in argparse.""" argv = [*shlex.split(fangorn_bin), "subscribe"] if owner is None or namespace is None: argv.append("--all") - if from_block is not None: + if from_block: argv += ["--from-block", str(from_block)] elif from_start: argv.append("--from-start") diff --git a/quickbeam/watcher.py b/quickbeam/watcher.py index fdfb54b..b34e62c 100644 --- a/quickbeam/watcher.py +++ b/quickbeam/watcher.py @@ -468,7 +468,8 @@ async def _stream_source_once(args, qdrant, embed_engine, role_map_ref, dim, tru # removals of already-absent cids are no-ops). if (ch_owner, ch_ns) not in snapshot["seeded"] and (ch_owner, ch_ns) not in tried: tried.add((ch_owner, ch_ns)) - await _seed_pair(args, qdrant, embed_engine, role_map_ref, dim, truncate, + await _seed_pair(_pair_cdn_args(args, qdrant, ch_owner, ch_ns), + qdrant, embed_engine, role_map_ref, dim, truncate, checkpoint, ch_owner, ch_ns, snapshot) state = _ns_state(snapshot, (ch_owner, ch_ns)) @@ -507,7 +508,8 @@ async def _stream_source_once(args, qdrant, embed_engine, role_map_ref, dim, tru ) status = f"{n} new record(s) embedded" if n else "no new records for the active profiles" print(f"[Watcher] {ch_key}: change applied — {status}") - _deliver_cdn(args, qdrant, n, change_edges, change_tombstones, + _deliver_cdn(_pair_cdn_args(args, qdrant, ch_owner, ch_ns), qdrant, n, + change_edges, change_tombstones, owner=ch_owner, namespace=ch_ns, app=args.app) rc = await proc.wait() @@ -609,10 +611,9 @@ def _source_args(args, app: str, owner: str | None, namespace: str | None, scoped = copy.copy(args) scoped.app = app if not pin_domain: - # ponytail: a wildcard source gets no live CDN delivery. Its pairs are only - # learned as commits arrive, so there is no single domain to bake at task - # start — and _deliver_cdn ships to one fixed domain per task. Give the - # delivery path a per-change domain if wildcard sources ever need shards. + # A wildcard source has no domain at task start — its pairs are only learned + # as commits arrive — so it is left None here and derived per change by + # _pair_cdn_args, which also bakes a pair's domain the first time it is seen. concrete = owner is not None and namespace is not None scoped.cdn_domain = (_domain_for(app, owner, namespace) if args.cdn_dir and concrete else None) @@ -621,6 +622,29 @@ def _source_args(args, app: str, owner: str | None, namespace: str | None, return scoped +def _pair_cdn_args(args, qdrant, owner: str, namespace: str): + """Per-pair delivery view of a WILDCARD task's args, or `args` unchanged otherwise. + + A pinned source fixes its domain once, at task start (_source_args + _bake_initial). + A wildcard source cannot: its pairs exist only once a commit names them, so the + domain is derived — and baked on first sight — here, per change. Everything else + (collection, checkpoint, role map) is deliberately shared with the task, exactly as + _source_args leaves it; only the domain varies per pair, or their shards intermix. + + The bake is not optional: append_domain EXTENDS a manifest, so with no base one + there is nothing to append to and every delivery fails. + """ + if not args.cdn_dir or args.cdn_domain: + return args # delivery is off, or the task is pinned and already has a domain + scoped = copy.copy(args) + scoped.cdn_domain = _domain_for(args.app, owner, namespace) + # Checked here rather than leaning on _bake_initial's own guard, which prints a + # line every time — that is once per change on a busy wildcard. + if not os.path.exists(os.path.join(args.cdn_dir, scoped.cdn_domain, "manifest.json")): + _bake_initial(scoped, qdrant, owner, namespace) + return scoped + + def _bake_initial(args, qdrant, owner: str, namespace: str) -> None: """Bootstrap live CDN delivery for one source. append_domain can only EXTEND an already-baked domain, so bake once here to give it a base manifest; `cdn serve` diff --git a/tests/test_sources.py b/tests/test_sources.py index 5ea694f..645c448 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -26,6 +26,9 @@ def test_subscribe_cmd_app_mode(): ["fangorn", "subscribe", "--all", "--from-start"] assert subscribe_cmd("fangorn", None, None, from_start=True, from_block=42) == \ ["fangorn", "subscribe", "--all", "--from-block", "42"] + # 0 is "no replay", not genesis — docker-compose.yml passes --from-block=${FROM_BLOCK:-0} + # unconditionally because compose cannot omit an argument. + assert subscribe_cmd("fangorn", None, None, from_block=0) == ["fangorn", "subscribe", "--all"] # --fangorn-bin may be a full command, shell-split. assert subscribe_cmd("node cli.js", None, None)[:2] == ["node", "cli.js"] diff --git a/tests/test_wildcard_cdn_domain.py b/tests/test_wildcard_cdn_domain.py new file mode 100644 index 0000000..3786a74 --- /dev/null +++ b/tests/test_wildcard_cdn_domain.py @@ -0,0 +1,70 @@ +"""A wildcard source's CDN domain, which is chosen per change instead of per task. + +A pinned source names its domain once, at task start. A wildcard one (`*:*`, `0x..:*`, +`*:ns`) learns its pairs only as commits arrive, so `_pair_cdn_args` derives the domain +from the change itself — and bakes it the first time a pair is seen, because +`append_domain` extends a manifest and has nothing to extend without one. + +The names here are the contract with `domainFor()` in the registry worker: byte drift +between the two returns an empty catalog with no error on either side. +""" +import argparse +import os + +from quickbeam import watcher + +OWNER = "0x7a7849231cF7Ab1EA003BcF0063CB89704D7Cce9" +OTHER = "0x8ce65916C8b83b4c62dAd51b462643A1ae59899b" + + +def _args(cdn_dir, cdn_domain=None, app="sond3r.test.0"): + return argparse.Namespace(cdn_dir=cdn_dir, cdn_domain=cdn_domain, app=app) + + +def _baked(monkeypatch) -> list: + """Record bakes instead of running one — a real bake needs Qdrant.""" + calls = [] + monkeypatch.setattr(watcher, "_bake_initial", + lambda args, qdrant, owner, ns: calls.append((args.cdn_domain, owner, ns))) + return calls + + +def test_wildcard_pair_gets_its_own_domain(tmp_path, monkeypatch): + calls = _baked(monkeypatch) + task = _args(str(tmp_path)) + + a = watcher._pair_cdn_args(task, None, OWNER, "media") + b = watcher._pair_cdn_args(task, None, OTHER, "media") + + assert a.cdn_domain == "sond3r-test-0-7a784923-media" + assert a.cdn_domain == watcher._domain_for(task.app, OWNER, "media") + # Same subspace name, different publisher: two domains, or their shards intermix. + assert b.cdn_domain == "sond3r-test-0-8ce65916-media" + # The task's own args must stay wildcard-shaped, or the first pair seen would + # capture every later pair's shards. + assert task.cdn_domain is None + assert [c[0] for c in calls] == [a.cdn_domain, b.cdn_domain] + + +def test_bake_only_when_the_domain_has_no_manifest(tmp_path, monkeypatch): + calls = _baked(monkeypatch) + task = _args(str(tmp_path)) + domain = watcher._domain_for(task.app, OWNER, "media") + os.makedirs(tmp_path / domain) + (tmp_path / domain / "manifest.json").write_text("{}") + + scoped = watcher._pair_cdn_args(task, None, OWNER, "media") + + assert scoped.cdn_domain == domain + assert calls == [] # already delivered by an earlier connection + + +def test_pinned_and_delivery_off_are_untouched(tmp_path, monkeypatch): + calls = _baked(monkeypatch) + pinned = _args(str(tmp_path), cdn_domain="pinned-domain") + off = _args(None) + + # Same object back, not a copy: a pinned task already baked its one domain. + assert watcher._pair_cdn_args(pinned, None, OWNER, "media") is pinned + assert watcher._pair_cdn_args(off, None, OWNER, "media") is off + assert calls == [] From 86f15b56ed8190fa80fcee972d544b859179de36 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Tue, 25 Aug 2026 10:09:29 -0500 Subject: [PATCH 2/8] Added github actions for quickbeam --- .github/workflows/tests.yml | 20 ++++++++++++++++++++ quickbeam.egg-info/SOURCES.txt | 4 ++++ tests/test_cdn_events.py | 8 ++++---- 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..642a4ff --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,20 @@ +name: tests + +on: + push: + pull_request: + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - run: pip install -e ".[cpu,dev]" + # ponytail: tests/ only — quickbeam/test_roles.py is a sys.exit self-check + # that kills collection if pytest walks the package dir. + - run: pytest tests -q diff --git a/quickbeam.egg-info/SOURCES.txt b/quickbeam.egg-info/SOURCES.txt index 2f63d77..4c69117 100644 --- a/quickbeam.egg-info/SOURCES.txt +++ b/quickbeam.egg-info/SOURCES.txt @@ -53,11 +53,15 @@ quickbeam/pipelines/prebake.py tests/test_adjacency_route.py tests/test_bucket_route.py tests/test_cdn_edges.py +tests/test_cdn_events.py tests/test_index.py +tests/test_ingest_checkpoint.py tests/test_mcp.py tests/test_roles.py tests/test_schemagen_identity.py +tests/test_scope.py tests/test_shell.py tests/test_sources.py tests/test_watchlist.py +tests/test_wildcard_cdn_domain.py tests/test_x402_agent.py \ No newline at end of file diff --git a/tests/test_cdn_events.py b/tests/test_cdn_events.py index 3686030..6031213 100644 --- a/tests/test_cdn_events.py +++ b/tests/test_cdn_events.py @@ -55,8 +55,8 @@ def _events(body, timeout=20.0): yield {"event": event, "data": json.loads(data)} -def test_events_stream(tmp_dir): - cdn_dir = tmp_dir +def test_events_stream(tmp_path): + cdn_dir = str(tmp_path) # One domain already baked before the client connects, one not yet present. _write_manifest(cdn_dir, "app8-owner8-media", ["shard-aaa.ndjson.gz"]) @@ -108,9 +108,9 @@ def test_events_stream(tmp_dir): server.should_exit = True -def test_traversal_is_rejected(tmp_dir): +def test_traversal_is_rejected(tmp_path): port = _free_port() - server = uvicorn.Server(uvicorn.Config(build_app(tmp_dir), host="127.0.0.1", + server = uvicorn.Server(uvicorn.Config(build_app(str(tmp_path)), host="127.0.0.1", port=port, log_level="error")) threading.Thread(target=server.run, daemon=True).start() for _ in range(100): From 97d4e5389c84543f439f11f65bce3098a7ef9034 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Tue, 25 Aug 2026 10:17:45 -0500 Subject: [PATCH 3/8] Fix missing path issue --- tests/test_watchlist.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_watchlist.py b/tests/test_watchlist.py index a125dc1..32dfe93 100644 --- a/tests/test_watchlist.py +++ b/tests/test_watchlist.py @@ -14,6 +14,8 @@ from pathlib import Path from unittest.mock import patch +import pytest + # Everything under test here is pure string/parsing logic, but importing the watcher # drags in the whole embedding stack. Stub the leaves so this file runs on a plain # checkout (`pip install -e .` with no [cpu] extra) instead of only inside the image. @@ -34,6 +36,10 @@ OWNER = "0x7a7849231cF7Ab1EA003BcF0063CB89704D7Cce9" WORKER = Path(__file__).resolve().parents[2] / "webworker/quickbeam-registry/src/index.js" +# The worker lives in the fangorn monorepo one level up, not in this repo — so a +# standalone quickbeam checkout (CI) has no copy to compare against. +needs_worker = pytest.mark.skipif(not WORKER.exists(), + reason="registry worker source not checked out alongside quickbeam") def _watchlist(payload, default_app=None): @@ -112,6 +118,7 @@ def test_app_reaches_the_qdrant_payload(): APP_ID = "0x" + "a3f19b2c" + "0" * 56 # shape of toAppId(): 0x + 64 hex +@needs_worker def test_domain_matches_the_worker(): """_domain_for(app, owner, namespace) == the worker's domainFor(app, owner, ns).""" src = WORKER.read_text() @@ -135,6 +142,7 @@ def test_domain_matches_the_worker(): assert _domain_for("sond3r.test.1", OWNER, "media") == "sond3r-test-1-7a784923-media" +@needs_worker def test_worker_canonicalises_app_to_an_id(): """The site can only ever send an app ID — `StateCommitted` carries the id and has no on-chain preimage, so an app the site has no name for still has to be expressible. From 5ad527e33651c8adee1dc2cae6014234d17fa035 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Tue, 25 Aug 2026 17:19:59 -0500 Subject: [PATCH 4/8] Update both the quickbeam readme and docker readme --- DOCKER-README.md | 302 ++++---------- README.md | 994 +++++++++-------------------------------------- 2 files changed, 271 insertions(+), 1025 deletions(-) diff --git a/DOCKER-README.md b/DOCKER-README.md index 978edab..3a92786 100644 --- a/DOCKER-README.md +++ b/DOCKER-README.md @@ -1,13 +1,9 @@ # Deploying Quickbeam **One shared instance serves everyone, and each namespace is embedded exactly once.** -You stand it up once. After that, a user creates a *view* from the Fangorn website — -no SSH, no compose edit, no new container. +The container only needs to be set up once. After that a user creates a *view* from the Fangorn website. -A **view** is a named set of namespaces belonging to one requester. It gets its own -search URL and its own MCP catalog, but it is a *filter* over the shared collection, -never a copy of the vectors. Two people asking for the same namespace get the same -points, and the second one costs no indexing work at all. +A **view** is a named set of namespaces belonging to one requester. It gets its own search URL and its own MCP catalog (based on a flag on the Fangorn website), but it is a *filter* over the shared collection and not a copy of the vectors. Two people asking for the same namespace get the same points, but the second one costs no indexing work at all. --- @@ -50,34 +46,15 @@ flowchart LR style chain fill:#fff,stroke:#ccc ``` -**Embed once, view many.** The watchlist is the deduplicated union of every view's -sources, so a namespace is watched while at least one view references it and drops off -when the last one goes — no refcount needed. - -**The per-view MCP is a filtered catalog.** `quickbeam mcp` is a pull-client whose -entire universe is whatever `/catalog` its `--cdn-url` returns, so the worker filtering -that catalog to a view's domains is what scopes it. By default the user runs the client -themselves; ticking "host an MCP for me" makes the worker create one **Cloud Run** -service per view instead — stateless, scale-to-zero, and nothing to do with this box. - -**The worker proxies queries** because the instance speaks plain HTTP and a browser on -HTTPS cannot call it (mixed content). Proxying supplies TLS with no per-namespace DNS -record or certificate, and it injects the view's `scope` so a view URL always means -that view's namespaces. - -**A source is the whole `app:publisher:subspace` triple.** The app is the first leg and -it is what every chain read resolves against, so each watched source carries its own — -one instance can serve views across several apps. `--app` is only the fallback for a -watch-list entry that names none; an entry with no app and no fallback is dropped rather -than guessed, because reading the wrong app silently indexes the wrong graph. - -**Naming rule, shared by two codebases:** a source's CDN domain is -`{app[2:10]}-{owner[2:10]}-{namespace}` — `_domain_for()` in `quickbeam/watcher.py` -names the directory, `domainFor()` in the worker names it back to filter a catalog. All -three legs are needed: two publishers both calling a namespace `music` would intermix -their shards in one domain, and so would one publisher holding `music` in two apps. A -bare app *name* (only reachable from a hand-run static watcher) is slugged whole -instead of sliced. +**Embed once, view many.** The watchlist is the deduplicated union of every view's sources, so a namespace is watched while at least one view references it and drops off when the last one goes. + +**The per-view MCP is a filtered catalog.** `quickbeam mcp` is a pull-client whose entire universe is whatever `/catalog` its `--cdn-url` returns, so the worker filtering that catalog to a view's domains is what scopes it. By default the user runs the client themselves. Ticking "host an MCP for me" makes the worker create one **Cloud Run** service per view instead. + +**The Cloudflare worker proxies queries** because, currently, the instance uses plain HTTP and a browser on HTTPS cannot call it (mixed content). Proxying supplies TLS with no per-namespace DNS record or certificate, and it injects the view's `scope` so a view URL always means that view's namespaces. + +**A source is the whole `app:publisher:subspace` triple.** The app is the first identifier and it is what every chain read resolves against, so each watched source carries its own. One instance can serve views across several apps. `APP` is only the fallback for a watchlist entry that names none. An entry with no app and no fallback is dropped. + +**Naming rule:** a source's CDN domain is `{app}-{owner[2:10]}-{namespace}`, where a 0x app *id* is sliced to its first 8 hex chars and a plain app *name* is slugged whole. All three identifiers are needed: two publishers both calling a namespace `music` would intermix their shards in one domain, and so would one publisher holding `music` in two apps. ### What happens when a commit lands @@ -96,43 +73,31 @@ sequenceDiagram W->>C: append delta shard to that namespace's domain ``` -`quickbeam watch` is **push-based**. `--poll-interval` is the reconnect backoff for a -dropped stream, *not* a polling timer. `--sources-refresh` is the separate interval at -which it re-reads the watch list. +`quickbeam watch` is **push-based**. `--poll-interval` is the reconnect backoff for a dropped stream, *not* a polling timer. `--sources-refresh` is the separate interval at which it re-reads the watch list. --- ## 1. Run it locally -Prove the image before any cloud is involved. Local runs don't need the worker — point -`SOURCES_URL` at a file. +Local runs don't need the worker. Instead, you can point `SOURCES_URL` at a `json` file. ```sh -cd quickbeam cp .env.example .env # set ETH_PRIVATE_KEY, PINATA_GATEWAY, QDRANT_API_KEY, APP docker compose up -d --build ``` -`--build` is not optional the first time, and not optional after a source change either: -the Dockerfile `COPY`s `quickbeam/` at build time, so a plain `docker compose up -d` -silently reuses the old layer and reproduces bugs you already fixed. To rebuild without -starting anything, `docker compose build`. +`--build` is not optional the first time, and not optional after a source change either. The Dockerfile `COPY`s `quickbeam/` at build time, so a plain `docker compose up -d` reuses the old layer. -For a static list, write `sources.json` into the shared volume and set -`SOURCES_URL=file:///data/sources.json` (`./deploy-sources.sh` does both halves on -the box — it is `deploy.sh` plus a `docker compose cp` of the file into the volume): +For a static list, write `sources.json` into the shared volume and set `SOURCES_URL=file:///data/sources.json`. An example `sources.json`: ```json [ {"app": "fangorn", "owner": "0x147c24c5Ea2f1EE1ac42AD16820De23bBba45Ef6", "namespace": "robinhood"} ] ``` -The list also accepts `"APP:OWNER:NAMESPACE"` strings, the older `"OWNER:NAMESPACE"` -form (which takes `APP` as its app), and either form wrapped in `{"sources": …}` — -which is the shape the worker's `/watchlist` returns. `*` on any of owner/namespace is -a wildcard that widens the subscription to the app level. +The list also accepts `"APP:OWNER:NAMESPACE"` strings, the older `"OWNER:NAMESPACE"` form (which takes the `APP` env var as its app), and either form wrapped in `{"sources": …}` (the shape the worker's `/watchlist` returns). `*` on owner or namespace widens the subscription to the app level. -Check it: +To check your container run: ```sh docker compose logs -f watch @@ -140,83 +105,56 @@ docker compose logs -f watch # [Watcher] 0x147c…:robinhood: subscribed (pid …) # [Watcher] 0x147c…:robinhood: seeded — N new record(s) embedded -curl 'localhost:8080/search?q=&scope=0x147c24c5…:robinhood' -curl localhost:8090/domains/7e1497af-147c24c5-robinhood/manifest # {app8}-{owner8}-{ns} +curl 'localhost:8080/search?q=&scope=fangorn:0x147c24c5…:robinhood' +curl localhost:8090/domains/7e1497af-147c24c5-robinhood/manifest ``` -**Add a second entry to the file and watch a second task start with no restart.** That -is the whole point of the design — if it needs a restart, something regressed. +A non-zero `N` in the seed line indicates the Node CLI, the chain read, the IPFS fetch, the projection and the embed all worked. -A non-zero `N` in the seed line is the real proof: the Node CLI, the chain read, the -IPFS fetch, the projection and the embed all worked. +You can also add a second entry to the file and watch a second task start with no restart required! > **`seeded — no new records` / `head: null`?** That namespace has nothing settled -> on-chain. Confirm with -> `docker compose exec watch fangorn read --owner ` — if it returns -> `"head":null` with empty arrays, nothing was ever pushed there. Not a deployment -> fault. +> on-chain. Confirm with `docker compose exec watch fangorn read --owner `. +> if it returns `"head":null` with empty arrays, nothing was ever pushed there. ### Configuration +You can see the fully annotated list in [`.env.example`](.env.example), but here are the most imporant ones: + | Variable | Notes | |---|---| -| `SOURCES_URL` | The registry worker's `/watchlist` — the deduplicated union of every view's sources. Namespaces arrive there, never here. | -| `SOURCES_REFRESH` | Seconds between watch-list polls. | -| `APP` | Fallback app for a watch-list entry that names none. Entries from the worker always carry their own, so this only covers a hand-written list — but keep it equal to the worker's `DEFAULT_APP`, since that is what the worker stamps on a view created without one. A wrong value reads an empty namespace with **no error**. | -| `ETH_PRIVATE_KEY` | **Required even though the container only reads** — the `fangorn` CLI refuses to start without one. A throwaway is correct: never spent, no funding, no registration. | -| `PINATA_GATEWAY` | Reads resolve every block by CID through this. The default is `ipfs.io`, DNS-filtered on many networks. | -| `QDRANT_API_KEY` | Any random string; Qdrant enforces it on every request. | -| `COLLECTION` | One collection for all namespaces. | -| `INTERVAL` | Reconnect backoff. Not a monitoring interval. | - -Do **not** bake a `~/.fangorn/config.json` into the image. The CLI returns early when -that file exists and ignores every environment variable above. - ---- +| `IMAGE` | The Docker tag (Artifact Registry) every service runs. Unset = build locally. Must be set to a pushed tag on the box. | +| `SOURCES_URL` | The worker's `/watchlist`, or a `file://` path. | +| `APP` | Fallback app for an entry naming none. When deployed, it should be the same as the worker's `DEFAULT_APP` (`fangorn`). | +| `ETH_PRIVATE_KEY` | **Required even though the container only reads** — the `fangorn` CLI refuses to start without one. Provide a throwaway since no onchain operations are performed. | +| `PINATA_GATEWAY` | Reads resolve every block by CID through this. We recommend you set this since the `ipfs.io` may not serve this content (ISP specific). | +| `FROM_BLOCK` | History replay before going live and is the **only** way a wildcard (`*:*`) source discovers namespaces published before the box existed. It costs one `eth_getLogs` per 1000 blocks. | +| `QDRANT_API_KEY` | Any random string. Qdrant enforces it on every request. | -## 2. Build the image +> Warning: Do **not** bake a `~/.fangorn/config.json` into the image. The CLI returns early when that file exists and ignores every environment variable above. -All four Quickbeam services are the **same image** with different commands, so it builds -once and is tagged `${IMAGE}` (default `quickbeam:local`). The build installs the CPU -ONNX stack and the Node CLI and bakes the embedding model in (`Dockerfile` — left to run -time it re-downloads on every container start), so expect several minutes and a ~2.5GB -image (measured 2026-08-13). +--- -```sh -docker compose build # tags quickbeam:local -``` +## 2. One-time cloud setup -**Build here, not on the box.** E2 shared-core types are burstable and small; installing -the ONNX stack on one is slow at best and OOMs at worst, and it burns the same burst -credits the watcher needs to seed. Push a built image instead — which is also why the box -never needs the source. +### Authorize +`gcloud auth login` -### Push it to Artifact Registry +### Artifact Registry ```sh PROJECT=$(gcloud config get-value project) REGION=us-east4 -IMG=$REGION-docker.pkg.dev/$PROJECT/quickbeam/quickbeam:latest - -# One-time, per project. -gcloud artifacts repositories create quickbeam \ - --repository-format=docker --location=$REGION +gcloud artifacts repositories create quickbeam --repository-format=docker --location=$REGION gcloud auth configure-docker $REGION-docker.pkg.dev -docker build -t $IMG . && docker push $IMG +# then in .env: +# IMAGE=us-east4-docker.pkg.dev//quickbeam/quickbeam:latest ``` -`IMAGE` in `.env` is what points the compose stack at that tag; it defaults to a local -build, so leaving it unset keeps step 1 working unchanged. - -`:latest` keeps the deploy a one-liner, at the cost of not being able to tell which build -a box is running — `docker compose pull` will not say whether the tag moved. If that -matters, tag `:$(git rev-parse --short HEAD)` as well and put that in the box's `.env`; -rollback is then the previous tag instead of a rebuild. - ---- +**For small compute instances and large datasets, it is recommended to build locally, not on the box.** E2 shared-core types are burstable and small. Installing the ONNX stack on one is slow at best and OOMs at worst, and it burns the same burst credits the watcher needs to seed. The image is ~2.8GB and takes several minutes: it carries the CPU ONNX stack, Node plus the pinned `@fangorn-network/sdk`, and the embedding model baked in (left to run time it re-downloads on every container start). -## 3. The instance +### The instance ```sh gcloud compute instances create quickbeam-1 \ @@ -225,40 +163,20 @@ gcloud compute instances create quickbeam-1 \ --zone=$REGION-a \ --image-family=debian-12 --image-project=debian-cloud \ --tags=quickbeam --scopes=cloud-platform -``` - -`--scopes=cloud-platform` is what lets the VM's default service account pull from -Artifact Registry; without it the pull 403s and nothing else on the box explains why. -Install Docker once: - -```sh gcloud compute ssh quickbeam-1 --zone=$REGION-a --command=' sudo apt-get update && sudo apt-get install -y docker.io docker-compose-v2 && - sudo usermod -aG docker $USER && + sudo usermod -aG docker $USER && sudo systemctl enable docker && gcloud auth configure-docker '$REGION'-docker.pkg.dev --quiet' ``` -### Ship it +`--scopes=cloud-platform` is what lets the VM's default service account pull from Artifact Registry. without it the pull 403s. `systemctl enable docker` is all that's needed to survive a reboot. Every service is `restart: unless-stopped`, so no unit file and no cron. -**The box never gets the source — only two files.** The image is already built, so a -clone there would be dead weight that also invites an accidental `--build` on a machine -that cannot afford one: - -```sh -gcloud compute scp docker-compose.yml .env quickbeam-1:~/ --zone=$REGION-a -gcloud compute ssh quickbeam-1 --zone=$REGION-a --command='docker compose pull && docker compose up -d' -``` +**Machine type.** `e2-medium` (4GB) is the working default. `e2-small` (2GB) runs with little headroom. -The `.env` you copy **must** contain `IMAGE=$IMG`. Without it the compose file falls back -to `quickbeam:local`, which does not exist on the box, and `docker compose up` tries to -build from a directory holding no Dockerfile. That is the one failure mode of this flow, -and its error message points at the build, not at the missing variable. +**Do big backfills elsewhere.** A full seed embed is a sustained burn that exhausts burst credits. Build the collection on a GPU box, migrate or import it, and let the instance handle deltas only. -`docker compose ps` should then show `qdrant`, `watch`, `cdn`, `serve` and `mcp` up, with -`cdn` restarting until the watcher bakes the first domain (see the gotchas). - -**Open the two ports to the worker:** +### Firewall ```sh gcloud compute firewall-rules create quickbeam-http \ @@ -266,124 +184,72 @@ gcloud compute firewall-rules create quickbeam-http \ --description="registry worker → search + cdn" ``` -Cloudflare Workers egress from the public internet with no fixed range, so -`--source-ranges` cannot be narrowed to them; the instance is reachable by anyone who -finds the IP. Both ports are read-only query surfaces, and the money actions live behind -the worker's signature checks. Two ports stay closed on purpose: Qdrant's `6333` (bound -to loopback in `docker-compose.yml`, and holding write routes), and the `mcp` service's -`8765`, which serves the **whole** corpus unscoped — per-view MCP is the user's own -client against `/q/{viewId}/cdn`. - -Then point the worker at the box and deploy it: +### Point the worker at the box ```sh -gcloud compute instances describe quickbeam-1 --zone=us-east4-a \ +gcloud compute instances describe quickbeam-1 --zone=$REGION-a \ --format='get(networkInterfaces[0].accessConfigs[0].natIP)' -# → set SEARCH_URL = http://:8080 and CDN_URL = http://:8090 -# in webworker/quickbeam-registry/wrangler.toml -cd ../webworker/quickbeam-registry && wrangler deploy ``` -There is no `MCP_URL`: a user's MCP is their own `quickbeam mcp` client pointed at -`/q/{viewId}/cdn`. +Set `SEARCH_URL` and `CDN_URL` in `webworker/quickbeam-registry/wrangler.toml` to `http://:8080` / `:8090`, or to a grey-cloud A record pointing at that IP, which is what the live deploy uses (`http://qb.sond3r.com:8080`) so the IP can change without a worker deploy. Then `wrangler deploy`. + +There is no `MCP_URL`: a user's MCP is their own `quickbeam mcp` client pointed at `/q/{viewId}/cdn`. -**Survive a reboot.** Every service is `restart: unless-stopped`, so Docker brings the -stack back as long as the daemon starts: `sudo systemctl enable docker`. Nothing else is -needed — no unit file, no cron. +--- + +## 3. Deploying -**Redeploying a code change** is the same three commands, and only `docker-compose.yml` -needs re-copying if it changed: +Both scripts run from the repo root and wrap the build → push → pull cycle. The box only gets `docker-compose.yml` and `.env`. ```sh -docker build -t $IMG . && docker push $IMG -gcloud compute scp docker-compose.yml quickbeam-1:~/ --zone=$REGION-a # if it changed -gcloud compute ssh quickbeam-1 --zone=$REGION-a --command='docker compose pull && docker compose up -d' +./deploy.sh # build, push, tell the box to pull +./deploy.sh --env # also copy .env — first deploy, or after rotating a key +./deploy.sh --dry-run # print every command, run none +./deploy.sh --fresh # wipe all state and re-embed from chain (prompts) +./deploy-sources.sh [file] # deploy with a STATIC watch list (default: data/sources.json) ``` -Compose recreates only the services whose image actually changed, and the `qdrant` and -`data` volumes are untouched, so the collection and the baked shards survive. +`INSTANCE` and `ZONE` are environment overrides (default `quickbeam-1` / `us-east4-a`). -`./deploy.sh --fresh` does the opposite deliberately: it prefixes the remote step with -`docker compose down -v`, so the box comes back with no vectors, no ingest checkpoint -and no shards and re-embeds everything. All three have to go together — the checkpoint -alone would make the watcher skip every record it has already seen and report "no new -records" over an empty collection. It refuses to wipe unless the box's `FROM_BLOCK` is -set, because a wildcard (`*:*`) watch list has no seed read and history replay is the -only way anything comes back. +What `deploy.sh` does: -**Machine type.** `e2-medium` (4GB) is the working default above. `e2-small` (2GB) runs -but leaves little headroom — if a seed OOMs there, stop the instance, change the type and -start it again; the disk survives, so it costs a minute. `e2-micro` (1GB, free tier) is -too small: the ONNX model plus one Node subscribe process per source will not fit. +- **Refuses to deploy with `IMAGE` unset or local.** Compose would fall back to `quickbeam:local`, which doesn't exist on the box, and then tries to build from a directory with no Dockerfile. +- **Rewrites `SOURCES_URL` on the box every run.** That variable is the box's mode, and a box left on a `file://` list ignores the worker. `deploy-sources.sh` calls back into `deploy.sh` with `WATCHLIST_URL` set to claim the other direction, so a box converges either way in one run. +- **Tags the git sha alongside `:latest`** (`-dirty` if `quickbeam/`, `pyproject.toml` or the `Dockerfile` has uncommitted or untracked changes, so the tag never lies). Roll back by pulling an older sha tag, `docker compose pull`, on a moving `:latest` cannot tell you what a box is running. +- **Prunes dangling images** after the pull. Every pull of a moving `:latest` leaves the previous ~2.8GB image untagged and invisible to plain `docker images`. +- **`--fresh` aborts unless the box's `FROM_BLOCK` is set.** A wildcard watch list gets no startup seed read, so history replay is the only way anything is discovered. Wiping without it leaves an empty collection forever. -**Do big backfills elsewhere.** E2 shared-core types are burstable (`e2-small` -baselines at 0.5 vCPU) and a full seed embed is a sustained burn that exhausts burst -credits. Build the collection on a GPU box and restore a snapshot here (see the -Snapshots section in `README.md`); the instance then only handles deltas. +`--fresh` drops **three** stores together via `docker compose down -v`: the qdrant volume (the vectors), `/data/db/checkpoint.json` (whose processed ids make the watcher skip what it already embedded) and `/data/cdn` (whose manifests make the append skip what it already delivered). -**What scales per namespace** is one `fangorn subscribe` Node process, not the model. -Measure its RSS before promising a namespace count. +`deploy-sources.sh` validates the JSON before shipping it. `_fetch_sources` skips an entry it cannot parse and *drops* one that names no app, so a typo otherwise comes back as a box that watches nothing. The file goes into the shared volume with `docker compose cp` (`/data` in the container *is* the volume, so a copy in the home directory is invisible to `watch`), and no restart is needed since `watch` re-reads it every `SOURCES_REFRESH` seconds. --- ## 4. Adding and removing views -**Adding** is a user action: sign in at fangorn.network with an active storage -subscription, name a view and give it a publisher + namespace, and press Create. The -worker stores the view and its sources join the watchlist; this instance picks up any -*new* namespace within `SOURCES_REFRESH` and logs `[Watcher] + owner:namespace`. - -A namespace another view already covers logs nothing at all — it is already embedded, -and the new view queries the same points. That silence is the design working. +**Adding** is a user action. Users sign in at fangorn.network with an active storage subscription to create views. The worker stores the view and its sources join the watchlist. The instance picks up any *new* namespace within `SOURCES_REFRESH` and logs `[Watcher] + owner:namespace`. -**Removing** is `POST /views/remove` with the view id, signed by the wallet that created -it (the website's "Stop watching" button), or `POST /admin/remove` for any view, signed -by a wallet in `ADMIN_WALLETS`. See `webworker/quickbeam-registry/README.md`. A source stops -being watched only when the **last** view referencing it goes; nothing expires on its -own, so a lapsed subscription keeps running until someone removes it. +A namespace another view already covers logs nothing at all since it is already embedded and the new view queries the same points. -Neither touches this box. +**Removing** is `POST /views/remove` with the view id, signed by the wallet that created it (the website's "Stop watching" button), or `POST /admin/remove` for any view, signed by a wallet in `ADMIN_WALLETS`. See `webworker/quickbeam-registry/README.md`. A source stops being watched only when the **last** view referencing it is removed. Nothing expires on its own, so a lapsed subscription keeps running until someone removes it. --- ## Gotchas -- **The compose `mcp` service serves the whole corpus,** not a view — it is bound to - one `--cdn-url` at startup. Per-view MCP is the user's own client pointed at - `/q/{viewId}/cdn`. Its endpoint is `/mcp`, not `/`: a healthy server returns 404 on - `/` and 400 on `/mcp` for a request without a handshake. -- **The CDN domain rule is duplicated in two languages.** `_domain_for()` in - `watcher.py` and `domainFor()` in the registry worker must agree, or a view's catalog - comes back empty. If you change one, change the other — `tests/test_watchlist.py` - re-derives the worker's version from its actual source and fails if they drift. -- **The watch-list fetch sends an explicit `User-Agent`.** The worker sits behind - Cloudflare, whose bot-signature check answers urllib's default `Python-urllib/x.y` - with a **403 (error 1010)** before the worker ever runs. `/watchlist` is - unauthenticated, so a 403 there means the edge blocked you, not that you lack access — - and the watcher then holds its current set forever with zero sources, which cascades - into `cdn` restart-looping because no domain is ever baked. -- **Points embedded before the app dimension have no `meta.app`.** They will not match - an app-scoped filter or bake into a three-part domain. A collection from an older - build needs a re-embed (drop it and let the seed rerun), not a migration. -- **`cdn` restart-loops for a few seconds on first boot** — `cdn serve` exits if its - directory does not exist yet, and the watcher creates it when it bakes the first - domain. The restart policy covers the window. -- **A registry blip does not tear down the fleet.** If the watch-list fetch fails, the - watcher keeps every running source rather than reading the failure as - "everyone unsubscribed". -- **`Api key is used with an insecure connection`** is expected inside the compose - network. It matters only if you expose Qdrant publicly. -- **The subscribe cursor lives at `/data/.fangorn/`** — that is why the working - directory is the mounted volume; an ephemeral one replays from scratch on restart. -- **The embedding model is baked into the image.** Changing `--embedding-model` means - rebuilding, or it downloads at every container start. -- **CPU-only hosts work** because `_build_text_embedding()` asks onnxruntime which - providers exist before requesting CUDA. A GPU box still selects CUDA automatically. -- **`/browse` is not namespace-scoped.** It returns the whole collection. The search - routes are the scoped ones. +- **The `fangorn` SDK version is pinned in the `Dockerfile` and must be bumped when the DataRegistry moves.** The registry address rides inside the SDK's `config.js`, so a cached layer on an old version reads a retired registry and sees none of the state published to the new one. +- **The compose `mcp` service serves the whole data collection.** It is bound to one `--cdn-url` at startup. Its endpoint is `/mcp` not `/` +- **The watch-list fetch sends an explicit `User-Agent`.** Cloudflare's bot check answers urllib's default `Python-urllib/x.y` with a **403 (error 1010)** before the worker ever runs. `/watchlist` is unauthenticated, so a 403 there means the edge blocked you and the watcher then holds a zero-source set forever, which cascades into `cdn` restart-looping because no domain is ever baked. +- **`cdn` restart-loops for a few seconds on first boot** — `cdn serve` exits if its directory doesn't exist yet, and the watcher creates it when it bakes the first domain. The restart policy covers this window. +- **A registry blip does not tear down the fleet.** If the watch-list fetch fails, the watcher keeps every running source rather than reading the failure as "everyone unsubscribed". +- **`Api key is used with an insecure connection`** is expected inside the compose network. It matters only if you expose Qdrant publicly. +- **The subscribe cursor lives at `/data/.fangorn/`** The working directory is the mounted volume since an ephemeral one replays from scratch on restart. +- **`/browse` is not namespace-scoped.** It returns the whole collection. The search routes are the scoped ones. ## Teardown ```sh docker compose down -v # -v also drops the Qdrant data and every CDN shard +gcloud compute instances delete quickbeam-1 --zone=$REGION-a +gcloud compute firewall-rules delete quickbeam-http ``` diff --git a/README.md b/README.md index ff5bb55..5953315 100644 --- a/README.md +++ b/README.md @@ -1,936 +1,316 @@ # quickbeam -This repo contains infrastructure for building and serving vector search over on-chain data sources registered with [Fangorn](https://github.com/fangorn-network/fangorn). The core script pulls manifests from The Graph, resolves payloads from IPFS, walks the typed graph they describe, and then builds embeddings via fastembed/ONNX. - -> **Two meanings of "bundle".** This doc uses the word in two unrelated ways: -> - **Schema bundle** (`--bundle`) — a registered subgraph schema whose v3 manifests carry typed node chunks plus an edge chunk. The builder walks those edges to join records. -> - **Snapshot bundle** (`--bundle-cid`, `/bundle/*`) — an exported NDJSON copy of the populated Qdrant collection, used to seed new instances without a GPU. +Semantic search over [Fangorn](https://github.com/fangorn-network/fangorn) knowledge graphs. A publisher versions a graph offchain and anchors it onchain; quickbeam reads that graph with the `fangorn` light client, embeds it into Qdrant, and serves it via three means: an HTTP search API, a static "Semantic CDN" of downloadable shards, and an MCP server for agents. --- -## How it works - -- **`quickbeam build`**: offline (local, trusted) embeddings builder. Pulls from subgraph, resolves IPFS, joins schemas, embeds, writes to Qdrant. -- **`quickbeam watch`**: live daemon that subscribes to one or more `PUBLISHER:SUBSPACE` sources within an `--app` (`*` widens either side to the whole app) and embeds commits as they land — push-based via the `fangorn` light client, no indexer and no polling. Keeps the GPU model loaded between changes. Each commit is a self-contained on-chain diff, so it embeds only what changed and **tombstones vertices the commit removed**. See [Watch for new events](#3-watch-for-new-events-optional) and [`docs/NEW_QUICKSTART.md`](docs/NEW_QUICKSTART.md). -- **`quickbeam serve`**: read-only API server. Connects to Qdrant and serves search, browse, and catalog endpoints. It does not ingest on startup, but instead expects the collection to already be populated, either by the builder or by seeding from a snapshot. Can optionally run the watcher alongside it (`serve --watch`) so one process both ingests and serves, and can gate the search routes behind [x402 payments](#x402-payment-gating). -- **`quickbeam mcp`**: a [Model Context Protocol](#mcp-server) layer for agents. A self-contained **local pull-client of the [Semantic CDN](#semantic-cdn)** — it pulls a dataset's shards and searches them locally (the query never leaves the process), exposing semantic search *and* typed-edge graph traversal over raw records, with on-chain provenance on every result, and can optionally charge the calling agent per tool call via [x402](#x402-payment-gating). -- **`quickbeam cdn` + `quickbeam pull`**: the [Semantic CDN](#semantic-cdn) — instead of running queries on the server (where the node sees every query = intent), the operator *bakes* the embedded graph into immutable, content-addressed shard files (a "domain") and *serves* them as static, resumable downloads. A user *pulls* a domain into their own local Qdrant and queries it offline. Knowledge moves to the user; the network never sees a query. See [docs/SEMANTIC_CDN.md](docs/SEMANTIC_CDN.md). - -The builder produces the record shape `{ track_id, fields, meta }` by walking a typed graph, through one of two data sources: - -- **Schema bundle (`--bundle`)** — a single bundle schema publishes manifests carrying typed node chunks (`{id, type, fields}`) and edge chunks (`{rel, from, to}`). The builder walks one publisher's graph. -- **Composed view (`--view`)** — fuses several publishers' bundles into one graph, joining on global identity (Entity URI + aliases + `sameAs` linksets) before projecting. - -Both are projected the same way: one or more **root profiles** (`--root-profile`, see `ROOT_PROFILES`) each walk the graph from a chosen root type and emit a distinct document. Everything downstream (role inference, embedding text, Qdrant payload) is identical. - -### Ingest engine layout - -The offline ingestion engine (shared by `build` and `watch`) lives in the `quickbeam.ingest` package: - -``` -quickbeam/ingest/ - build.py the `quickbeam build` CLI (parse_args + main) - identity.py deterministic point ids + the matryoshka vector transform - checkpoint.py resumable-build state (ingest checkpoint + role map) - embed.py fastembed engine (GPU-OOM resilient), doc-text composition, Qdrant indexes + upload - umap.py 2-D UMAP projection → catalog-map artifact / px-py payloads - commits.py git-native tips: unwrap commits, diff, tombstone removed entities - sources/subgraph.py The Graph event queries - sources/ipfs.py IPFS CID resolution - graph/projection.py ROOT_PROFILES, the graph walk, and the shared join helpers - graph/bundle.py single-publisher bundle join - graph/view.py multi-source view fusion (union-find over global identity) -``` - -`quickbeam/embeddings.py` is now a thin back-compat facade that re-exports this package, so existing `from quickbeam.embeddings import ...` imports keep working; new code should import from the specific `quickbeam.ingest.*` module. - ---- - -## Installation +## Install ```sh -# From the repo root -python -m venv venv -source venv/bin/activate +python -m venv venv && source venv/bin/activate -pip install -e ".[gpu]" # CUDA-accelerated embeddings (recommended for build) -pip install -e ".[cpu]" # CPU-only fallback +pip install -e ".[gpu]" # CUDA embeddings (fastembed-gpu) +pip install -e ".[cpu]" # CPU-only +pip install -e ".[agent]" # + MCP server and x402 payments +pip install -e ".[dev]" # + pytest ``` -This installs the `quickbeam` CLI entry point. Run `quickbeam --help` to see all commands. - -``` -quickbeam build Build embeddings from subgraph / IPFS data into Qdrant -quickbeam watch Live daemon: poll subgraph for new events and embed automatically -quickbeam serve Start the Fangorn search API server (optionally with --watch + x402) -quickbeam mcp Run the MCP server exposing search as agent tools (x402-aware) -quickbeam cdn Semantic CDN: bake the embedded graph into static, pullable domain shards -quickbeam pull Pull a domain from a Semantic CDN into a local Qdrant collection -quickbeam export Export the Qdrant collection as an NDJSON bundle -quickbeam migrate Migrate a local Qdrant collection to Qdrant Cloud -quickbeam data An ETL pipeline to generate seed / test data from public data sources -``` +Requires Python ≥3.12 and the [`fangorn` CLI](https://github.com/fangorn-network/fangorn) on `PATH` (or pass `--fangorn-bin`). The fangorn CLI refuses to start without +`ETH_PRIVATE_KEY`, but using a throwaway key is fine if you are not writing onchain. We recommend setting `PINATA_GATEWAY` too since the default `ipfs.io` is not guaranteed to serve content (ISP provider specific). -The `mcp` and x402 layers need extra dependencies (FastMCP + EIP-712 signing): +## Commands -```sh -pip install -e ".[agent]" # fastmcp + eth-account + httpx -pip install -e ".[dev]" # pytest + fastmcp + eth-account (to run the test-suite) -``` - -### Running the Audius demo - -Two sovereign publishers fused by a linkset, searched client-side in the browser. -**[`examples/audius/audius-build/RUNBOOK.md`](examples/audius/audius-build/RUNBOOK.md)** takes it from a fresh clone to a -running demo — Part 0 is setup, Part 1 rebuilds the graph with one command, Part 2 is -the optional on-chain publish. The build outputs (220 MB) are gitignored and -regenerated; only the crawl cache can't be reproduced byte-for-byte from the repo. +| Command | What it does | +|---|---| +| `quickbeam build` | One shot: `fangorn read` one or more namespaces, project, embed into Qdrant | +| `quickbeam watch` | Daemon: `fangorn subscribe` and embed commits as they land (push-based) | +| `quickbeam serve` | The search API (`--watch` also runs the daemon in the same box) | +| `quickbeam cdn bake\|append\|edges\|precompute\|index\|serve` | Bake and serve static shards | +| `quickbeam pull` | Pull a CDN domain into a local Qdrant collection | +| `quickbeam mcp` | MCP server for agents. A local CDN pull-client | +| `quickbeam export` | Dump the collection as NDJSON | +| `quickbeam migrate` | Move a local collection to Qdrant Cloud | +| `quickbeam data …` | ETL pipelines and pluggable scraper sources | + +Every command has its own `--help` flag for more info. --- ## Quickstart -> **New: the git-native flow.** Datasets are now versioned **repos** you `commit` and -> `push` (git for data), and `watch`/`build` embed off the commit diff. For the -> end-to-end runbook in that model — including delete propagation and inherited -> embedding contracts — see **[`docs/NEW_QUICKSTART.md`](docs/NEW_QUICKSTART.md)**. The -> classic subgraph-event runbook below still works. - -### 1. Run Qdrant +### Run everything with compose ```sh -docker run -d -p 6333:6333 -p 6334:6334 \ - -v "$(pwd)/python/qdrant_storage:/qdrant/storage:z" \ - --name qdrant-demo \ - qdrant/qdrant +cp .env.example .env # set PINATA_GATEWAY, ETH_PRIVATE_KEY, QDRANT_API_KEY, SOURCES_URL +docker compose up -d --build ``` -### 2. Build embeddings +Currently, there are four services contained in one image: `qdrant`, `watch`, `serve` (:8080), `cdn` (:8090), `mcp` (:8765). Namespaces are **not** configured in compose. The `watch` command polls the `SOURCES_URL` for its watch list and starts or cancels a stream per namespace without requiring a restart. Full deployment guide: **[`DOCKER-README.md`](DOCKER-README.md)**. -Link the NVIDIA libraries if using CUDA: +### Or run the pieces by hand ```sh -export LD_LIBRARY_PATH=\ -$VIRTUAL_ENV/lib/python3.12/site-packages/nvidia/cudnn/lib:\ -$VIRTUAL_ENV/lib/python3.12/site-packages/nvidia/cublas/lib:\ -$LD_LIBRARY_PATH -# verify Cuda is available -python -c "import onnxruntime as ort; print('Available Providers:', ort.get_available_providers())" -``` - -#### From a schema bundle - -```sh -quickbeam build \ - --bundle fangorn.mb.creativecore.v1=0xac92db425c174e4301cd41e81e16d99fd2c5f4e2f13b739004996e95875e990d \ - --root-profile track \ - --graph-api-key <> \ - --ipfs-gateway https://green-reasonable-heron-957.mypinata.cloud/ipfs \ - --dim 256 \ - --umap \ - --reset -``` - -Pass `--root-profile` at least once (repeatable) to choose which projection(s) to emit. Use `--view NAME=0x...` instead of `--bundle` to fuse several publishers' bundles into one graph before projecting. +# 1. Qdrant +docker run -d -p 6333:6333 -p 6334:6334 -v "$(pwd)/db/qdrant:/qdrant/storage" qdrant/qdrant -#### Resuming a build +# 2. Embed a namespace (one shot) +quickbeam build --source 0x147c24c5...:robinhood --root-profile asset -`quickbeam build` is fully resumable. Progress is saved to `--checkpoint-file` (default `./db/ingest_checkpoint.json`) at the granularity of individual bundle manifests. On re-run without `--reset`, already-completed manifests are skipped before any IPFS data is fetched — RAM usage stays flat regardless of how many records have already been embedded. - -The checkpoint tracks: -- `completed_manifest_cids` — manifests that have been fully embedded (skipped on re-run). -- `processed_track_ids` — records within the currently in-flight manifest, used only for crash-recovery mid-manifest. Cleared when the manifest completes. -- `last_tip` — per-schema, the last-built **commit** CID. The watcher diffs the new tip - against it to embed only the delta and tombstone removed entities (git-native flow). - -#### UMAP only (reproject existing collection) - -```sh -quickbeam build --umap-only +# 3. Serve it +quickbeam serve --port 8080 +curl 'localhost:8080/search?q=semiconductors&n_results=5' ``` -### 3. Watch for new events (optional) +For CUDA, link the NVIDIA libs first (see [`gpu-env.sh`](gpu-env.sh)) and check with `python -c "import onnxruntime as ort; print(ort.get_available_providers())"`. Without CUDA it falls back to CPU automatically. -After an initial build, run `quickbeam watch` to keep the collection up to date as commits land on-chain. It is push-based: one `fangorn subscribe` light-client stream per source, no indexer and no polling (`--poll-interval` is only the reconnect backoff). +Live ingestion instead of a oneshot build: ```sh -quickbeam watch \ - --app sond3r.test.1 --source 0x7a78...:media \ - --collection sond3r --cdn-dir ./cdn --cdn-domain sond3r +quickbeam watch --app fangorn --source 0x147c...:robinhood --cdn-dir ./cdn ``` -On startup each source is seeded once with `fangorn read` so the existing corpus is embedded before the stream goes live; after that every commit arrives as a self-contained diff, is applied to an in-memory snapshot, re-projected, embedded, and (with `--cdn-dir`/`--cdn-domain`) shipped as a CDN delta. The GPU model is loaded once and kept alive. +--- -#### Namespaces: `app : publisher : subspace` +## Concepts -A namespace is a triple. Two of the three come from the source string, the third does not: +### A namespace is `app:publisher:subspace` -| Part | Where it comes from | -|---|---| -| **app** | `--app `. **Not** part of `--source`. Omit it and you get whatever the local fangorn client is set to (`appId` in `~/.fangorn/config.json`, `fangorn set-app`, or `FANGORN_APP_ID`) — fine interactively, a footgun for a daemon, so pass it explicitly. | -| **publisher** | the address left of the `:` in `--source` | -| **subspace** | the name right of the `:` | +How the app portion is supplied depends on how sources are given. -All three are indexed topics on the `StateCommitted` event, so the subscription is a node-side filter — `*` on either side of the `:` widens it: +**Static `--source` flags** — `build`, `watch` and `serve` take `OWNER:NAMESPACE` (repeatable, **two parts**). All of them run under the single `--app`, so one process covers one app. `*` on either side widens to the whole app: ```sh -# One publisher's one subspace — the tightest filter: -quickbeam watch --app sond3r.test.1 --source 0x7a78...:media - -# One publisher, every subspace they write in this app: -quickbeam watch --app sond3r.test.1 --source '0x7a78...:*' - -# That subspace name across every publisher: -quickbeam watch --app sond3r.test.1 --source '*:media' - -# The whole app — every publisher, every subspace: -quickbeam watch --app sond3r.test.1 --source '*:*' +--source 0x147c...:robinhood # one publisher, one subspace +--source '0x147c...:*' # one publisher, every subspace in the app +--source '*:docs' # that subspace name across every publisher +--source '*:*' # the whole app ``` -The same override exists on the fangorn CLI itself as a global option, ahead of the subcommand: `fangorn --app sond3r.test.1 read media --owner 0x8ce6...`, `fangorn --app sond3r.test.1 subscribe --all`. - -`--source` is repeatable; each source gets its own independent subscription, and a wildcard source keeps a separate in-memory snapshot per `(publisher, subspace)` pair it sees, seeding each one the first time a commit for it arrives. - -#### Catching up on history - -A subscription starts at the current chain tip, so a wildcard source only learns about namespaces that commit while it is running (there is no single namespace for it to seed with `fangorn read`). To pick up what was published earlier, replay from a block: +**A watch list — `watch --sources-url`** (what the compose deployment uses). Here **each entry carries its own app**, so one instance serves several. For example: -```sh -quickbeam watch --app sond3r.test.1 --source '*:*' --from-block 297435100 +```json +["app1Id::", "app2Id::", "fangorn:0x147c...:robinhood"] ``` +Here, we are watching all of app1, app2, but only a specific namespace for a specific publisher in the fangorn app. -Catch-up is fetched in windows of `FANGORN_LOG_WINDOW` blocks (default 1000) because RPC providers cap the block span of one `eth_getLogs` call — the public Arbitrum Sepolia endpoint rejects anything wider with a bare `internal server errror`. Pick a block near the first publish; `--from-start` (genesis) is only practical against a private RPC with a large window. - -#### Watching returns nothing +Entries are `"APP:OWNER:NAMESPACE"` strings or `{app, owner, namespace}` objects, and an empty or `*` part is a wildcard. So `app1Id::` means *everything in app1*. `--app` (the `APP` env var in compose) is only the **fallback** for an entry naming no app. An entry with no app and no fallback is **dropped** because reading the wrong app silently indexes the wrong graph. -In order of likelihood: +> Warning: **Don't pass the three-part form to `--source`.** It splits on the first colon only, so `--source fangorn:0xA:docs` silently becomes owner `fangorn`, namespace `0xA:docs` with no error and nothing watched. -1. **The app id is not what you think.** Without `--app` the watcher inherits the local client's app (`~/.fangorn/config.json`); topic 2 of the `eth_getLogs` call in the error output is `keccak256(appId)`, so you can check which one it used. Passing the *app* name as the *subspace* — `--source 0xyou:sond3r.test.1` when `sond3r.test.1` is your app — filters on a namespace nobody has ever written to. -2. **The namespace really is empty.** Check directly: `fangorn read --owner `. A `head: null` with `vertices: []` means nothing has been committed there. Use `--source '*:*'` to see what the app *does* contain. -3. **Live-from-tip.** Nothing has committed since you started — see catch-up above. -4. **RPC rate limits.** `429 Too Many Requests` from `https://sepolia-rollup.arbitrum.io/rpc` kills the stream and the watcher reconnects on the backoff. Point the SDK at a private endpoint. +> Note `--source` means something different on two other commands: `cdn edges --source` is a **path** to a linkset JSON, and `data events-fetch --source` names a **scraper** (`eventbrite` / `eventbrite-location` / `tribe`). Check the `--help` flag for the command you're running. -### 4. Start the server +A wildcard source has no single namespace to seed with `fangorn read`, so it only sees namespaces published before it started if you also pass `--from-block N`. -```sh -quickbeam serve \ - --collection fangorn \ - --qdrant-host localhost --qdrant-port 6333 -``` +### One collection, scoped by filter -The server starts immediately and serves whatever is already in Qdrant. Use `POST /reingest` to pull new subgraph data without restarting. +Every watched namespace embeds into **one** Qdrant collection. Points carry `owner` at the payload top level and `meta.app` / `meta.namespace` nested, so a caller's slice is a filter instead of a copy of the vectors: -#### Serve + watch in one process - -Pass `--watch` to run the live embedding daemon alongside the server, so one deployment both ingests and serves. **Everything before `--watch` configures the server; everything after it is forwarded verbatim to `quickbeam watch`.** The watcher writes to Qdrant; the server reads from it; the watcher is a child process that is terminated when the server exits. - -```sh -quickbeam serve \ - --collection fangorn \ - --watch \ - --bundle "fangorn.mb.bundle.v1=0xabc123..." \ - --graph-api-key \ - --ipfs-gateway https://your-gateway.mypinata.cloud/ipfs \ - --poll-interval 120 ``` - -Note this loads the embedding model twice (once in each process), so plan VRAM accordingly — or run the two commands as separate services against the same Qdrant. - ---- - -## Deployment - -For running Quickbeam as a service rather than from a shell, see -**[`DOCKER-README.md`](DOCKER-README.md)**. - -The short version: **one shared instance, and each namespace is embedded exactly -once.** Docker Compose brings up `qdrant`, `watch`, `cdn serve`, `serve` and `mcp`; -`watch` polls `--sources-url` for its watch list and starts or cancels a stream per -namespace with no restart, so namespaces arrive from the Fangorn website rather than -by editing config here. - -Users get a **view** — a named set of namespaces with its own search URL and MCP -catalog. A view is a [`scope` filter](#api) over the one shared collection, never a -copy of the vectors, so a second requester asking for an already-watched namespace -costs no extra indexing. `webworker/quickbeam-registry` holds the views, serves the -**deduplicated union** of their sources as the watch list, gates writes on the -caller's subscription, and proxies queries so a browser reaches the instance over -HTTPS. - -That shape exists because the embedding model and the embedding work are the expensive -residents: one process loads the model once for every namespace it watches, where a -container per namespace would load a copy each and re-embed identical content. - ---- - -## Snapshots - -A snapshot is a portable copy of the populated Qdrant collection. It lets you seed a new instance — including one without a GPU — from a pinned IPFS artifact. - -``` sh -curl -X POST localhost:6333/collections/fangorn/snapshots -# grab the latest snapshot from qdrant -docker exec qdrant-core find /qdrant -name "*.snapshot" -# exfiltrate the latest snapshot from docker and store locally -docker cp qdrant-core:/qdrant/snapshots/fangorn/fangorn-8009660693873684-2026-06-16-22-11-57.snapshot ~/.snapshot -# zip the snapshot -gzip -k ~/.snapshot -# pin to ipfs (from the root) -node src/pinata.mjs upload ~/.snapshot.gz "fangorn-8009660693873684-2026-06-16-22-11-57.snapshot.gz" -# note the sha256 sum of the snapshot before cleanup -sha256sum ~/.snapshot -rm -rf ~/.snapshot ~/.snapshot.gz +?scope=APP:OWNER:NAMESPACE repeatable; triples OR, parts within a triple AND +?scope=:0xA:tracks any part may be empty to leave it unconstrained +?scope=0xA:tracks two-part OWNER:NAMESPACE still accepted ``` -### Export +`scope` takes precedence over the separate `app` / `owner` / `namespace` params. `/browse` is not scoped and returns the whole collection. -```sh -# Full bundle (fields + embeddings — use this to seed a complete server) -quickbeam export --src http://localhost:8080 --out bundle.ndjson - -# Embeddings only (track_id + vector — minimal artifact for vector-space clients) -quickbeam export --src http://localhost:8080 --out embeddings.ndjson --embeddings-only -``` +### Root profiles -### Pin to IPFS +A profile walks the graph from every vertex carrying a given tag and folds its neighbors into one document. With no `--root-profile`, one profile is auto-derived per distinct vertex tag present in the source. Override with `--profiles-file` (see [`quickbeam/profiles.example.json`](quickbeam/profiles.example.json)): -```sh -gzip -k bundle.ndjson -node src/pinata.mjs upload bundle.ndjson.gz "quickbeam-bundle-v1" +```json +{ "file": { "root_type": "File", "max_depth": 2, "include": ["File"], + "content_fields": ["filename", "text"] } } ``` -See [Managing Pinata data](#managing-pinata-data) for listing and bulk-deleting pinned files. - -### Export a Qdrant snapshot - -```sh -# Write snapshot to Qdrant storage -curl -X POST localhost:6333/collections/fangorn/snapshots +`--max-depth`, `--label-cap` (max folded labels per relation group) and `--node-cap` (max nodes visited per root) bound the walk. -# Find the file -docker exec qdrant-core find /qdrant -name "*.snapshot" +### Semantic roles -# Copy out and compress -docker cp qdrant-core:/qdrant/snapshots/fangorn/ ~/.snapshot -gzip -k ~/.snapshot +`quickbeam/roles.py` infers which fields are `title`, `subtitle`, `tags`, `temporal`, `spatial`, `media` from field names and value shapes so the same server and UI work over music, filings, or OSM changesets with no per-domain config. The inferred map is cached in `--role-map-file` and served at `GET /schema`. -# Pin to IPFS -node src/pinata.mjs upload ~/.snapshot.gz ".gz" - -# Record the sha256 before cleanup -sha256sum ~/.snapshot -rm ~/.snapshot ~/.snapshot.gz -``` - -### Seed on startup +--- -```sh -quickbeam serve \ - -s test.sond3r.track.invariants.3=0x... \ - --bundle-cid QmYourBundleCIDHere -``` +## The search API (`quickbeam serve`) -If the collection is empty and `--bundle-cid` is provided, the server fetches the NDJSON from IPFS and upserts it in the background. The server is live immediately — results populate as the seed progresses. If the collection already has points, the seed is skipped. +JSON everywhere. Hits are `{ id, fields, owner, meta, score?, embedding? }`, where `meta` carries on-chain provenance. -### Manual import +| Route | | +|---|---| +| `GET /search?q=&n_results=&scope=` | Semantic search; embeds the query server-side | +| `POST /search/vector` | Query by raw vector — `{embedding, n_results, scope}` | +| `POST /search/text` | Lexical search over title/subtitle/tags | +| `POST /embed` | Embed text with the ingestion model — `{text}` or `{texts}` | +| `GET /browse?limit=&offset=` | Paginated browse of the whole collection | +| `GET /records?ids=a,b,c` | Fetch specific records by id (max 200), with vectors | +| `GET /adjacency?id=&rel=&dir=` | Relation groups, or the neighbor records. Needs `--adjacency-db`, else 501 | +| `GET /bucket/{n}?owner=` | Private retrieval — see below. 501 without `--index-layout`; empty without `cdn index --push-cells` | +| `GET /schema` | Inferred role map + facet vocabularies | +| `GET /catalog/map`, `POST /catalog/map/refresh` | 2-D UMAP projection of the collection | +| `GET /bundle/export?scope=` | Stream the collection as NDJSON | +| `POST /bundle/import`, `POST /bundle/upsert` | Load points back in | +| `POST /reingest`, `POST /reingest/full` | Re-read `--source` namespaces (changed only / everything) | +| `GET /health`, `GET /ready`, `GET /debug` | Counts, caches, checkpoint, join diagnostics | + +Useful flags: `--collection`, `--qdrant-url` + `--qdrant-api-key` (remote Qdrant), `--catalog-map-file` (serve a prebuilt map instead of recomputing), `--dim` (default: read from the collection and Matryoshka-truncate queries to match). + +`serve --watch` runs the daemon as a child process. Everything before `--watch` configures the server and everything after is forwarded to `quickbeam watch`. ```sh -# From a local file -cat bundle.ndjson | curl -X POST http://localhost:8080/bundle/import \ - -H "Content-Type: application/x-ndjson" \ - --data-binary @- - -# Stream directly between two instances -curl -N http://host-a:8080/bundle/export \ - | curl -X POST http://host-b:8080/bundle/import \ - -H "Content-Type: application/x-ndjson" \ - --data-binary @- +quickbeam serve --collection fangorn --watch --source 0xA:docs --app fangorn ``` --- ## Semantic CDN -The search server runs queries **server-side** — which means the node observes every -query vector, and a semantic query *is* intent. The Semantic CDN inverts this: the -operator distributes the **public** embeddings as static, content-addressed artifacts; -the user pulls a slice into their **own** local Qdrant and queries it offline. Knowledge -moves to the user, the network never sees a query. Full walkthrough in -[docs/SEMANTIC_CDN.md](docs/SEMANTIC_CDN.md); the short version: +Running queries server-side would mean the node sees every query vector. Because the embeddings are not opaque, vec2text-style inversion reconstructs short inputs almost exactly. The CDN inverts this paradigm. It bakes the collection into immutable shard files, serves them as static resumable downloads, and lets the client search locally. ```sh -# (operator) declare domains as filters over the collection -cat > domains.json <<'JSON' -{ "domains": { - "music": { "description": "Recordings & artists", "filter": { "entityType": ["Recording","Artist"] } }, - "venues": { "description": "Places & events", "filter": { "entityType": ["Place","Event"] } } -} } -JSON - -# (operator) bake immutable shards from Qdrant, then serve them statically -quickbeam cdn bake --config domains.json --cdn-dir ./cdn --collection fangorn -quickbeam cdn serve --cdn-dir ./cdn --port 8090 +# operator: declare domains as filters over the collection (domains.json), then bake +quickbeam cdn bake --config domains.json --cdn-dir ./cdn --collection fangorn +quickbeam cdn edges --cdn-dir ./cdn --domain mydomain --source linkset.json # a FILE here +quickbeam cdn serve --cdn-dir ./cdn --cors --port 8090 -# (user) pull a domain into a LOCAL collection, then query it offline -quickbeam pull music --cdn-url http://localhost:8090 --collection music_local -quickbeam serve --collection music_local # local search — CDN sees nothing +# user: pull a domain into a LOCAL collection and query it offline +quickbeam pull mydomain --cdn-url http://localhost:8090 --collection mydomain ``` -A **domain** is operator-declared (a named `entityType`/`owner` filter, in `domains.json`). -`bake` writes `cdn//shard-NNNN.ndjson.gz` (reusing the `/bundle/export` row shape) -plus a `manifest.json` carrying a **sha256 per shard**, and a top-level `catalog.json`. - -Each `manifest.json` is also **self-describing** so a pulled domain drives a generic, -schema-agnostic client with no hardcoding: an inferred `role_map` -(title/subtitle/tags/spatial/…) and an `entity_types` vocabulary with per-type counts are -always baked in. Two optional per-domain keys in `domains.json` add more — `bundle_schema` -(path to a Fangorn bundle schema → copies its type + relationship vocabulary into -`manifest.bundle`) and `presentation` (an overlay of icons / accent colors / `fieldLabels` / -`externalUrl` templates, passed through verbatim for UI polish). See -[docs/SEMANTIC_CDN.md](docs/SEMANTIC_CDN.md#1-declare-domains-domainsjson). -`serve` is a separate minimal FastAPI app exposing only static reads (`/catalog`, -`/domains/{name}/manifest`, `/domains/{name}/edges`, `/domains/{name}/shards/{file}`) with -HTTP **Range** support, so shards are cacheable and downloads resume. `pull` verifies every -shard against its sha256 and loads it into the local collection with deterministic point -ids, so an interrupted or repeated pull is safe. - -Alongside the semantic axis (record shards), a domain can carry a **relational axis** — its -linkset of typed edges (`{rel, from, to, fromType, toType}`, see [linkgen](quickbeam/pipelines/linkgen.py)). -Edges live at `cdn//edges.json`, served at `/domains/{name}/edges`, so a pull-client (the -[MCP server](#mcp-server)) can walk the knowledge graph offline. Edge endpoints are the same -node ids as records' `track_id`, so the two axes join by id. Two ways to populate it: - -- **Live** — `quickbeam watch` ships the typed edges it fetches on-chain each cycle, - merging them into `edges.json` (deduped, incremental — the relational counterpart to the - record delta shards). The relational axis stays fresh with the stream, no manual step. -- **One-shot** — `quickbeam cdn edges --domain --source ` installs a - linkset from a file (e.g. a staged `stage_volumes/*_edges.json`). - -Re-run `cdn serve` after first attaching edges so the running app picks up the `/edges` route. - ---- - -## Managing Pinata data +**Order matters: `bake` first, then `edges` / `precompute` / `index`.** The latter three write sidecars into the domain directory, and `bake` rmtrees and atomically replaces it. This means a re-bake discards everything installed before it. -`src/pinata.mjs` is a small CLI for the Pinata account that backs your IPFS pins (snapshots, bundles). It needs `PINATA_JWT` in the environment (or a `.env` at the repo root). +A domain filter keys on `entityType`, `owner`, `namespace`, `app` (AND-ed; an empty filter selects everything which, on a shared collection, bakes *every* namespace into one domain, so always set at least `namespace`). The linkset `cdn edges` takes is a list of `{rel, from, to, fromType, toType}`, or `{"edges": [...]}` which is the shape `data linkgen` emits. `quickbeam watch --cdn-dir` bakes and appends live, naming each domain `{app8}-{owner8}-{namespace}`. -```sh -# Upload / pin a file (replaces the old pin.mjs) -node src/pinata.mjs upload ~/.snapshot.gz "sond3r.snapshot.2026-06-14.gz" - -# List pins (optionally filter by name substring) -node src/pinata.mjs list -node src/pinata.mjs list --name sond3r - -# Delete by file ID(s) -node src/pinata.mjs delete +CDN routes: `GET /catalog`, `/events`, `/domains/{name}/manifest`, `/edges`, `/edges.gz`, `/shards/{file}`, `/index/{file}`, `/health`. -# Bulk-delete every file whose name matches a substring (prompts unless --yes) -node src/pinata.mjs delete-pattern "sond3r.snapshot" +### Private retrieval (`cdn index` + `/bucket`) -# Delete everything in the account (prompts unless --yes) -node src/pinata.mjs delete-all -``` - -Pinata's name filter is a **contains** match, not a strict prefix — name files with a consistent prefix (e.g. `sond3r.snapshot.*`) for clean targeting. Also available via `npm run pinata -- `. +For data collections that are too large to ship whole, `quickbeam cdn index` fits a **public** codebook over the vectors and buckets its cells. The client embeds locally, finds its nearest centroid, and asks the server for that centroid's *bucket*. The bucket is one integer and a deterministic public function of the query which is cacheable. It then re-ranks the returned candidates against its true vector. `--report` measures what the disclosure costs in recall. ---- - -## Migrating to Qdrant Cloud +Setup: ```sh -quickbeam migrate +quickbeam cdn bake --config domains.json --cdn-dir ./cdn --collection fangorn +quickbeam cdn index --cdn-dir ./cdn --domain mydomain \ + --push-cells --collection fangorn # writes the codebook AND backfills Qdrant +quickbeam cdn serve --cdn-dir ./cdn --cors --port 8090 # delivers codebook.i8 + layout.json +quickbeam serve --collection fangorn --port 8080 \ + --index-layout ./cdn/mydomain/index/layout.json ``` -`migrate.py` contains hardcoded source/destination credentials — edit it before running. - -Then point the server at the cloud cluster: +> Note: **`--push-cells` is required and queries will fail silently if it is not included.** `/bucket` filters Qdrant on a `cell` payload field that only `--push-cells` writes, so a server started with `--index-layout` against a collection that was never backfilled answers `200` with `{"count": 0, "results": []}`. -```sh -quickbeam serve \ - -s ... \ - --qdrant-url https://your-cluster.cloud.qdrant.io:6334 \ - --qdrant-api-key -``` +`cdn serve` and `serve` are separate processes on separate ports. The client pulls the public codebook from the first and sends its bucket id to the second. See [`examples/audius/audius-large-build/RUNBOOK.md`](examples/audius/audius-large-build/RUNBOOK.md) for a working four-process local setup over 1.9M records. --- -## x402 payment gating - -[x402](https://www.x402.org/) is the HTTP `402 Payment Required` protocol for paid APIs. quickbeam implements the `exact` scheme over an EVM stablecoin (USDC by default) using EIP-3009 `transferWithAuthorization` signatures. It lives in [`quickbeam/x402.py`](quickbeam/x402.py) and is used in two independent places: - -1. **HTTP gating** — `quickbeam serve --x402-pay-to 0x...` installs middleware that gates the search routes (`/search`, `/search/vector`, `/search/text`). -2. **Per-tool gating** — `quickbeam mcp --x402-pay-to 0x...` charges the calling agent per MCP tool call (see [MCP server](#mcp-server)). +## MCP server (`quickbeam mcp`) -### The flow - -1. Client calls a gated route with no `X-PAYMENT` header → server replies `402` with a JSON body `{ x402Version, accepts: [requirements], error }`. -2. Client signs an EIP-3009 authorization for the quoted price, base64-encodes the payment into `X-PAYMENT`, and retries. -3. Server verifies the signature, settles, and serves the response with an `X-PAYMENT-RESPONSE` header describing settlement. - -Verification is pluggable. By default a **local verifier** recovers the EIP-712 signer and checks the authorization terms without broadcasting — suitable for testnets, demos, and tests. Point `--x402-facilitator ` at a real facilitator for on-chain verify + settle. +A self-contained **local pull-client** of the CDN. It downloads a domain's shards and searches them locally so the agent's query vector never leaves the machine. ```sh -# Gate the HTTP search routes at 0.001 USDC per request on Base Sepolia: -quickbeam serve \ - -s test.sond3r.track.invariants.3=0x... \ - --x402-pay-to 0xYourReceivingAddress \ - --x402-price 0.001 \ - --x402-network base-sepolia +quickbeam mcp --cdn-url http://localhost:8090 --transport http --host 0.0.0.0 --port 8765 +quickbeam mcp --cdn-url http://localhost:8090 --transport stdio # MCP Inspector / Claude Desktop ``` -Supported networks: `base-sepolia` (default), `base`, `avalanche-fuji`. Each has a default USDC contract; override with `--x402-asset`. - -### Agent-side helper +The endpoint is **`/mcp`**, not `/`. -`quickbeam/x402.py` also ships `PayingClient`, an `httpx.AsyncClient` wrapper that transparently pays any `402` it receives (sign → retry → record settlement). This is the agent side, used by the test-suite and available for any Python client. +Tools: `list_datasets`, `describe`, `search`, `get`, `neighbors`, `relations`, `aggregate`, `export`, `refresh`. There are two ways to navigate: semantic (`search`) and relational (`neighbors` walks typed linkset edges by node id). `aggregate` reduces in-process and returns an N-row table. `export` writes a column slice to a local file and returns only its path. Both exist so analytics over the entirety of the data don't stream every record through the model's context. Every result carries on-chain provenance. --- -## MCP server - -`quickbeam mcp` is a [Model Context Protocol](https://modelcontextprotocol.io/) server ([`quickbeam/mcp_server.py`](quickbeam/mcp_server.py)) that exposes on-chain-published knowledge to agents. It is a **self-contained, local pull-client of the [Semantic CDN](#semantic-cdn)**: it pulls a dataset's immutable shards into an in-process index and searches them **locally** — the agent's query vector never leaves the process. That is the *"intent is private"* half of the Fangorn thesis, applied to the agent path (no query hits a central server, and there is no dependency on a live `quickbeam serve`). - -Agents get back the **raw record fields** (not a lossy title/subtitle/tags role-map projection — an LLM reasons over JSON fine) and navigate two axes: **semantic** (vector similarity) and **relational** (typed linkset edges — the knowledge-mesh axis). - -> **New here?** [`docs/MCP_QUICKSTART.md`](docs/MCP_QUICKSTART.md) walks an agent from zero to querying a live dataset (serve → watch → MCP → the five tools → registering with Claude Code). - -```sh -# Phase 1 — free tools, remote streamable-http transport: -quickbeam mcp --transport http --host 0.0.0.0 --port 8765 \ - --cdn-url http://localhost:8090 - -# local stdio (MCP Inspector / Claude Desktop): -quickbeam mcp --transport stdio --cdn-url http://localhost:8090 -``` - -### Tools - -- **`list_datasets()`** — the CDN catalog: what knowledge exists (name, description, count, entity types, embedding dim). Free. -- **`describe(dataset)`** — a dataset's entity types, real field vocabulary, relationship types (for `neighbors`), and embedding contract (model + dim). Free. -- **`search(dataset, query, limit=10, entity_type=None, owner=None)`** — meaning-based search. Embeds the query locally, returns records as `{ id, entityType, fields, score, provenance }` with the **raw fields**. Optional structured pre-filters by `entity_type` / `owner`. -- **`get(dataset, id)`** — one record by its exact id (which is also its graph node endpoint, e.g. `rh:asset:NVDA`). Free. -- **`neighbors(dataset, id, rel=None, direction="both", limit=25)`** — walk the linkset edges from a node ("what is connected to NVDA, and how"). Neighbors inside the dataset resolve to full `fields`; those outside it come back as `{ id, entityType }` endpoints. - -> **Relational-axis delivery.** `neighbors` sources edges from the CDN's `/domains/{name}/edges` endpoint, which [`quickbeam watch` keeps fresh live](#semantic-cdn) (or [`cdn edges`](#semantic-cdn) one-shot). If a domain has no CDN linkset yet, it falls back to a local one via `--edges ` (a JSON list of `{rel, from, to, fromType, toType}`, the shape linkgen/robinhood stage), and reports `relational_axis: "not delivered"` if neither is present. - -### Provenance - -Every result carries on-chain provenance as a first-class field, sourced from each Qdrant point's `meta`: - -```json -"provenance": { - "source_cid": "Qm…", // manifest CID the record was published in - "published": "2026-06-14T…", // ISO8601 from the block timestamp - "version": 1, - "publisher": "0x…" // publisher address -} -``` - -### Phase 2 — charge agents per call +## x402 payment gating -x402 gating for the MCP is **phased and isolated** in [`quickbeam/mcp_payments.py`](quickbeam/mcp_payments.py); with no `--x402-pay-to`, none of it runs and the tools are free. When enabled, each gated tool gains an optional `payment` argument: +Set `--x402-pay-to` on `serve` or `mcp` and gated routes/tools return HTTP 402 until the caller supplies a valid `X-PAYMENT` header (x402 v1, `exact` scheme, EIP-3009 `transferWithAuthorization`). ```sh -quickbeam mcp --transport http \ - --x402-pay-to 0xYourReceivingAddress \ - --x402-price 0.001 --x402-network base-sepolia +quickbeam serve --x402-pay-to 0xRECV --x402-price 0.001 --x402-network base-sepolia ``` -Since MCP has no HTTP headers, payment rides on a tool argument instead of `X-PAYMENT`: - -1. Agent calls `search(dataset, query)` with no `payment` → the tool returns the x402 requirements: `{ payment_required: true, accepts: [...] }`. -2. Agent signs the quoted requirement and calls again with `payment=` → the tool returns results plus a `payment` settlement receipt. - -The gated tools are the compute-bearing ones (`search`, `neighbors`); discovery (`list_datasets`, `describe`, `get`) stays free. The verify/settle primitives are reused verbatim from `x402.py`; only the transport (tool argument vs HTTP header) differs. - -> **Embedding quality note.** nomic-embed-text-v1.5 is asymmetric — documents are embedded with a `search_document:` prefix and queries with `search_query:`. The pull-client embeds queries locally with the `search_query:` prefix and applies the **same matryoshka transform** (LayerNorm → slice-to-dim → L2-normalize) the builder applied to documents, so query and document vectors share one space. Reusing that single transform (`quickbeam.embeddings.matryoshka`) is what keeps local retrieval correct. +`--x402-asset` defaults to the network's USDC. `--x402-decimals` converts the price to atomic units. Without `--x402-facilitator` the server verifies signatures locally without broadcasting. Point it at a facilitator to verify and settle on-chain. `quickbeam/x402.py` is self-contained and also implements the agent side which signs and retries a 402. --- -## Data pipelines - -The `quickbeam data` subcommands generate seed data for testing. `quickbeam data fetch` outputs flat `{ name, fields }` JSONL consumed by the ingest server's flat-schema path. `quickbeam data mb` outputs v3 bundle chunk files (node chunks + edge chunk) consumed by `quickbeam build --bundle`. - -### Last.fm + MusicBrainz - -Scrapes artist discographies via the Last.fm API and optionally enriches with ISRC codes and contributors from MusicBrainz. +## Publishing data -```sh -export LASTFM_API_KEY=your_key +quickbeam is datasource agnostic, but requires developers to implement their own scraper. Implement the `Source` contract (`read` / `build_graph` / `next_cursor`, or subclass `SourceBase`) and hand it to `Publisher`: -quickbeam data fetch --volume 1 --max-gb 9.5 -# Resumes automatically if interrupted — re-run the same command. -# When the volume ceiling is hit, upload and increment --volume. +```python +import quickbeam as qb +from my_scraper import MySource -quickbeam data fetch --volume 2 --max-gb 9.5 -quickbeam data fetch --volume 1 --artists-file artists.txt # custom seed list -quickbeam data fetch --volume 1 --no-mb # skip MusicBrainz lookups +qb.Publisher(MySource(), namespace="widgets").run() # ingest → repo init → commit → push ``` -Outputs `volume__core.jsonl` (structural) and `volume__taxonomy.jsonl` (genres/moods/themes/contexts). - -### MusicBrainz JSON dump - -Downloads the full MusicBrainz `release.tar.xz` dump (~23 GB) and extracts up to `--target-count` tracks with tag data. Resumable at every stage — re-run to pick up where it left off. The latest dump URL is discovered automatically; pass `--dump-url` to pin a specific one. - -```sh -quickbeam data mb --volume 1 --target-count 50000 --output-dir ./data -quickbeam data mb --volume 1 --target-count 50000 --connections 8 # faster download -quickbeam data mb --help -``` +A source package that registers itself under the `quickbeam.sources` entry-point group gets its own `quickbeam data ` command along with the shared harness flags (`--watch`, `--publish`, `--dry-run`) without changes to the repo. Core registers none. -The download uses `--connections` (default 4) parallel HTTP range requests, each writing to a non-overlapping slice of a pre-allocated file. A `.parts` sidecar tracks completed chunks so interrupted runs skip them on restart. Pass `--connections 1` to fall back to single-connection streaming. +Built-in ETL pipelines under `quickbeam data`: `fetch` (Last.fm + MusicBrainz), `mb` / `mbpg` (MusicBrainz dump / Postgres), `places-fetch` (Google Places), `events-fetch` (Eventbrite / Tribe), `schemagen`, `linkgen`, `keylink`, `prebake`. -Outputs three v3 bundle chunk files — ready to upload to IPFS and register as a bundle schema: - -| File | Contents | -|---|---| -| `volume__tracks.json` | `[{ id, type: "Track", fields: { trackId, isrcCode, title, byArtist, albumName, datePublished, durationMs, contributors, _mbid } }, ...]` | -| `volume__taxonomies.json` | `[{ id: "taxonomy:", type: "TrackTaxonomy", fields: { trackId, genres, moods, themes, contexts } }, ...]` | -| `volume__edges.json` | `[{ rel: "hasTaxonomy", from: "", to: "taxonomy:" }, ...]` | - -These three files are the raw v3 bundle chunks — Track + TrackTaxonomy node files plus an edge file. Use `src/publish_mb_bundle.ts` (see **End-to-end workflow** below) to register schemas and publish them to Fangorn, then run `quickbeam build --bundle` to embed. - -### OpenStreetMap changesets - -Fetches recent changesets within a bounding box from the public OSM API. Demonstrates that adding a new domain is a schema change, not an architecture change — the same ingest server handles OSM data automatically via role inference (title←comment, subtitle←user_id, spatial←bbox, etc.). - -```sh -# Edit BBOX, TARGET_COUNT, DAYS_BACK in quickbeam/pipelines/osm.py first, then: -quickbeam data osm -``` - -Outputs `stage_volumes/osm_changesets.json`. +[`build_place.py`](build_place.py) wraps geocode → scrape → graph → embed → bake → demo into one interactive session; see [`docs/RUNNING_SCRIPT.md`](docs/RUNNING_SCRIPT.md). --- -## End-to-end workflow (MusicBrainz → Fangorn → Qdrant) +## Examples -### Step 1 — Generate the bundle chunk files +Each is a full publisher + app built on the Quickbeam toolchain: -```sh -quickbeam data mb --volume 1 --target-count 50000 --output-dir ./data -# produces: data/volume_1_tracks.json -# data/volume_1_taxonomies.json -# data/volume_1_edges.json -``` +- [`examples/audius`](examples/audius/audius-build/RUNBOOK.md): two sovereign publishers + fused by a linkset, searched client-side in the browser +- [`examples/surgext`](examples/surgext/manual/README.md): the Surge XT manual as a + searchable graph +- [`examples/sherwood`](examples/sherwood/example-robinhood-source/README.md): a + `Source` implementation to copy +- [`examples/places`](examples/places/README.md): local discovery over Places + Events -### Step 2 — Register schemas and publish to Fangorn +--- -> **Publishing is moving to the git-native flow.** The `publish_*.ts` scripts below still -> work, but they use the older raw-manifest publish path — no commit history, no -> structural sharing, no embed contract. The target is `fangorn commit --bundle/--view` -> + `fangorn push` (the same primitives record-set repos already use today), with the -> dataset-shaping/sharding half of these scripts folding into `quickbeam data publish`. -> See [`docs/NEW_QUICKSTART.md`](docs/NEW_QUICKSTART.md) for the flow and what's live vs. -> planned. +## Deployment -`src/publish_mb_bundle.ts` must be placed in the fangorn-sdk `src/` directory alongside `setup-embeddings-testdata.ts` (it imports `TestBed` and the SDK type system from there). +Refer to [`DOCKER-README.md`](DOCKER-README.md) for the full guide. ```sh -# from the fangorn-sdk root: -cp /path/to/quickbeam/embeddings/src/publish_mb_bundle.ts src/ - -pnpm dotenvx run -f .env -- tsx src/publish_mb_bundle.ts \ - --tracks /path/to/data/volume_1_tracks.json \ - --taxonomies /path/to/data/volume_1_taxonomies.json \ - --edges /path/to/data/volume_1_edges.json \ - --dataset ds.mb.v1 +./deploy.sh [--env] [--fresh] [--dry-run] # build, push to Artifact Registry, pull on the box +./deploy-sources.sh [path/to/sources.json] # deploy with a STATIC watch list instead ``` -On first run this registers three schemas (all idempotent — safe to re-run): - -| Schema | Name | Description | -|---|---|---| -| Track | `fangorn.mb.track.v1` | Invariant metadata per recording | -| TrackTaxonomy | `fangorn.mb.track.taxonomy.v1` | Genre / mood tags | -| Bundle | `fangorn.mb.bundle.v1` | Track `—hasTaxonomy→` TrackTaxonomy | - -Large volumes are published in batches (`--batch-size`, default 2000). Progress is saved to `tmp/mb-publish-ledger.json` — re-run the same command to resume after a failure. - -When done the script prints the bundle name and ID: - -``` - bundle name : fangorn.mb.bundle.v1 - bundle id : 0xabc123... -``` +`--fresh` drops the qdrant and data volumes and re-embeds everything from chain. -### Step 3 — Build embeddings +## Run Tests ```sh -quickbeam build \ - --bundle "fangorn.mb.bundle.v1=0xabc123..." \ - --root-profile track \ - --graph-api-key \ - --ipfs-gateway https://your-gateway.mypinata.cloud/ipfs \ - --ipfs-gateway-key \ - --dim 256 \ - --umap \ - --reset -``` - ---- - -## Configuration reference - -All config is via CLI flags. Run `quickbeam build --help` or `quickbeam serve --help` for the full list. - -### `quickbeam build` - -| Flag | Default | Description | -|---|---|---| -| `--bundle` | | `NAME=0x...` bundle schema — walks one publisher's typed graph. | -| `--view` | | `NAME=0x...` composed view — fuses several publishers' bundles into one graph before projecting. Mutually exclusive with `--bundle`. | -| `--root-profile` | required | Named projection to emit, repeatable (see `ROOT_PROFILES`). e.g. `--root-profile track` | -| `--profiles-file` | | JSON file of custom/override root profiles, merged over the built-ins | -| `--max-depth` | `2` | Graph-walk depth for profiles that don't set one | -| `--subgraph-url` | Fangorn studio URL | The Graph subgraph endpoint | -| `--graph-api-key` | `""` | The Graph gateway API key | -| `--ipfs-gateway` | `https://gateway.pinata.cloud/ipfs` | IPFS gateway | -| `--qdrant-host` | `localhost` | Qdrant host | -| `--qdrant-port` | `6333` | Qdrant HTTP port | -| `--qdrant-grpc-port` | `6334` | Qdrant gRPC port | -| `--collection` | `quickbeam` | Qdrant collection name | -| `--checkpoint-file` | `./db/ingest_checkpoint.json` | Resume state file | -| `--embedding-model` | `nomic-ai/nomic-embed-text-v1.5` | fastembed model name | -| `--dim` | `256` | Matryoshka output dimensions: 256, 512, or 768 | -| `--embed-batch` | `16` | GPU embed batch size — lower for small VRAM | -| `--searchable-fields` | `auto` | Comma-separated field allowlist, or `auto` | -| `--page-size` | `100` | Subgraph pagination page size | -| `--ipfs-timeout` | `20` | IPFS request timeout in seconds | -| `--concurrency` | `16` | Max concurrent IPFS fetches | -| `--umap` | `false` | Compute and store UMAP px/py after ingest | -| `--umap-only` | `false` | Skip ingest; only (re)compute UMAP on existing collection | -| `--umap-neighbors` | `15` | UMAP n_neighbors parameter | -| `--umap-min-dist` | `0.05` | UMAP min_dist parameter | -| `--reset` | `false` | Delete and recreate the Qdrant collection on startup | - -### `quickbeam watch` - -| Flag | Default | Description | -|---|---|---| -| `--source` | required | `PUBLISHER:SUBSPACE` to watch, repeatable. `*` on either side widens to the app-level filter (`0x..:*`, `*:docs`, `*:*`) | -| `--app` | local client's app | App (global namespace) to watch — a name or a 32-byte app id | -| `--fangorn-bin` | `fangorn` | How to invoke the fangorn CLI — may be a full command, shell-split | -| `--from-block` | | Replay each source's commits from this block before going live | -| `--from-start` | `false` | Replay from genesis (ignored if `--from-block` is set); needs a private RPC | -| `--root-profile` | auto | Named projection to emit, repeatable. Omitted: one profile per vertex tag actually present | -| `--profiles-file` | | JSON file of custom/override root profiles | -| `--max-depth` | `1` | Graph-walk depth per profile | -| `--poll-interval` | `60` | Reconnect backoff in seconds — the watch itself is push-based | -| `--seed-timeout` | `180` | Max seconds for the startup `fangorn read` seed before going live without it | -| `--cdn-dir` | | Baked CDN directory — enables live delta delivery when set | -| `--cdn-domain` | | Domain to append new records to (baked on first run if missing) | -| `--cdn-config` | `domains.json` | Domain config used to resolve the append scan filter | -| `--qdrant-host` | `localhost` | Qdrant host | -| `--qdrant-port` | `6333` | Qdrant HTTP port | -| `--qdrant-grpc-port` | `6334` | Qdrant gRPC port | -| `--collection` | `fangorn` | Qdrant collection name | -| `--checkpoint-file` | `./db/ingest_checkpoint.json` | Shared with `build` — tracks the projected vertex set per `publisher:subspace` | -| `--embedding-model` | `nomic-ai/nomic-embed-text-v1.5` | fastembed model name | -| `--dim` | `256` | Matryoshka output dimensions | -| `--embed-batch` | `16` | GPU embed batch size | -| `--role-map-file` | `./db/role_map.json` | Role map path — loaded if present, inferred on first real batch otherwise | -| `--searchable-fields` | `auto` | Field allowlist or `auto` | -| `--label-cap` | `50` | Max folded labels per group | -| `--node-cap` | `2000` | Max nodes visited per root | - -### `quickbeam serve` - -| Flag | Default | Description | -|---|---|---| -| `--schema` / `-s` | | `NAME=0x...` schema ID pair. Repeatable. | -| `--primary` / `-p` | first schema | Join key schema | -| `--subgraph-url` | Fangorn studio URL | The Graph endpoint | -| `--graph-api-key` | `""` | The Graph gateway API key | -| `--ipfs-gateway` | `https://gateway.pinata.cloud/ipfs` | IPFS gateway | -| `--qdrant-url` | `None` | Qdrant Cloud URL — overrides `--qdrant-host`/`--qdrant-port` | -| `--qdrant-api-key` | `None` | Qdrant Cloud API key | -| `--qdrant-host` | `localhost` | Qdrant host (local) | -| `--qdrant-port` | `6333` | Qdrant HTTP port | -| `--qdrant-grpc-port` | `6334` | Qdrant gRPC port | -| `--collection` | `quickbeam` | Qdrant collection name | -| `--embedding-model` | `nomic-ai/nomic-embed-text-v1.5` | Must match the builder | -| `--bundle-cid` | `None` | IPFS CID of an NDJSON bundle to seed from on first startup | -| `--searchable-fields` | `auto` | Field allowlist or `auto` | -| `--host` | `0.0.0.0` | Bind host | -| `--port` | `8080` | Bind port | -| `--reset` | `false` | Drop and recreate collection on startup | -| `--x402-pay-to` | `None` | Recipient address. Enables x402 gating on the search routes when set. | -| `--x402-price` | `0.001` | Price per gated request in whole token units | -| `--x402-network` | `base-sepolia` | EVM network: `base-sepolia`, `base`, `avalanche-fuji` | -| `--x402-asset` | network USDC | Token contract address | -| `--x402-decimals` | `6` | Token decimals for the price → atomic conversion | -| `--x402-facilitator` | `None` | Facilitator URL for on-chain verify+settle (omit for local verification) | - -Plus `--watch ` to run the [live daemon alongside the server](#serve--watch-in-one-process). - -### `quickbeam mcp` - -| Flag | Default | Description | -|---|---|---| -| `--cdn-url` | `http://localhost:8090` | Base URL of the Semantic CDN it pulls datasets from | -| `--edges` | `None` | Local linkset JSON file or directory (relational axis), until the CDN delivers edges | -| `--transport` | `http` | `http` (streamable-http), `stdio`, or `sse` | -| `--host` | `0.0.0.0` | Bind host (http/sse) | -| `--port` | `8765` | Bind port (http/sse) | -| `--x402-pay-to` | `None` | Recipient address. Enables per-tool payment (Phase 2) when set. | -| `--x402-price` | `0.001` | Price per gated tool call in whole token units | -| `--x402-network` | `base-sepolia` | EVM network | -| `--x402-asset` | network USDC | Token contract address | -| `--x402-decimals` | `6` | Token decimals | -| `--x402-facilitator` | `None` | Facilitator URL (omit for local verification) | - -Env equivalents: `QUICKBEAM_CDN_URL`, `QUICKBEAM_EDGES`. - -### `quickbeam export` - -| Flag | Default | Description | -|---|---|---| -| `--src` | required | Source server URL, e.g. `http://localhost:8080` | -| `--out` | `bundle.ndjson` | Output file path | -| `--owner` | `None` | Filter export to a single owner address | -| `--embeddings-only` | `false` | Export only `track_id` + `embedding`, omit fields and metadata | - -### `quickbeam cdn bake` - -| Flag | Default | Description | -|---|---|---| -| `--config` | `domains.json` | Operator domain config: `name → { description, filter }` | -| `--cdn-dir` | `./cdn` | Output directory for baked shards | -| `--collection` | `fangorn` | Source Qdrant collection to bake from | -| `--domain` | all | Bake only this one domain from the config | -| `--shard-size` | `50000` | Points per shard file | -| `--limit` | `0` | Cap total points baked per domain (0 = all). Use a small value for a lightweight in-browser snapshot. | -| `--scroll-batch` | `2000` | Qdrant scroll page size | -| `--embedding-model` | `nomic-ai/nomic-embed-text-v1.5` | Recorded in the manifest (Qdrant doesn't store it) | -| `--qdrant-url` / `--qdrant-api-key` | `None` | Qdrant Cloud (overrides host/port) | -| `--qdrant-host` / `--qdrant-port` / `--qdrant-grpc-port` | `localhost`/`6333`/`6334` | Local Qdrant | - -A domain's `filter` accepts `entityType: [...]` and `owner: [...]` (each a `MatchAny`); -multiple keys are AND-ed. An empty/missing filter selects the whole collection. - -### `quickbeam cdn edges` - -| Flag | Default | Description | -|---|---|---| -| `--cdn-dir` | `./cdn` | Baked CDN directory (the domain must already be baked) | -| `--domain` | required | Domain to attach the linkset to | -| `--source` | required | Linkset JSON — a list of `{rel, from, to, fromType, toType}` edges, or `{edges:[...]}` | - -Installs the relational axis as `cdn//edges.json` (served at -`/domains/{name}/edges`) and records the edge count + relation types in the catalog. - -### `quickbeam cdn serve` - -| Flag | Default | Description | -|---|---|---| -| `--cdn-dir` | `./cdn` | Directory of baked shards to serve | -| `--host` | `0.0.0.0` | Bind host | -| `--port` | `8090` | Bind port | -| `--cors` | `false` | Enable permissive CORS (for browser-based pulls) | - -### `quickbeam pull` - -| Flag | Default | Description | -|---|---|---| -| `domain` | required | Positional — domain name to pull (see the CDN's `/catalog`) | -| `--cdn-url` | `http://localhost:8090` | Base URL of the Semantic CDN | -| `--collection` | domain name | Local Qdrant collection to load into | -| `--cache-dir` | `./db/cdn_cache` | Where downloaded shards are cached | -| `--concurrency` | `4` | Parallel shard downloads | -| `--batch` | `500` | Upsert batch size | -| `--reset` | `false` | Recreate the local collection before loading | -| `--download-only` | `false` | Fetch + verify shards but don't load into Qdrant | -| `--qdrant-url` / `--qdrant-api-key` / `--qdrant-host` / `--qdrant-port` / `--qdrant-grpc-port` | local | Target Qdrant for the local collection | - ---- - -## API - -All endpoints return JSON. Hits are shaped as `{ id, fields, owner, meta, score?, embedding? }`, where `meta` carries on-chain provenance `{ manifestCid, blockTimestamp, version, owner }`. - -> When `--x402-pay-to` is set, `/search`, `/search/vector`, and `/search/text` require an `X-PAYMENT` header — see [x402 payment gating](#x402-payment-gating). - -> **Scoping.** The three search endpoints take `owner`, `namespace`, and the -> repeatable `scope=OWNER:NAMESPACE`. One collection holds every watched namespace -> (see [Deployment](#deployment)) — each point carries `owner` at the payload top -> level and `meta.namespace` nested — and embeddings are **not** duplicated per -> caller, so a caller's slice of the corpus is a filter. -> -> Several `scope` pairs OR together, each pair AND-ing its own halves, which is what -> lets one endpoint span several namespaces: -> `?scope=0xA:tracks&scope=0xB:reviews`. Either half may be empty (`0xA:`, `:tracks`) -> to leave it unconstrained. `scope` takes precedence over `owner`/`namespace`. -> -> `/browse` is **not** scoped: it returns the whole collection. - -### `GET /browse` -Paginated browse. `?limit=20&offset=0` - -### `GET /search` -Semantic search by text — embeds the query server-side. -`?q=late+night+driving&n_results=10&scope=0xA:robinhood&scope=0xB:music` - -### `POST /search/vector` -Query by raw embedding vector. -```json -{ "embedding": [...], "n_results": 20, "scope": ["0xA:robinhood", "0xB:music"] } -``` - -### `POST /search/text` -Lexical search over an in-memory index of title, subtitle, and tag fields. Faster than semantic search for exact name lookups. -```json -{ "q": "arctic monkeys", "limit": 20, "scope": ["0xA:robinhood"] } -``` - -### `POST /embed` -Embed text using the same model as ingestion — keeps client and server embedding spaces aligned. -```json -{ "text": "late night melancholic indie" } → { "embedding": [...] } -{ "texts": ["track one", "track two"] } → { "embeddings": [[...], [...]] } -``` - -### `GET /schema` -Inferred semantic role map (`title`, `subtitle`, `tags`, etc.) and facet vocabularies for the active dataset. - -### `GET /catalog/map` -2D UMAP projection of the full collection for a galaxy/map view. Computed on first request and cached. Returns `{ "computing": true }` while still running. - -### `POST /catalog/map/refresh` -Invalidates the map cache and recomputes in the background. - -### `GET /bundle/export` -Streams the full collection as NDJSON — one point per line. `?owner=0x...&limit=1000&offset=0` - -### `POST /bundle/import` -Streaming NDJSON import — reads line by line, upserts in batches of 500. - -### `POST /bundle/upsert` -JSON body upsert for pre-embedded points (smaller programmatic use). - -### `GET /health` -Collection count, schema map, role map, cache state, checkpoint info. - -### `POST /reingest` -Triggers a background re-ingestion from the subgraph. Only re-embeds changed documents. - -### `POST /reingest/full` -Clears the checkpoint and re-ingests everything from scratch. Does not drop the Qdrant collection. - -### `GET /debug` -Join diagnostics — matched/unmatched track IDs across primary and secondary schemas. - ---- - -## Join semantics - -### Graph projection - -- One record is emitted per node whose `type` matches a `--root-profile`'s `root_type`; the root node's stable, publisher-assigned `id` (or its Entity URI in a view) is the join key. -- The profile walks the **undirected** graph up to `max_depth` hops from the root, folding the neighbor types it lists in `include` into grouped, deduped, capped label lists. -- Nodes not reachable from any root contribute no fields. -- Manifests that are not valid bundles (missing `kind: "bundle"` or `edgeChunks`) are skipped. - -The semantic role map (`title`, `subtitle`, `tags`, `spatial`, etc.) is inferred automatically from field names and value shapes across the merged dataset. This is what makes the same server and app work for music tracks, OSM changesets, or any other Fangorn schema without per-domain configuration. +pytest tests -q +``` + +## Layout + +``` +quickbeam/ + cli.py typer entry point; every command passes through to its own argparse + ingest/ the shared ingestion engine (build + watch) + build.py `quickbeam build` + embed.py fastembed engine (GPU-OOM resilient), doc text, Qdrant indexes + identity.py deterministic point ids + the matryoshka transform + checkpoint.py resumable state + umap.py 2-D projection → catalog map + graph/projection.py root profiles, the graph walk, join helpers + sources/fangorn.py `fangorn read` / `fangorn subscribe` bridge + scrapers/ the Source contract + the ingestion harness + watcher.py `quickbeam watch` + server.py `quickbeam serve` (FastAPI) + cdn.py `quickbeam cdn *` — bake / append / edges / precompute / index / serve + index.py codebook, quantization, the privacy/recall harness + pull.py `quickbeam pull` + mcp_server.py `quickbeam mcp` + x402.py HTTP 402 payments, both sides + roles.py schema-agnostic semantic role inference + objects.py the git-native object model (commit/tree/blob), Python side + publish.py the `Publisher` façade + pipelines/ built-in ETL sources +``` + +## Gotchas + +- **`--poll-interval` is a reconnect backoff, not a monitoring interval.** `watch` is push-based off `fangorn subscribe`. The watch-list refresh is `--sources-refresh`. +- **`--app` must match whatever published the data.** A mismatched app resolves an empty namespace with no error anywhere. +- **`PINATA_GATEWAY` is a bare host**. The SDK appends `/ipfs/` itself. +- **`ETH_PRIVATE_KEY` is required even for read-only use.** Don't bake a `~/.fangorn/config.json` into an image since it will short circuit every env var. +- **A domain with no `namespace` filter bakes every namespace in the collection.** +- The MCP endpoint is `/mcp`. From dc6688088ce5263c3b38721fd00f2664ae5ab912 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Tue, 25 Aug 2026 17:31:05 -0500 Subject: [PATCH 5/8] Fix formatting issues --- DOCKER-README.md | 3 +-- README.md | 15 +++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/DOCKER-README.md b/DOCKER-README.md index 3a92786..4f73680 100644 --- a/DOCKER-README.md +++ b/DOCKER-README.md @@ -91,8 +91,7 @@ docker compose up -d --build For a static list, write `sources.json` into the shared volume and set `SOURCES_URL=file:///data/sources.json`. An example `sources.json`: ```json -[ {"app": "fangorn", "owner": "0x147c24c5Ea2f1EE1ac42AD16820De23bBba45Ef6", - "namespace": "robinhood"} ] +[ {"app": "fangorn", "owner": "0x147c24c5Ea2f1EE1ac42AD16820De23bBba45Ef6", "namespace": "robinhood"} ] ``` The list also accepts `"APP:OWNER:NAMESPACE"` strings, the older `"OWNER:NAMESPACE"` form (which takes the `APP` env var as its app), and either form wrapped in `{"sources": …}` (the shape the worker's `/watchlist` returns). `*` on owner or namespace widens the subscription to the app level. diff --git a/README.md b/README.md index 5953315..ec6b49b 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,9 @@ cp .env.example .env # set PINATA_GATEWAY, ETH_PRIVATE_KEY, QDRANT_API_KEY, docker compose up -d --build ``` -Currently, there are four services contained in one image: `qdrant`, `watch`, `serve` (:8080), `cdn` (:8090), `mcp` (:8765). Namespaces are **not** configured in compose. The `watch` command polls the `SOURCES_URL` for its watch list and starts or cancels a stream per namespace without requiring a restart. Full deployment guide: **[`DOCKER-README.md`](DOCKER-README.md)**. +Currently, there are four services contained in one image: `qdrant`, `watch`, `serve` (:8080), `cdn` (:8090), `mcp` (:8765). Namespaces are **not** configured in compose. The `watch` command polls the `SOURCES_URL` for its watch list and starts or cancels a stream per namespace without requiring a restart. The full deployment guide can be found in the **[`DOCKER-README.md`](DOCKER-README.md)**. -### Or run the pieces by hand +### Run by hand ```sh # 1. Qdrant @@ -77,7 +77,7 @@ quickbeam watch --app fangorn --source 0x147c...:robinhood --cdn-dir ./cdn How the app portion is supplied depends on how sources are given. -**Static `--source` flags** — `build`, `watch` and `serve` take `OWNER:NAMESPACE` (repeatable, **two parts**). All of them run under the single `--app`, so one process covers one app. `*` on either side widens to the whole app: +**Static `--source` flags**: `build`, `watch` and `serve` take `OWNER:NAMESPACE` (repeatable, **two parts**). All of them run under the single `--app`, so one process covers one app. `*` on either side widens to the whole app: ```sh --source 0x147c...:robinhood # one publisher, one subspace @@ -86,18 +86,18 @@ How the app portion is supplied depends on how sources are given. --source '*:*' # the whole app ``` -**A watch list — `watch --sources-url`** (what the compose deployment uses). Here **each entry carries its own app**, so one instance serves several. For example: +**A watch list: `watch --sources-url`** (what the compose deployment uses). Here **each entry carries its own app**, so one instance serves several. For example: ```json ["app1Id::", "app2Id::", "fangorn:0x147c...:robinhood"] ``` -Here, we are watching all of app1, app2, but only a specific namespace for a specific publisher in the fangorn app. +In the example above, we are watching all of app1, app2, but only a specific namespace for a specific publisher in the fangorn app. Entries are `"APP:OWNER:NAMESPACE"` strings or `{app, owner, namespace}` objects, and an empty or `*` part is a wildcard. So `app1Id::` means *everything in app1*. `--app` (the `APP` env var in compose) is only the **fallback** for an entry naming no app. An entry with no app and no fallback is **dropped** because reading the wrong app silently indexes the wrong graph. > Warning: **Don't pass the three-part form to `--source`.** It splits on the first colon only, so `--source fangorn:0xA:docs` silently becomes owner `fangorn`, namespace `0xA:docs` with no error and nothing watched. -> Note `--source` means something different on two other commands: `cdn edges --source` is a **path** to a linkset JSON, and `data events-fetch --source` names a **scraper** (`eventbrite` / `eventbrite-location` / `tribe`). Check the `--help` flag for the command you're running. +> Note: `--source` means something different on two other commands: `cdn edges --source` is a **path** to a linkset JSON, and `data events-fetch --source` names a **scraper** (`eventbrite` / `eventbrite-location` / `tribe`). Check the `--help` flag for the command you're running. A wildcard source has no single namespace to seed with `fangorn read`, so it only sees namespaces published before it started if you also pass `--from-block N`. @@ -118,8 +118,7 @@ Every watched namespace embeds into **one** Qdrant collection. Points carry `own A profile walks the graph from every vertex carrying a given tag and folds its neighbors into one document. With no `--root-profile`, one profile is auto-derived per distinct vertex tag present in the source. Override with `--profiles-file` (see [`quickbeam/profiles.example.json`](quickbeam/profiles.example.json)): ```json -{ "file": { "root_type": "File", "max_depth": 2, "include": ["File"], - "content_fields": ["filename", "text"] } } +{ "file": { "root_type": "File", "max_depth": 2, "include": ["File"], "content_fields": ["filename", "text"] } } ``` `--max-depth`, `--label-cap` (max folded labels per relation group) and `--node-cap` (max nodes visited per root) bound the walk. From 5ab3c33bb7b7874f9f9c4ace4c6012fb10078679 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Tue, 25 Aug 2026 18:58:08 -0500 Subject: [PATCH 6/8] Fix grammar and update service count in README Corrected grammatical errors and updated service count in README. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ec6b49b..ad29f4f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # quickbeam -Semantic search over [Fangorn](https://github.com/fangorn-network/fangorn) knowledge graphs. A publisher versions a graph offchain and anchors it onchain; quickbeam reads that graph with the `fangorn` light client, embeds it into Qdrant, and serves it via three means: an HTTP search API, a static "Semantic CDN" of downloadable shards, and an MCP server for agents. +Semantic search over [Fangorn](https://github.com/fangorn-network/fangorn) knowledge graphs. A publisher versions a graph offchain and anchors it onchain. Quickbeam reads that graph with the `fangorn` light client, embeds it into Qdrant, and serves it via three means: an HTTP search API, a static "Semantic CDN" of downloadable shards, and an MCP server for agents. --- @@ -45,7 +45,7 @@ cp .env.example .env # set PINATA_GATEWAY, ETH_PRIVATE_KEY, QDRANT_API_KEY, docker compose up -d --build ``` -Currently, there are four services contained in one image: `qdrant`, `watch`, `serve` (:8080), `cdn` (:8090), `mcp` (:8765). Namespaces are **not** configured in compose. The `watch` command polls the `SOURCES_URL` for its watch list and starts or cancels a stream per namespace without requiring a restart. The full deployment guide can be found in the **[`DOCKER-README.md`](DOCKER-README.md)**. +Currently, there are five services contained in one image: `qdrant`, `watch`, `serve` (:8080), `cdn` (:8090), `mcp` (:8765). Namespaces are **not** configured in compose. The `watch` command polls the `SOURCES_URL` for its watch list and starts or cancels a stream per namespace without requiring a restart. The full deployment guide can be found in the **[`DOCKER-README.md`](DOCKER-README.md)**. ### Run by hand @@ -54,7 +54,7 @@ Currently, there are four services contained in one image: `qdrant`, `watch`, `s docker run -d -p 6333:6333 -p 6334:6334 -v "$(pwd)/db/qdrant:/qdrant/storage" qdrant/qdrant # 2. Embed a namespace (one shot) -quickbeam build --source 0x147c24c5...:robinhood --root-profile asset +quickbeam build --source 0x147c24c5...:robinhood --root-profile asset # 3. Serve it quickbeam serve --port 8080 From d7390175b11745ecca8e049ccfa46f6d7b367853 Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Tue, 25 Aug 2026 19:00:06 -0500 Subject: [PATCH 7/8] Update README for ETH_PRIVATE_KEY and PINATA_GATEWAY Chang ISP provider to just ISP --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ad29f4f..4b93e17 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ pip install -e ".[dev]" # + pytest ``` Requires Python ≥3.12 and the [`fangorn` CLI](https://github.com/fangorn-network/fangorn) on `PATH` (or pass `--fangorn-bin`). The fangorn CLI refuses to start without -`ETH_PRIVATE_KEY`, but using a throwaway key is fine if you are not writing onchain. We recommend setting `PINATA_GATEWAY` too since the default `ipfs.io` is not guaranteed to serve content (ISP provider specific). +`ETH_PRIVATE_KEY`, but using a throwaway key is fine if you are not writing onchain. We recommend setting `PINATA_GATEWAY` too since the default `ipfs.io` is not guaranteed to serve content (ISP specific). ## Commands From 0a3da85ec1152a2afc9064d27aae709938c9347b Mon Sep 17 00:00:00 2001 From: Coleman Irby Date: Wed, 26 Aug 2026 08:29:46 -0500 Subject: [PATCH 8/8] Removed ponytail comments --- .github/workflows/tests.yml | 2 +- deploy.sh | 2 +- examples/audius/audius-demo-large/src/lib/kernel.tsx | 2 +- examples/audius/audius-demo/src/lib/kernel.tsx | 2 +- quickbeam/cdn.py | 2 +- quickbeam/index.py | 4 ++-- quickbeam/watcher.py | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 642a4ff..ae6f1d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,6 +15,6 @@ jobs: cache: pip cache-dependency-path: pyproject.toml - run: pip install -e ".[cpu,dev]" - # ponytail: tests/ only — quickbeam/test_roles.py is a sys.exit self-check + # tests/ only — quickbeam/test_roles.py is a sys.exit self-check # that kills collection if pytest walks the package dir. - run: pytest tests -q diff --git a/deploy.sh b/deploy.sh index f4928cc..9d19e02 100755 --- a/deploy.sh +++ b/deploy.sh @@ -81,7 +81,7 @@ FRESH_CMD="" if [[ $FRESH -eq 1 ]]; then # Checked on the BOX, not here, because .env only ships with --env — the local copy # is not what the watcher reads. Aborts before the wipe, never after. - # ponytail: FROM_BLOCK is the only rebuild path while the watch list is wildcard + # FROM_BLOCK is the only rebuild path while the watch list is wildcard # (`*:*`) sources — those get no startup seed read, so their pairs are only # discovered by replaying StateCommitted history. Relax this to a warning if pinned # `owner:namespace` entries ever come back, since those DO seed themselves. diff --git a/examples/audius/audius-demo-large/src/lib/kernel.tsx b/examples/audius/audius-demo-large/src/lib/kernel.tsx index ba22474..c89505b 100644 --- a/examples/audius/audius-demo-large/src/lib/kernel.tsx +++ b/examples/audius/audius-demo-large/src/lib/kernel.tsx @@ -52,7 +52,7 @@ function load(): Saved | null { } function save(s: Saved) { - // ponytail: synchronous write per signal (~110 KB at skip_window=20). Well under + // synchronous write per signal (~110 KB at skip_window=20). Well under // a frame and far under the 5 MB quota; debounce only if a profile says so. try { localStorage.setItem(KEY, JSON.stringify(s)); } catch { /* quota / private mode */ } } diff --git a/examples/audius/audius-demo/src/lib/kernel.tsx b/examples/audius/audius-demo/src/lib/kernel.tsx index ba22474..c89505b 100644 --- a/examples/audius/audius-demo/src/lib/kernel.tsx +++ b/examples/audius/audius-demo/src/lib/kernel.tsx @@ -52,7 +52,7 @@ function load(): Saved | null { } function save(s: Saved) { - // ponytail: synchronous write per signal (~110 KB at skip_window=20). Well under + // synchronous write per signal (~110 KB at skip_window=20). Well under // a frame and far under the 5 MB quota; debounce only if a profile says so. try { localStorage.setItem(KEY, JSON.stringify(s)); } catch { /* quota / private mode */ } } diff --git a/quickbeam/cdn.py b/quickbeam/cdn.py index 871b1de..b6cb630 100644 --- a/quickbeam/cdn.py +++ b/quickbeam/cdn.py @@ -1268,7 +1268,7 @@ def catalog(): # The watcher writes shards from a DIFFERENT container (shared /data volume), so # there is no in-process signal to hook. This polls each watched manifest instead # and diffs its shard list. - # ponytail: mtime-gated polling, not inotify — one small stat per domain per + # mtime-gated polling, not inotify — one small stat per domain per # connection per tick, against a handful of domains and a handful of clients. # Reach for watchfiles only if that ever shows up in a profile. _POLL_SECONDS = 2.0 diff --git a/quickbeam/index.py b/quickbeam/index.py index 9b71786..a67724d 100644 --- a/quickbeam/index.py +++ b/quickbeam/index.py @@ -34,7 +34,7 @@ WHAT IS DELIBERATELY NOT HERE ----------------------------- -ponytail: the codebook is FLAT, not the two-level tree the design calls for. The tree +The codebook is FLAT, not the two-level tree the design calls for. The tree is a client-side routing accelerator — it finds the same nearest centroid, just in ~512 comparisons instead of K — so it cannot change any number this harness reports. Fit it in Stage D from the flat codebook, when there is a client that cares. @@ -134,7 +134,7 @@ def spherical_kmeans(X, k: int, iters: int = 25, seed: int = 0, chunk: int = 100 transitively under umap-learn), and the balanced assignment below is something sklearn cannot do anyway, so both halves are written out. - ponytail: full-batch, chunked. At 500k x 4096 that is ~10s/iteration, fine for + Full-batch, chunked. At 500k x 4096 that is ~10s/iteration, fine for the gate. If this is still the fit path at 20M, switch to minibatch (sample a batch per iteration, same update rule) — the ceiling is wall-clock, not quality. """ diff --git a/quickbeam/watcher.py b/quickbeam/watcher.py index b34e62c..1c5608c 100644 --- a/quickbeam/watcher.py +++ b/quickbeam/watcher.py @@ -706,7 +706,7 @@ def _fetch_sources(url: str, default_app: str | None = None) -> set: guessed: reads resolve against the app, so watching the wrong one silently indexes the wrong graph, which is worse than watching nothing. - ponytail: urllib in a thread rather than adding an async HTTP client for one + urllib in a thread rather than adding an async HTTP client for one poll every --sources-refresh seconds. The User-Agent is NOT cosmetic: the registry worker sits behind Cloudflare, whose @@ -744,7 +744,7 @@ def _make_qdrant(args) -> QdrantClient: """Qdrant connection: a remote URL (with optional API key) when given, else the local host/port pair. Mirrors pull.py's helper of the same name. - ponytail: the URL branch does NOT set prefer_grpc. A remote Qdrant is typically + The URL branch does NOT set prefer_grpc. A remote Qdrant is typically reached through an HTTPS reverse proxy on 443, which does not carry gRPC on 6334 — forcing gRPC there fails to connect. REST is slower per upload and correct everywhere; switch it on if a deployment actually exposes gRPC.