Feat/bypass ww

.env.example

@@ -14,6 +14,13 @@
 SOURCES_URL=https://quickbeam-registry.<subdomain>.workers.dev/watchlist
 SOURCES_REFRESH=60
 
+# Replay from this block before going live. Only meaningful for a WILDCARD source
+# (`APP::` / `*:*`), which cannot be seeded with `fangorn read` and so would otherwise
+# see nothing published before the box started. 0 (the default) means no replay.
+# Cost: 1000 blocks per eth_getLogs, so pick a block just before the commits you want —
+# a million blocks back is a thousand sequential calls on the public RPC.
+FROM_BLOCK=0
+
 # Fallback app for a watch-list entry that names none. Every entry the worker serves
 # carries its own app, so this only covers a hand-written list — but keep it equal to
 # the worker's DEFAULT_APP, which is what it stamps on a view created without one.

.github/workflows/tests.yml

@@ -0,0 +1,20 @@
+name: tests
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+      - run: pip install -e ".[cpu,dev]"
+      # tests/ only — quickbeam/test_roles.py is a sys.exit self-check
+      # that kills collection if pytest walks the package dir.
+      - run: pytest tests -q

Dockerfile

@@ -19,7 +19,11 @@ RUN apt-get update \
  && rm -rf /var/lib/apt/lists/*
 
 # Provides the `fangorn` binary on PATH — the default --fangorn-bin.
-RUN npm i -g @fangorn-network/sdk && npm cache clean --force
+# PIN the version: the DataRegistry address rides inside the SDK's config.js, so an
+# unpinned install baked into a cached layer silently keeps reading a retired registry
+# (an image built 2026-08-13 was still on 0x9dfa…572c and saw none of the state
+# published to 0x97d6…df91). Bump this when the registry moves.
+RUN npm i -g @fangorn-network/sdk@2026.8.18-dev && npm cache clean --force
 
 WORKDIR /app
 COPY pyproject.toml ./

deploy-sources.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Deploy quickbeam to the GCE box with a STATIC watch list: the JSON file in
+# data/ instead of the registry worker's /watchlist.
+#
+#   ./deploy-sources.sh [--env] [--dry-run] [path/to/sources.json]
+#
+# The image half is deploy.sh's job and is not duplicated here; this only adds
+# the watch list. The file goes INTO the shared `data` volume via
+# `docker compose cp`, not onto the box's disk — /data in the container IS that
+# volume, so a copy left in the home directory is invisible to `watch`.
+set -euo pipefail
+cd "$(dirname "$0")"
+
+INSTANCE=${INSTANCE:-quickbeam-1}
+ZONE=${ZONE:-us-east4-a}
+
+SOURCES=data/sources.json
+args=()
+for a in "$@"; do
+  case "$a" in
+    --env|--dry-run) args+=("$a") ;;
+    -*) echo "usage: $0 [--env] [--dry-run] [sources.json]" >&2; exit 2 ;;
+    *) SOURCES=$a ;;
+  esac
+done
+DRY=0; [[ " ${args[*]} " == *" --dry-run "* ]] && DRY=1
+run() { if [[ $DRY -eq 1 ]]; then echo "  + $*"; else "$@"; fi; }
+
+[[ -f $SOURCES ]] || { echo "no watch list at $SOURCES" >&2; exit 1; }
+# Validate before shipping: nothing downstream treats a bad list as an error.
+# _fetch_sources silently skips an entry it cannot parse, and DROPS one that
+# names no app — so a typo here comes back as a box that watches nothing and
+# says so exactly once, in a log line nobody reads.
+python3 - "$SOURCES" <<'PY'
+import json, sys
+items = json.load(open(sys.argv[1]))
+items = items.get("sources", []) if isinstance(items, dict) else items
+if not items:
+    sys.exit("watch list is empty")
+bad = [i for i in items
+       if not ((isinstance(i, str) and len(i.split(":")) == 3)
+               or (isinstance(i, dict) and i.get("app")))]
+if bad:
+    sys.exit(f"entries must name an app (APP:OWNER:NAMESPACE, `*` = any): {bad}")
+print(f"==> {len(items)} source(s): " + ", ".join(map(str, items)))
+PY
+
+# The image half, plus the box's SOURCES_URL — WATCHLIST_URL is how this script says
+# "that mode, not yours": deploy.sh writes the line and drops any COMPOSE_FILE overlay,
+# so a box that was on the worker converges here in one run instead of silently staying
+# on the worker's list.
+WATCHLIST_URL=file:///data/sources.json ./deploy.sh "${args[@]}"
+run gcloud compute scp "$SOURCES" "$INSTANCE:~/sources.json" --zone="$ZONE"
+# Into the volume, since /data in the container IS the volume. No restart: `watch`
+# re-reads the file every SOURCES_REFRESH seconds and converges in place — it may log
+# one "sources fetch failed" first, in the window between the container starting and
+# this copy landing.
+run gcloud compute ssh "$INSTANCE" --zone="$ZONE" --command='
+  docker compose cp ~/sources.json watch:/data/sources.json &&
+  docker compose exec -T watch cat /data/sources.json'

deploy.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
-# Deploy the quickbeam image to the shared GCE instance.
-# Run from repo root:  ./deploy.sh [--env] [--dry-run]
+# Deploy the quickbeam image to the shared GCE instance, watching the REGISTRY
+# WORKER's /watchlist — views created from the website are the source of truth.
+# For a static list on the box instead, use ./deploy-sources.sh.
+# Run from repo root:  ./deploy.sh [--env] [--fresh] [--dry-run]
 #
 # Wraps the three-command redeploy in DOCKER-README.md ("Redeploying a code
 # change"): build here, push to Artifact Registry, then tell the box to pull.
@@ -13,15 +15,21 @@ INSTANCE=${INSTANCE:-quickbeam-1}
 ZONE=${ZONE:-us-east4-a}
 
 PUSH_ENV=0
+FRESH=0
 DRY=0
 for a in "$@"; do
   case "$a" in
     # Copy .env too. Off by default: it carries ETH_PRIVATE_KEY and
     # QDRANT_API_KEY, and the box's copy may have been edited in place.
     # Needed on a first deploy, or after rotating a key / changing a port.
     --env)     PUSH_ENV=1 ;;
+    # Deploy as if the box had never run: drop the qdrant and data volumes, so it
+    # comes up with no collection, no ingest checkpoint and no baked CDN shards and
+    # re-embeds everything from chain. Slow (a full --from-block replay + one
+    # `fangorn read` and embed per pair), so it is opt-in, not the default.
+    --fresh)   FRESH=1 ;;
     --dry-run) DRY=1 ;;
-    *) echo "usage: $0 [--env] [--dry-run]" >&2; exit 2 ;;
+    *) echo "usage: $0 [--env] [--fresh] [--dry-run]" >&2; exit 2 ;;
   esac
 done
 
@@ -39,6 +47,49 @@ IMG=$(grep -E '^IMAGE=' .env | tail -1 | cut -d= -f2- || true)
 [[ -n "$IMG" ]] || { echo "IMAGE is unset in .env (compose would fall back to quickbeam:local)" >&2; exit 1; }
 [[ "$IMG" != *"quickbeam:local"* ]] || { echo "IMAGE points at a local build, not Artifact Registry: $IMG" >&2; exit 1; }
 
+# The mode lives in ONE variable: the box's SOURCES_URL. It is rewritten on every
+# deploy rather than assumed, because deploy-sources.sh points it at a file and a box
+# left in that mode ignores the worker silently — it keeps serving whatever list it
+# has and nothing anywhere says why. COMPOSE_FILE goes for the same reason: a
+# standalone overlay pinned there re-applies the file mode under a bare
+# `docker compose up -d`.
+# WATCHLIST_URL, when set, is deploy-sources.sh calling in to reuse the image half of
+# this script while owning the mode itself — so the .env guard below is skipped.
+if [[ -n "${WATCHLIST_URL:-}" ]]; then
+  SOURCES_URL=$WATCHLIST_URL
+else
+  SOURCES_URL=$(grep -E '^SOURCES_URL=' .env | tail -1 | cut -d= -f2- || true)
+  [[ -n "$SOURCES_URL" ]] || { echo "SOURCES_URL is unset in .env (needs the worker's /watchlist URL)" >&2; exit 1; }
+  case "$SOURCES_URL" in
+    file://*) echo "SOURCES_URL in .env is a file:// list — that is ./deploy-sources.sh's job, not this script's" >&2; exit 1 ;;
+    http://*|https://*) ;;
+    # A bare host is what you get from copying the worker's hostname out of the
+    # dashboard. urlopen() rejects it with "unknown url type" on every poll, which the
+    # watcher reports as a fetch failure and then keeps its old list forever.
+    *) SOURCES_URL="https://$SOURCES_URL" ;;
+  esac
+fi
+
+# --fresh's remote half. Three separate stores hold ingest state and ALL of them have
+# to go together: the qdrant volume (the vectors), /data/db/checkpoint.json (whose
+# processed_track_ids + per-source vertex_cids make the watcher skip anything it has
+# already embedded) and /data/cdn (whose manifests make append_domain skip anything
+# already delivered). Wiping only qdrant leaves the other two claiming the work is
+# done, and the watcher then logs "no new records" over an empty collection forever.
+# `down -v` drops both named volumes in one go — see DOCKER-README.md's Teardown.
+FRESH_CMD=""
+if [[ $FRESH -eq 1 ]]; then
+  # Checked on the BOX, not here, because .env only ships with --env — the local copy
+  # is not what the watcher reads. Aborts before the wipe, never after.
+  # FROM_BLOCK is the only rebuild path while the watch list is wildcard
+  # (`*:*`) sources — those get no startup seed read, so their pairs are only
+  # discovered by replaying StateCommitted history. Relax this to a warning if pinned
+  # `owner:namespace` entries ever come back, since those DO seed themselves.
+  FRESH_CMD="grep -qE '^FROM_BLOCK=[1-9]' .env \
+    || { echo 'FROM_BLOCK is 0/unset on the box: a wiped instance would never rebuild, because wildcard sources have no seed read. Set it in .env and redeploy with --env.' >&2; exit 1; } \
+    && docker compose down -v && "
+fi
+
 # Also tag the immutable git sha. `docker compose pull` on a moving :latest
 # cannot tell you which build a box is running; this leaves a tag you can roll
 # back to. -dirty when the tree has uncommitted changes, so the tag never lies.
@@ -47,14 +98,25 @@ IMG=$(grep -E '^IMAGE=' .env | tail -1 | cut -d= -f2- || true)
 # --porcelain (rather than `git diff`) because it also catches an UNTRACKED file
 # under quickbeam/, which `COPY quickbeam ./quickbeam` would bake in regardless.
 SHA=$(git rev-parse --short HEAD)
-[[ -z "$(git status --porcelain -- quickbeam pyproject.toml)" ]] || SHA="$SHA-dirty"
+[[ -z "$(git status --porcelain -- quickbeam pyproject.toml Dockerfile)" ]] || SHA="$SHA-dirty"
 SHA_IMG="${IMG%:*}:$SHA"
 
 echo "==> $IMG"
 echo "==> $SHA_IMG"
 echo "==> $INSTANCE ($ZONE)"
+echo "==> watch list: $SOURCES_URL"
 [[ $DRY -eq 1 ]] && echo "-- dry run, nothing will be built, pushed or restarted --"
 
+# Asked BEFORE the build so a stray --fresh costs a keystroke, not five minutes and a
+# re-embed. Skipped with no tty (CI) and on a dry run, which wipes nothing anyway.
+if [[ $FRESH -eq 1 && $DRY -eq 0 ]]; then
+  echo "==> FRESH: drops the qdrant + data volumes (vectors, checkpoints, shards)"
+  if [[ -t 0 ]]; then
+    read -r -p "    re-embed everything from chain? [y/N] " ans
+    [[ $ans == [yY] ]] || { echo "aborted"; exit 1; }
+  fi
+fi
+
 run docker build -t "$IMG" -t "$SHA_IMG" .
 run docker push "$IMG"
 run docker push "$SHA_IMG"
@@ -66,13 +128,23 @@ files=(docker-compose.yml)
 run gcloud compute scp "${files[@]}" "$INSTANCE:~/" --zone="$ZONE"
 
 # Recreates only the services whose image actually changed; the qdrant and data
-# volumes are untouched, so the collection and baked shards survive.
+# volumes are untouched, so the collection and baked shards survive — unless --fresh
+# put a `down -v` in front, which is exactly what drops them.
+#
+# The prune is not optional housekeeping. IMAGE is a moving :latest, so every pull
+# leaves the previous ~2.8GB image untagged — invisible to plain `docker images`,
+# which is why a 20GB box fills up while it still reports 3GB of images. Six of them
+# had accumulated (14.8GB) by 2026-08-20. Dangling only: the image the stack now runs
+# is tagged, and rollback images live in Artifact Registry, not here.
 run gcloud compute ssh "$INSTANCE" --zone="$ZONE" \
-  --command='docker compose pull && docker compose up -d && docker compose ps'
+  --command="sed -i -e '/^SOURCES_URL=/d' -e '/^COMPOSE_FILE=/d' .env \
+    && echo 'SOURCES_URL=$SOURCES_URL' >> .env \
+    && ${FRESH_CMD}docker compose pull && docker compose up -d && docker image prune -f && docker compose ps"
 
 if [[ $DRY -eq 0 ]]; then
   echo
   echo "deployed $SHA. Roll back with:"
   echo "  ssh the box and run: docker compose pull ${SHA_IMG%:*}:<older-sha>"
   echo "cdn restarts until watch bakes the first domain — see DOCKER-README.md."
+  [[ $FRESH -eq 1 ]] && echo "fresh: the collection rebuilds by replaying from FROM_BLOCK — follow it with 'docker compose logs -f watch'."
 fi

docker-compose.yml

@@ -52,6 +52,14 @@ services:
       - --role-map-file=/data/db/role_map.json
       - --qdrant-url=http://qdrant:6333
       - --qdrant-api-key=${QDRANT_API_KEY}
+      # Replay history before going live, which is the ONLY way a wildcard source
+      # (`*:*`) discovers namespaces published before this box existed — it has no
+      # single namespace to seed with `fangorn read`. Overrides the CLI's saved cursor,
+      # so it re-runs on every restart. 0 = off (the default), NOT genesis.
+      # Cost: the catch-up is 1000 blocks per eth_getLogs, so a million blocks back is a
+      # thousand sequential calls against the public RPC. Pick a block just before the
+      # commits you want, not a round number far in the past.
+      - --from-block=${FROM_BLOCK:-0}
       # Reconnect backoff for a dropped subscribe stream — NOT a monitoring interval.
       # `watch` is push-based off `fangorn subscribe` and reacts as commits land.
       - --poll-interval=${INTERVAL:-60}

examples/audius/audius-demo-large/src/lib/kernel.tsx

@@ -52,7 +52,7 @@ function load(): Saved | null {
 }
 
 function save(s: Saved) {
-  // ponytail: synchronous write per signal (~110 KB at skip_window=20). Well under
+  // synchronous write per signal (~110 KB at skip_window=20). Well under
   // a frame and far under the 5 MB quota; debounce only if a profile says so.
   try { localStorage.setItem(KEY, JSON.stringify(s)); } catch { /* quota / private mode */ }
 }

examples/audius/audius-demo/src/lib/kernel.tsx

@@ -52,7 +52,7 @@ function load(): Saved | null {
 }
 
 function save(s: Saved) {
-  // ponytail: synchronous write per signal (~110 KB at skip_window=20). Well under
+  // synchronous write per signal (~110 KB at skip_window=20). Well under
   // a frame and far under the 5 MB quota; debounce only if a profile says so.
   try { localStorage.setItem(KEY, JSON.stringify(s)); } catch { /* quota / private mode */ }
 }

quickbeam.egg-info/SOURCES.txt

@@ -53,11 +53,15 @@ quickbeam/pipelines/prebake.py
 tests/test_adjacency_route.py
 tests/test_bucket_route.py
 tests/test_cdn_edges.py
+tests/test_cdn_events.py
 tests/test_index.py
+tests/test_ingest_checkpoint.py
 tests/test_mcp.py
 tests/test_roles.py
 tests/test_schemagen_identity.py
+tests/test_scope.py
 tests/test_shell.py
 tests/test_sources.py
 tests/test_watchlist.py
+tests/test_wildcard_cdn_domain.py
 tests/test_x402_agent.py

quickbeam/cdn.py

@@ -1268,7 +1268,7 @@ def catalog():
     # The watcher writes shards from a DIFFERENT container (shared /data volume), so
     # there is no in-process signal to hook. This polls each watched manifest instead
     # and diffs its shard list.
-    # ponytail: mtime-gated polling, not inotify — one small stat per domain per
+    # mtime-gated polling, not inotify — one small stat per domain per
     # connection per tick, against a handful of domains and a handful of clients.
     # Reach for watchfiles only if that ever shows up in a profile.
     _POLL_SECONDS = 2.0

quickbeam/index.py

@@ -34,7 +34,7 @@
 
 WHAT IS DELIBERATELY NOT HERE
 -----------------------------
-ponytail: the codebook is FLAT, not the two-level tree the design calls for. The tree
+The codebook is FLAT, not the two-level tree the design calls for. The tree
 is a client-side routing accelerator — it finds the same nearest centroid, just in
 ~512 comparisons instead of K — so it cannot change any number this harness reports.
 Fit it in Stage D from the flat codebook, when there is a client that cares.
@@ -134,7 +134,7 @@ def spherical_kmeans(X, k: int, iters: int = 25, seed: int = 0, chunk: int = 100
     transitively under umap-learn), and the balanced assignment below is something
     sklearn cannot do anyway, so both halves are written out.
 
-    ponytail: full-batch, chunked. At 500k x 4096 that is ~10s/iteration, fine for
+    Full-batch, chunked. At 500k x 4096 that is ~10s/iteration, fine for
     the gate. If this is still the fit path at 20M, switch to minibatch (sample a
     batch per iteration, same update rule) — the ceiling is wall-clock, not quality.
     """

quickbeam/ingest/sources/fangorn.py

@@ -103,13 +103,21 @@ def subscribe_cmd(fangorn_bin: str, owner: str | None, namespace: str | None,
 
     `from_block` (or `from_start`, its genesis form) replays history before going live —
     how an app-level watch discovers namespaces published before it started, since a
-    wildcard source can't be seeded with `read` (which needs one exact namespace). Prefer
-    `from_block`: the catch-up is windowed, and genesis on a fast chain like Arbitrum is
-    hundreds of thousands of windows."""
+    wildcard source can't be seeded with `read` (which needs one exact namespace). It
+    OVERRIDES the CLI's saved cursor (cli.js reads the flag before the cursor file), so
+    it replays on every reconnect, not only the first.
+
+    Prefer `from_block` to `from_start`: the catch-up is windowed at 1000 blocks per
+    eth_getLogs (FANGORN_LOG_WINDOW in the SDK), so each million blocks is a thousand
+    sequential RPC calls and genesis on Arbitrum is hundreds of thousands of them.
+
+    0 means "no replay", not "from genesis" — that is `from_start`. The falsy check is
+    what lets docker-compose.yml pass `--from-block=${FROM_BLOCK:-0}` unconditionally;
+    compose cannot omit an argument, and an empty one dies in argparse."""
     argv = [*shlex.split(fangorn_bin), "subscribe"]
     if owner is None or namespace is None:
         argv.append("--all")
-    if from_block is not None:
+    if from_block:
         argv += ["--from-block", str(from_block)]
     elif from_start:
         argv.append("--from-start")