Merge pull request #7218 from JSONbored/claude/open-issues-triage-a5f79f

feat(discovery-index): add the hosted discovery-index query service

Author: JSONbored

.env.example

@@ -66,6 +66,10 @@ LOOPOVER_REVIEW_ENRICHMENT=false
 #                                            #   available; otherwise GITHUB_PUBLIC_TOKEN).
 # REES_ANALYZERS=all                        # all | comma-list of exact names.
 #                                            # Unknown names warn and are ignored; a typo-only list runs no analyzers.
+# --profile discovery-index (#4250, #7164): the optional hosted discovery-index service. Local orchestration
+# only -- public hosting/DNS/TLS is a separate, still-open decision (#7167), not part of this profile.
+# DISCOVERY_INDEX_SHARED_SECRET=             # bearer secret required to call /v1/discovery-index/*; unset fails closed
+# DISCOVERY_INDEX_GITHUB_TOKEN=              # this service's own GitHub token, isolated from REES's/the engine's
 # BEGIN GENERATED REES ANALYZERS
 # Current analyzer names:
 # dependency,dependencyDiff,lockfileDrift,secret,license,installScript,heavyDependency
@@ -373,6 +377,7 @@ REDIS_URL=redis://redis:6379                # REQUIRED for the self-host review
 # POSTGRES_MEM_LIMIT=2g                      # --profile postgres / --profile pgbouncer
 # QDRANT_MEM_LIMIT=2g                        # --profile qdrant
 # REES_MEM_LIMIT=512m                        # --profile rees (the review-enrichment service)
+# DISCOVERY_INDEX_MEM_LIMIT=512m             # --profile discovery-index (the hosted discovery-index service)
 # OLLAMA_MEM_LIMIT=20g                       # --profile ollama; raise this before pulling a large local model
 #                                            #   (default sized for an embed model + one vision model resident
 #                                            #   together, #4335 -- raise further for additional/larger models)

codecov.yml

@@ -57,8 +57,8 @@ comment:
   require_changes: false
 
 # Coverage is collected by vitest (v8) over src/**, packages/loopover-engine/src/**,
-# and packages/loopover-miner/lib/** (plus review-enrichment/src/** via the standalone
-# `rees` flag, c8 over its built dist remapped through source maps); mirror vitest's
+# packages/loopover-miner/lib/**, and packages/discovery-index/src/** (plus review-enrichment/src/** via
+# the standalone `rees` flag, c8 over its built dist remapped through source maps); mirror vitest's
 # exclusions here so the Codecov total matches the local report.
 ignore:
   - "src/env.d.ts"
@@ -74,3 +74,6 @@ ignore:
   # gated on PG_TEST_URL) + the real-PG boot. The dialect translation itself IS unit-tested (pg-dialect.ts).
   - "src/selfhost/pg-adapter.ts"
   - "src/selfhost/pg-queue.ts"
+  # discovery-index's process entrypoint (#7164) -- same "Docker build+boot, not unit-coverable" reasoning
+  # as src/server.ts above; app.ts (everything it wires together) is what tests actually import.
+  - "packages/discovery-index/src/server.ts"

docker-compose.yml

@@ -23,6 +23,7 @@
 #   --profile workflows         n8n workflow automation (Slack/Teams/email fan-out, scheduled reports)
 #   --profile storage           MinIO S3-compatible object store (Litestream destination, artifact blobs)
 #   --profile caddy             Caddy HTTPS terminator with auto-TLS (set DOMAIN= in .env)
+#   --profile discovery-index   Local orchestration for the hosted discovery-index service (#4250, #7164)
 #   --profile observability     Prometheus + Alertmanager + Loki + Promtail + Grafana (pre-wired)
 #   --profile ams-observability Exports AMS (loopover-miner) attempt-log/prediction-ledger data to the
 #                               AMS Attempt Log / Prediction Ledger Grafana datasources
@@ -670,6 +671,47 @@ services:
       start_period: 20s
       retries: 3
 
+  # ── discovery-index (--profile discovery-index) ────────────────────────────
+  # Local orchestration only, for testing against the hosted discovery-index service (#4250, #7164) before
+  # it exists anywhere public -- the actual public hosting/DNS/TLS decision is separate (#7167), not this.
+  # Unlike rees above, the build context is the repo root: this service depends on the @loopover/engine
+  # workspace package, so its Dockerfile installs via npm workspaces rather than a self-contained `npm ci`.
+  discovery-index:
+    build:
+      context: .
+      dockerfile: packages/discovery-index/Dockerfile
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["discovery-index"]
+    environment:
+      PORT: "8080"
+      # Soft defaults, not ":?required" -- compose interpolates every service's env vars for the whole file
+      # even when --profile discovery-index isn't active (same reasoning as REES_SHARED_SECRET above). Unset
+      # DISCOVERY_INDEX_SHARED_SECRET means the service fails closed (503) on every query call; unset
+      # DISCOVERY_INDEX_GITHUB_TOKEN means /ready reports not-ready.
+      DISCOVERY_INDEX_SHARED_SECRET: ${DISCOVERY_INDEX_SHARED_SECRET:-}
+      DISCOVERY_INDEX_GITHUB_TOKEN: ${DISCOVERY_INDEX_GITHUB_TOKEN:-}
+    # Never published to the host -- reached only over the internal docker network, same in-network-only
+    # posture as rees above.
+    expose:
+      - "8080"
+    deploy:
+      resources:
+        limits:
+          memory: "${DISCOVERY_INDEX_MEM_LIMIT:-512m}"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "node",
+          "-e",
+          "fetch('http://127.0.0.1:8080/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))",
+        ]
+      interval: 30s
+      timeout: 5s
+      start_period: 20s
+      retries: 3
+
   # ── Litestream (--profile litestream) ─────────────────────────────────────
   # Continuous WAL backup of the SQLite DB to S3/B2/R2. Copy litestream.yml.example
   # → litestream.yml and fill in your bucket. Set LITESTREAM_* secrets in .env.

package-lock.json

@@ -99,7 +99,7 @@
     "test:smoke:observability:metrics": "node scripts/smoke-observability-metrics.mjs",
     "test:smoke:browser:install": "playwright install chromium",
     "test:smoke:browser": "node scripts/smoke-ui-browser.mjs",
-    "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run test --workspace @loopover/engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run release-manifest:sync:check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
+    "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run test --workspace @loopover/engine && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run release-manifest:sync:check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build",
     "test:release": "npm run test:ci && npm run changelog:check",
     "test:release:mcp": "npm run test:ci",
     "test:watch": "vitest",

package.json

@@ -0,0 +1,4 @@
+node_modules
+dist
+*.tsbuildinfo
+.DS_Store

packages/discovery-index/.dockerignore

@@ -0,0 +1,35 @@
+# LoopOver discovery-index service (#4250, #7164). Standalone Hono/Node service, deployment shape mirrors
+# review-enrichment/ (own Dockerfile, own port, no Cloudflare Worker involvement). Build context =
+# the packages/discovery-index/ directory — but it depends on the @loopover/engine workspace package
+# (packages/loopover-engine), so the build stage copies the whole monorepo root and installs via npm
+# workspaces rather than `npm ci` inside this directory alone.
+FROM node:22-slim AS build
+WORKDIR /app
+COPY package.json package-lock.json ./
+COPY packages/loopover-engine/package.json packages/loopover-engine/package.json
+COPY packages/discovery-index/package.json packages/discovery-index/package.json
+RUN npm ci --workspace @loopover/engine --workspace @loopover/discovery-index --include-workspace-root
+COPY packages/loopover-engine packages/loopover-engine
+COPY packages/discovery-index packages/discovery-index
+RUN npm run build --workspace @loopover/engine \
+  && npm run build --workspace @loopover/discovery-index \
+  && npm prune --omit=dev --workspace @loopover/discovery-index
+
+FROM node:22-slim AS runtime
+# Least privilege: run as a non-root user.
+RUN useradd --create-home --uid 10001 discovery-index
+WORKDIR /app
+ENV NODE_ENV=production
+COPY --from=build /app/node_modules ./node_modules
+COPY --from=build /app/packages/loopover-engine/node_modules ./packages/loopover-engine/node_modules
+COPY --from=build --chown=discovery-index:discovery-index /app/packages/loopover-engine/dist ./packages/loopover-engine/dist
+COPY --from=build --chown=discovery-index:discovery-index /app/packages/loopover-engine/package.json ./packages/loopover-engine/package.json
+COPY --from=build --chown=discovery-index:discovery-index /app/packages/discovery-index/dist ./packages/discovery-index/dist
+COPY --from=build --chown=discovery-index:discovery-index /app/packages/discovery-index/package.json ./packages/discovery-index/package.json
+WORKDIR /app/packages/discovery-index
+USER discovery-index
+ENV PORT=8080
+EXPOSE 8080
+# Provide at runtime (NOT baked into the image): DISCOVERY_INDEX_SHARED_SECRET (bearer auth for callers),
+# DISCOVERY_INDEX_GITHUB_TOKEN (this service's own GitHub token, isolated from any other component's).
+CMD ["node", "dist/server.js"]

packages/discovery-index/Dockerfile

@@ -0,0 +1,36 @@
+# Discovery-index service
+
+A standalone microservice implementing the hosted half of the [discovery plane](../loopover-miner/docs/discovery-plane-operator-guide.md) (#4250): a shared, cached GitHub issue/search index that opted-in `loopover-miner` instances can query instead of each independently fanning out to GitHub's search/listing APIs. Metadata-only — see [`@loopover/engine`'s discovery-index contract](../loopover-engine/src/discovery-index-contract.ts) for the exact public-safe candidate shape and the forbidden-field boundary this service can never cross.
+
+This is optional, shared infrastructure to reduce duplicate GitHub API pressure across the miner fleet (the rate-limit incident this mitigates: #1936). Self-hosted AMS/ORB deployments are completely unaffected whether or not this service exists — opting in is a separate, default-off client change (#7168).
+
+## API
+
+| Route                            | Purpose                                                                                       |
+| --------------------------------- | ----------------------------------------------------------------------------------------------- |
+| `GET /health`                    | Liveness health check.                                                                          |
+| `GET /ready`                     | Readiness — checks this service's own GitHub token is configured.                               |
+| `GET /metrics`                   | Prometheus text-format metrics.                                                                 |
+| `POST /v1/discovery-index/query` | `Authorization: Bearer <DISCOVERY_INDEX_SHARED_SECRET>` → `DiscoveryIndexRequest` → `DiscoveryIndexResponse`. |
+
+See `packages/loopover-engine/src/discovery-index-contract.ts` for the full request/response contract (`normalizeDiscoveryIndexRequest`/`normalizeDiscoveryIndexResponse`), which this service both consumes and emits through.
+
+## Configuration
+
+| Env var                             | Purpose                                                                                     |
+| ------------------------------------ | ----------------------------------------------------------------------------------------------- |
+| `DISCOVERY_INDEX_SHARED_SECRET`     | Bearer secret required to call `/v1/discovery-index/*`. Unset ⇒ the service fails closed (503). |
+| `DISCOVERY_INDEX_GITHUB_TOKEN`      | This service's own GitHub token, isolated from any other component's (REES, the main engine's installation tokens, etc.). Unset ⇒ `/ready` reports not-ready. |
+| `DISCOVERY_INDEX_CACHE_TTL_MS`      | TTL for cached query results, per unique `(repos, orgs, searchTerms)` scope. Default `300000` (5 minutes). |
+| `PORT`                              | HTTP port. Default `8080`.                                                                      |
+
+## Deployment
+
+Build and run via the included `Dockerfile` (build context = the **repository root**, since this service depends on the `@loopover/engine` workspace package):
+
+```sh
+docker build -f packages/discovery-index/Dockerfile -t loopover-discovery-index .
+docker run -p 8080:8080 -e DISCOVERY_INDEX_SHARED_SECRET=... -e DISCOVERY_INDEX_GITHUB_TOKEN=... loopover-discovery-index
+```
+
+Hosting/DNS/TLS/observability wiring for a public deployment is tracked separately (#7167), not part of this service's own code.

packages/discovery-index/README.md

@@ -0,0 +1,24 @@
+{
+  "name": "@loopover/discovery-index",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "description": "LoopOver hosted discovery-index service (#4250) — a shared, cached GitHub issue/search fan-out and soft-claim coordinator opted-in miner instances can query instead of each independently hammering GitHub. Deploys standalone; see #7164.",
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "start": "node dist/server.js",
+    "dev": "node --experimental-strip-types --watch src/server.ts"
+  },
+  "dependencies": {
+    "@hono/node-server": "^1.19.14",
+    "@loopover/engine": "^3.4.0",
+    "hono": "^4.12.27"
+  },
+  "devDependencies": {
+    "@types/node": "^22.20.0",
+    "typescript": "^5.9.3"
+  }
+}