Skip to content

Commit e9e7039

Browse files
JSONboredJSONbored
committed
feat(discovery-index): add the soft-claim coordination endpoint
Adds POST /v1/discovery-index/soft-claim as a second endpoint on the discovery-index service (#7218), accepting the payload shape discovery-soft-claim.ts's buildSoftClaimRequest already produces client-side. Lets opted-in miner instances avoid starting duplicate work on the same discovered opportunity: a claim is accepted if the key is free, or reported already-held with its age (and its TTL refreshed) if not; a release frees the key immediately. Storage reuses the existing TtlCache (a third instance, since it holds different data than the query-result/policy caches) rather than a new mechanism. The shipped SoftClaimRequest wire type carries no caller identity at all (buildSoftClaimRequest hardcodes note and instanceId to null), so "refresh rather than double-count on repeat calls" can only mean any repeat claim on a still-active key refreshes it -- there is no identity on the wire to distinguish callers. The parser never reads note/instanceId/any other field, so nothing forbidden can leak into the stored record or response regardless of what a caller sends. Closes #7166
1 parent b4e988e commit e9e7039

7 files changed

Lines changed: 404 additions & 1 deletion

File tree

‎packages/discovery-index/README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,11 @@ This is optional, shared infrastructure to reduce duplicate GitHub API pressure
1212
| `GET /ready` | Readiness — checks this service's own GitHub token is configured. |
1313
| `GET /metrics` | Prometheus text-format metrics. |
1414
| `POST /v1/discovery-index/query` | `Authorization: Bearer <DISCOVERY_INDEX_SHARED_SECRET>` → `DiscoveryIndexRequest` → `DiscoveryIndexResponse`. |
15+
| `POST /v1/discovery-index/soft-claim` | `Authorization: Bearer <DISCOVERY_INDEX_SHARED_SECRET>` → the payload shape `discovery-soft-claim.ts`'s `buildSoftClaimRequest` produces → `{contractVersion, accepted, ageMs}`. |
1516

16-
See `packages/loopover-engine/src/discovery-index-contract.ts` for the full request/response contract (`normalizeDiscoveryIndexRequest`/`normalizeDiscoveryIndexResponse`), which this service both consumes and emits through.
17+
See `packages/loopover-engine/src/discovery-index-contract.ts` for the full query request/response contract (`normalizeDiscoveryIndexRequest`/`normalizeDiscoveryIndexResponse`), which this service both consumes and emits through, and `packages/loopover-engine/src/discovery-soft-claim.ts` for the soft-claim payload builder.
18+
19+
Soft-claim design note: the shipped client payload never carries caller identity (`buildSoftClaimRequest` hardcodes `note`/`instanceId` to `null`) — this endpoint only ever sees `repoFullName` + `issueNumber` + `action`. A repeat `claim` call on a still-active key is reported as `accepted: false` (with the existing claim's age) and refreshes its TTL, since there is no identity on the wire to distinguish "the same caller checking in" from "a different caller."
1720

1821
## Configuration
1922

@@ -22,6 +25,7 @@ See `packages/loopover-engine/src/discovery-index-contract.ts` for the full requ
2225
| `DISCOVERY_INDEX_SHARED_SECRET` | Bearer secret required to call `/v1/discovery-index/*`. Unset ⇒ the service fails closed (503). |
2326
| `DISCOVERY_INDEX_GITHUB_TOKEN` | This service's own GitHub token, isolated from any other component's (REES, the main engine's installation tokens, etc.). Unset ⇒ `/ready` reports not-ready. |
2427
| `DISCOVERY_INDEX_CACHE_TTL_MS` | TTL for cached query results, per unique `(repos, orgs, searchTerms)` scope. Default `300000` (5 minutes). |
28+
| `DISCOVERY_INDEX_SOFT_CLAIM_TTL_MS` | TTL for a soft claim before it's reclaimable. Default `1800000` (30 minutes). |
2529
| `PORT` | HTTP port. Default `8080`. |
2630

2731
## Deployment

‎packages/discovery-index/src/app.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
// the pieces underneath.
77
import { Hono } from "hono";
88
import {
9+
DISCOVERY_INDEX_CONTRACT_VERSION,
910
type AiPolicyVerdict,
1011
type DiscoveryIndexCandidate,
1112
normalizeDiscoveryIndexRequest,
@@ -14,12 +15,14 @@ import { normalizeSharedSecret, verifyBearer } from "./auth.js";
1415
import type { TtlCache } from "./cache.js";
1516
import { runDiscoveryQuery, type GitHubClientLike } from "./discovery-query.js";
1617
import { incr, observe, renderMetrics } from "./metrics.js";
18+
import { parseSoftClaimRequest, softClaimKey, type SoftClaimStoreLike } from "./soft-claim.js";
1719

1820
export interface AppDeps {
1921
github: GitHubClientLike;
2022
resultCache: TtlCache<DiscoveryIndexCandidate[]>;
2123
policyCache: TtlCache<AiPolicyVerdict>;
2224
cacheTtlMs: number;
25+
softClaimStore: SoftClaimStoreLike;
2326
/** Whether this service's own GitHub token is configured — surfaced on /ready. */
2427
githubConfigured: boolean;
2528
}
@@ -32,6 +35,11 @@ export function createApp(deps: AppDeps): Hono {
3235
observe("discovery_index_query_request_duration_seconds", (Date.now() - startedAtMs) / 1000);
3336
}
3437

38+
function recordSoftClaimOutcome(status: string, startedAtMs: number): void {
39+
incr("discovery_index_soft_claim_requests_total", { status });
40+
observe("discovery_index_soft_claim_request_duration_seconds", (Date.now() - startedAtMs) / 1000);
41+
}
42+
3543
app.get("/health", (c) => c.json({ status: "ok", service: "discovery-index" }));
3644
app.get("/ready", (c) => c.json({ ready: deps.githubConfigured }, deps.githubConfigured ? 200 : 503));
3745
app.get("/metrics", (c) => c.text(renderMetrics()));
@@ -80,5 +88,46 @@ export function createApp(deps: AppDeps): Hono {
8088
}
8189
});
8290

91+
app.post("/v1/discovery-index/soft-claim", async (c) => {
92+
const startedAtMs = Date.now();
93+
try {
94+
const secret = normalizeSharedSecret(process.env.DISCOVERY_INDEX_SHARED_SECRET);
95+
if (!secret) {
96+
recordSoftClaimOutcome("service_not_configured", startedAtMs);
97+
return c.json({ error: "service_not_configured" }, 503);
98+
}
99+
if (!verifyBearer(c.req.header("authorization"), secret)) {
100+
recordSoftClaimOutcome("unauthorized", startedAtMs);
101+
return c.json({ error: "unauthorized" }, 401);
102+
}
103+
104+
const body: unknown = await c.req.json().catch(() => null);
105+
if (body === null) {
106+
recordSoftClaimOutcome("bad_request", startedAtMs);
107+
return c.json({ error: "invalid_json" }, 400);
108+
}
109+
110+
const parsed = parseSoftClaimRequest(body);
111+
if (parsed === null) {
112+
recordSoftClaimOutcome("bad_request", startedAtMs);
113+
return c.json({ error: "invalid_request" }, 400);
114+
}
115+
116+
const key = softClaimKey(parsed.repoFullName, parsed.issueNumber);
117+
let outcome: { accepted: boolean; ageMs: number | null };
118+
if (parsed.action === "release") {
119+
deps.softClaimStore.release(key);
120+
outcome = { accepted: true, ageMs: null };
121+
} else {
122+
outcome = deps.softClaimStore.claim(key);
123+
}
124+
recordSoftClaimOutcome("ok", startedAtMs);
125+
return c.json({ contractVersion: DISCOVERY_INDEX_CONTRACT_VERSION, ...outcome });
126+
} catch (error) {
127+
recordSoftClaimOutcome("error", startedAtMs);
128+
throw error;
129+
}
130+
});
131+
83132
return app;
84133
}

‎packages/discovery-index/src/metrics.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,14 @@ export const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
3939
"discovery_index_github_requests_total",
4040
{ help: "discovery-index outbound GitHub API requests, by outcome (ok/retried/failed).", type: "counter" },
4141
],
42+
[
43+
"discovery_index_soft_claim_requests_total",
44+
{ help: "discovery-index /v1/discovery-index/soft-claim call outcomes, by status.", type: "counter" },
45+
],
46+
[
47+
"discovery_index_soft_claim_request_duration_seconds",
48+
{ help: "discovery-index /v1/discovery-index/soft-claim request handling duration in seconds.", type: "histogram" },
49+
],
4250
];
4351
const metricMeta = new Map<string, MetricMeta>(DEFAULT_METRIC_META);
4452

‎packages/discovery-index/src/server.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,21 @@ import { createApp } from "./app.js";
1010
import { TtlCache } from "./cache.js";
1111
import { DEFAULT_CACHE_TTL_MS } from "./discovery-query.js";
1212
import { GitHubClient } from "./github-client.js";
13+
import { DEFAULT_SOFT_CLAIM_TTL_MS, SoftClaimStore } from "./soft-claim.js";
1314

1415
const githubToken = process.env.DISCOVERY_INDEX_GITHUB_TOKEN ?? "";
1516
const configuredCacheTtlMs = Number(process.env.DISCOVERY_INDEX_CACHE_TTL_MS);
1617
const cacheTtlMs = Number.isFinite(configuredCacheTtlMs) && configuredCacheTtlMs > 0 ? configuredCacheTtlMs : DEFAULT_CACHE_TTL_MS;
18+
const configuredSoftClaimTtlMs = Number(process.env.DISCOVERY_INDEX_SOFT_CLAIM_TTL_MS);
19+
const softClaimTtlMs =
20+
Number.isFinite(configuredSoftClaimTtlMs) && configuredSoftClaimTtlMs > 0 ? configuredSoftClaimTtlMs : DEFAULT_SOFT_CLAIM_TTL_MS;
1721

1822
const app = createApp({
1923
github: new GitHubClient({ token: githubToken }),
2024
resultCache: new TtlCache<DiscoveryIndexCandidate[]>(),
2125
policyCache: new TtlCache<AiPolicyVerdict>(),
2226
cacheTtlMs,
27+
softClaimStore: new SoftClaimStore(new TtlCache(), softClaimTtlMs),
2328
githubConfigured: githubToken.trim().length > 0,
2429
});
2530

‎packages/discovery-index/src/soft-claim.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// Soft-claim coordination for POST /v1/discovery-index/soft-claim (#7166): lets opted-in miner instances
2+
// avoid starting duplicate work on the same discovered opportunity. Accepts the payload shape
3+
// packages/loopover-engine/src/discovery-soft-claim.ts's buildSoftClaimRequest already produces client-side.
4+
//
5+
// Design note: buildSoftClaimRequest hardcodes `note: null` / `instanceId: null` on the wire -- the shipped
6+
// client contract carries NO caller identity at all, only repoFullName + issueNumber + claimedAt + action.
7+
// So this endpoint's "refresh rather than double-count on repeat calls from the same identifier" (the
8+
// issue's own wording) can only mean: since there is no identity field to distinguish callers, ANY repeat
9+
// "claim" call for a still-active key refreshes its TTL rather than erroring -- the server cannot and does
10+
// not attempt caller-identity tracking the contract doesn't transmit. This module never reads `note` or
11+
// `instanceId` from the incoming payload at all (structural safety, same pattern as discovery-query.ts):
12+
// nothing forbidden can leak into the stored record or the response because nothing but repoFullName/
13+
// issueNumber/action is ever looked at.
14+
import type { TtlCache } from "./cache.js";
15+
16+
export const DEFAULT_SOFT_CLAIM_TTL_MS = 1_800_000; // 30 minutes
17+
18+
export type SoftClaimAction = "claim" | "release";
19+
20+
export interface ParsedSoftClaimRequest {
21+
repoFullName: string;
22+
issueNumber: number;
23+
action: SoftClaimAction;
24+
}
25+
26+
export interface SoftClaimOutcome {
27+
accepted: boolean;
28+
/** The existing claim's age in ms when not accepted (already held); null when accepted or on release. */
29+
ageMs: number | null;
30+
}
31+
32+
/** `owner/repo` with exactly one slash and non-empty halves; anything else -> null. */
33+
function normalizeRepoFullName(value: string): string | null {
34+
const parts = value.trim().split("/");
35+
if (parts.length !== 2) return null;
36+
const [owner, repo] = parts;
37+
if (!owner || !repo) return null;
38+
return `${owner}/${repo}`;
39+
}
40+
41+
/**
42+
* Tolerant parse of an incoming soft-claim request. Returns null (never throws) if `repoFullName` doesn't
43+
* normalize to `owner/repo`, `issueNumber` isn't a positive integer, or `action` isn't `"claim"`/`"release"`.
44+
* Deliberately never reads `note`/`instanceId`/any other field -- see this module's header.
45+
*/
46+
export function parseSoftClaimRequest(raw: unknown): ParsedSoftClaimRequest | null {
47+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
48+
const record = raw as Record<string, unknown>;
49+
const repoFullName = typeof record.repoFullName === "string" ? normalizeRepoFullName(record.repoFullName) : null;
50+
if (repoFullName === null) return null;
51+
const issueNumber = record.issueNumber;
52+
if (typeof issueNumber !== "number" || !Number.isInteger(issueNumber) || issueNumber <= 0) return null;
53+
const action = record.action;
54+
if (action !== "claim" && action !== "release") return null;
55+
return { repoFullName, issueNumber, action };
56+
}
57+
58+
export function softClaimKey(repoFullName: string, issueNumber: number): string {
59+
return `${repoFullName}#${issueNumber}`;
60+
}
61+
62+
interface SoftClaimRecord {
63+
claimedAt: number;
64+
}
65+
66+
/** The subset of SoftClaimStore the app actually calls — kept as an interface so tests can inject a plain
67+
* stub (e.g. one whose methods throw, to exercise the route's error path), mirroring GitHubClientLike in
68+
* discovery-query.ts. */
69+
export interface SoftClaimStoreLike {
70+
claim(key: string): SoftClaimOutcome;
71+
release(key: string): void;
72+
}
73+
74+
/** Thin TTL-backed claim/release store, reusing cache.ts's TtlCache (the issue's own deliverable: reuse an
75+
* existing store rather than adding a new storage mechanism) rather than the discovery-query result/policy
76+
* caches themselves, which hold semantically different data. */
77+
export class SoftClaimStore implements SoftClaimStoreLike {
78+
constructor(
79+
private readonly cache: TtlCache<SoftClaimRecord>,
80+
private readonly ttlMs: number,
81+
private readonly now: () => number = Date.now,
82+
) {}
83+
84+
/** Accepts a fresh claim, or reports+refreshes an existing unexpired one. `claimedAt` is never reset on a
85+
* refresh, so the reported age stays meaningful (how long ago the ORIGINAL claim was made) across repeats. */
86+
claim(key: string): SoftClaimOutcome {
87+
const existing = this.cache.get(key);
88+
if (existing) {
89+
this.cache.set(key, existing, this.ttlMs);
90+
return { accepted: false, ageMs: this.now() - existing.claimedAt };
91+
}
92+
this.cache.set(key, { claimedAt: this.now() }, this.ttlMs);
93+
return { accepted: true, ageMs: null };
94+
}
95+
96+
/** Idempotent: removing an absent key is a no-op, same as removing a present one. */
97+
release(key: string): void {
98+
this.cache.delete(key);
99+
}
100+
}

‎test/unit/discovery-index/app.test.ts‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { createApp, type AppDeps } from "../../../packages/discovery-index/src/a
44
import { TtlCache } from "../../../packages/discovery-index/src/cache";
55
import { resetMetrics } from "../../../packages/discovery-index/src/metrics";
66
import type { GitHubClientLike } from "../../../packages/discovery-index/src/discovery-query";
7+
import { SoftClaimStore } from "../../../packages/discovery-index/src/soft-claim";
78

89
function makeDeps(overrides: Partial<AppDeps> = {}): AppDeps {
910
const github: GitHubClientLike = {
@@ -22,6 +23,7 @@ function makeDeps(overrides: Partial<AppDeps> = {}): AppDeps {
2223
resultCache: new TtlCache<DiscoveryIndexCandidate[]>(),
2324
policyCache: new TtlCache<AiPolicyVerdict>(),
2425
cacheTtlMs: 300_000,
26+
softClaimStore: new SoftClaimStore(new TtlCache(), 1_800_000),
2527
githubConfigured: true,
2628
...overrides,
2729
};
@@ -151,4 +153,126 @@ describe("discovery-index Hono app (#7164)", () => {
151153
expect(await res.json()).toEqual({ error: "internal_error" });
152154
});
153155
});
156+
157+
describe("POST /v1/discovery-index/soft-claim", () => {
158+
it("fails closed with 503 when no shared secret is configured", async () => {
159+
vi.stubEnv("DISCOVERY_INDEX_SHARED_SECRET", "");
160+
const app = createApp(makeDeps());
161+
const res = await app.request("/v1/discovery-index/soft-claim", {
162+
method: "POST",
163+
headers: { "content-type": "application/json" },
164+
body: JSON.stringify({ repoFullName: "acme/widgets", issueNumber: 1, action: "claim" }),
165+
});
166+
expect(res.status).toBe(503);
167+
expect(await res.json()).toEqual({ error: "service_not_configured" });
168+
});
169+
170+
it("returns 401 for a missing or incorrect bearer token", async () => {
171+
vi.stubEnv("DISCOVERY_INDEX_SHARED_SECRET", "sek");
172+
const app = createApp(makeDeps());
173+
const res = await app.request("/v1/discovery-index/soft-claim", {
174+
method: "POST",
175+
headers: { authorization: "Bearer nope", "content-type": "application/json" },
176+
body: JSON.stringify({ repoFullName: "acme/widgets", issueNumber: 1, action: "claim" }),
177+
});
178+
expect(res.status).toBe(401);
179+
});
180+
181+
it("returns 400 for an unparseable JSON body", async () => {
182+
vi.stubEnv("DISCOVERY_INDEX_SHARED_SECRET", "sek");
183+
const app = createApp(makeDeps());
184+
const res = await app.request("/v1/discovery-index/soft-claim", {
185+
method: "POST",
186+
headers: { authorization: "Bearer sek", "content-type": "application/json" },
187+
body: "not json",
188+
});
189+
expect(res.status).toBe(400);
190+
expect(await res.json()).toEqual({ error: "invalid_json" });
191+
});
192+
193+
it("returns 400 for a structurally invalid request", async () => {
194+
vi.stubEnv("DISCOVERY_INDEX_SHARED_SECRET", "sek");
195+
const app = createApp(makeDeps());
196+
const res = await app.request("/v1/discovery-index/soft-claim", {
197+
method: "POST",
198+
headers: { authorization: "Bearer sek", "content-type": "application/json" },
199+
body: JSON.stringify({ repoFullName: "no-slash", issueNumber: 1, action: "claim" }),
200+
});
201+
expect(res.status).toBe(400);
202+
expect(await res.json()).toEqual({ error: "invalid_request" });
203+
});
204+
205+
it("accepts a fresh claim, reports an already-held repeat claim with its age, then accepts again after release", async () => {
206+
vi.stubEnv("DISCOVERY_INDEX_SHARED_SECRET", "sek");
207+
const deps = makeDeps();
208+
const app = createApp(deps);
209+
const claimReq = () =>
210+
app.request("/v1/discovery-index/soft-claim", {
211+
method: "POST",
212+
headers: { authorization: "Bearer sek", "content-type": "application/json" },
213+
body: JSON.stringify({ repoFullName: "acme/widgets", issueNumber: 1, action: "claim" }),
214+
});
215+
216+
const first = await claimReq();
217+
expect(first.status).toBe(200);
218+
expect(await first.json()).toMatchObject({ accepted: true, ageMs: null });
219+
220+
const second = await claimReq();
221+
expect(second.status).toBe(200);
222+
const secondBody = (await second.json()) as { accepted: boolean; ageMs: number | null };
223+
expect(secondBody.accepted).toBe(false);
224+
expect(secondBody.ageMs).toBeGreaterThanOrEqual(0);
225+
226+
const release = await app.request("/v1/discovery-index/soft-claim", {
227+
method: "POST",
228+
headers: { authorization: "Bearer sek", "content-type": "application/json" },
229+
body: JSON.stringify({ repoFullName: "acme/widgets", issueNumber: 1, action: "release" }),
230+
});
231+
expect(release.status).toBe(200);
232+
expect(await release.json()).toMatchObject({ accepted: true, ageMs: null });
233+
234+
const third = await claimReq();
235+
expect(await third.json()).toMatchObject({ accepted: true, ageMs: null });
236+
});
237+
238+
it("never echoes forbidden or identity-shaped fields even when the caller sends them", async () => {
239+
vi.stubEnv("DISCOVERY_INDEX_SHARED_SECRET", "sek");
240+
const app = createApp(makeDeps());
241+
const res = await app.request("/v1/discovery-index/soft-claim", {
242+
method: "POST",
243+
headers: { authorization: "Bearer sek", "content-type": "application/json" },
244+
body: JSON.stringify({
245+
repoFullName: "acme/widgets",
246+
issueNumber: 1,
247+
action: "claim",
248+
note: "look at me",
249+
instanceId: "instance-123",
250+
reward: 999,
251+
wallet: "0xdeadbeef",
252+
}),
253+
});
254+
const body = (await res.json()) as Record<string, unknown>;
255+
expect(Object.keys(body).sort()).toEqual(["accepted", "ageMs", "contractVersion"]);
256+
});
257+
258+
it("returns 500 via the centralized error handler when the store throws", async () => {
259+
vi.stubEnv("DISCOVERY_INDEX_SHARED_SECRET", "sek");
260+
const throwingStore = {
261+
claim() {
262+
throw new Error("store exploded");
263+
},
264+
release() {
265+
throw new Error("store exploded");
266+
},
267+
};
268+
const app = createApp(makeDeps({ softClaimStore: throwingStore }));
269+
const res = await app.request("/v1/discovery-index/soft-claim", {
270+
method: "POST",
271+
headers: { authorization: "Bearer sek", "content-type": "application/json" },
272+
body: JSON.stringify({ repoFullName: "acme/widgets", issueNumber: 1, action: "claim" }),
273+
});
274+
expect(res.status).toBe(500);
275+
expect(await res.json()).toEqual({ error: "internal_error" });
276+
});
277+
});
154278
});

0 commit comments

Comments
 (0)