feat(miner-discovery-plane): define the public-data-only discovery-index API contract

packages/gittensory-engine/src/discovery-index-contract.ts

@@ -0,0 +1,238 @@
+// Discovery-index API contract (#4300). The typed request/response shape a miner uses to query the OPTIONAL hosted
+// discovery-index service (the server side is #4250, maintainer-only, explicitly blocked on this contract). The
+// plane exists to mitigate the rate-limit incident (#1936): one shared GitHub-metadata crawler across the fleet
+// instead of every miner independently hammering the same repos' search/listing endpoints.
+//
+// This module is schema/shape ONLY — no server, no deployed endpoint, no client HTTP. It stays inside the Phase 1
+// boundary (packages/gittensory-miner/docs/cross-repo-discovery-phase1.md): metadata-only, GET/list/search-only,
+// and NO raw scores / rewards / wallet / hotkey data / source contents crossing the public boundary. The response
+// candidate shape deliberately maps onto opportunity-ranker.js's `normalizeCandidate` fields so a miner can swap a
+// local fan-out for a hosted query without the ranker changing. Tolerant-parser convention, mirroring
+// miner-goal-spec.ts / fleet-run-manifest.ts: every field optional, malformed input degrades to a documented
+// default with a warning rather than throwing.
+
+export const DISCOVERY_INDEX_CONTRACT_VERSION = 1;
+
+const MAX_QUERY_ITEMS = 200;
+const MAX_PAGE_LIMIT = 200;
+const DEFAULT_PAGE_LIMIT = 50;
+
+/** Query scope for a discovery-index request: which repos/orgs/search-terms to fan out over, plus pagination. */
+export type DiscoveryIndexQuery = {
+  /** Canonical `owner/repo` targets. */
+  repos: readonly string[];
+  /** Bare `owner` (org/user) targets — every open issue across the owner's repos. */
+  orgs: readonly string[];
+  /** Free-text GitHub issue-search terms. */
+  searchTerms: readonly string[];
+  /** Page size, clamped to [1, 200]. Default 50. */
+  limit: number;
+  /** Opaque forward pagination cursor from a previous response's `nextCursor`, or null for the first page. */
+  cursor: string | null;
+};
+
+export type DiscoveryIndexRequest = {
+  contractVersion: number;
+  query: DiscoveryIndexQuery;
+};
+
+export type DiscoveryIndexAiPolicySource = "AI-USAGE.md" | "CONTRIBUTING.md" | "none";
+
+/** One metadata-only candidate issue. Field-for-field compatible with opportunity-ranker.js `normalizeCandidate`
+ *  output, so `rankCandidateIssues` consumes hosted results exactly like a local fan-out. Public-safe by contract:
+ *  no scores/rewards/wallet/hotkey/source fields ever appear here (see {@link DISCOVERY_INDEX_FORBIDDEN_FIELDS}). */
+export type DiscoveryIndexCandidate = {
+  owner: string;
+  repo: string;
+  repoFullName: string;
+  issueNumber: number;
+  title: string;
+  labels: readonly string[];
+  commentsCount: number;
+  createdAt: string | null;
+  updatedAt: string | null;
+  htmlUrl: string | null;
+  aiPolicyAllowed: boolean;
+  aiPolicySource: DiscoveryIndexAiPolicySource;
+};
+
+export type DiscoveryIndexResponse = {
+  contractVersion: number;
+  candidates: readonly DiscoveryIndexCandidate[];
+  /** Forward cursor for the next page, or null when the result set is exhausted. */
+  nextCursor: string | null;
+};
+
+export type ParsedDiscoveryIndexRequest = {
+  request: DiscoveryIndexRequest;
+  warnings: string[];
+};
+
+export type ParsedDiscoveryIndexResponse = {
+  response: DiscoveryIndexResponse;
+  warnings: string[];
+};
+
+/** Field-name fragments that must NEVER cross the public discovery boundary (Phase 1 acceptance:
+ *  cross-repo-discovery-phase1.md:13-14,54). A candidate carrying any of these is rejected, not silently trimmed,
+ *  so a misbehaving server can't smuggle raw economic/identity/source data past the contract. */
+export const DISCOVERY_INDEX_FORBIDDEN_FIELDS: readonly string[] = Object.freeze([
+  "score",
+  "reward",
+  "wallet",
+  "hotkey",
+  "coldkey",
+  "mnemonic",
+  "payout",
+  "ranking",
+  "rawtrust",
+  "trustscore",
+  "sourcecontent",
+  "diff",
+  "patch",
+]);
+
+/** Owner/repo names of forbidden-field violations present on a raw candidate object (own enumerable keys whose
+ *  lower-cased name contains a forbidden fragment). Empty array = public-safe. */
+export function discoveryIndexBoundaryViolations(raw: unknown): string[] {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return [];
+  const violations: string[] = [];
+  for (const key of Object.keys(raw)) {
+    const lower = key.toLowerCase();
+    if (DISCOVERY_INDEX_FORBIDDEN_FIELDS.some((fragment) => lower.includes(fragment))) violations.push(key);
+  }
+  return violations;
+}
+
+function normalizeStringList(value: unknown, transform: (entry: string) => string | null): string[] {
+  if (!Array.isArray(value)) return [];
+  const result: string[] = [];
+  const seen = new Set<string>();
+  for (const entry of value) {
+    if (typeof entry !== "string") continue;
+    if (result.length >= MAX_QUERY_ITEMS) break;
+    const normalized = transform(entry);
+    if (normalized === null || seen.has(normalized)) continue;
+    seen.add(normalized);
+    result.push(normalized);
+  }
+  return result;
+}
+
+/** `owner/repo` with exactly one slash and non-empty halves; anything else → null (mirrors normalizeCandidate). */
+function normalizeRepoFullName(value: string): string | null {
+  const [owner, repo, extra] = value.trim().split("/");
+  if (!owner || !repo || extra !== undefined) return null;
+  return `${owner}/${repo}`;
+}
+
+function normalizeOwner(value: string): string | null {
+  const trimmed = value.trim();
+  if (!trimmed || trimmed.includes("/")) return null;
+  return trimmed;
+}
+
+function normalizeSearchTerm(value: string): string | null {
+  const trimmed = value.trim();
+  return trimmed || null;
+}
+
+function clampLimit(value: unknown, warnings: string[]): number {
+  if (value === undefined || value === null) return DEFAULT_PAGE_LIMIT;
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    warnings.push(`DiscoveryIndexRequest "limit" must be a number; falling back to ${DEFAULT_PAGE_LIMIT}.`);
+    return DEFAULT_PAGE_LIMIT;
+  }
+  const floored = Math.floor(value);
+  if (floored < 1) return 1;
+  if (floored > MAX_PAGE_LIMIT) return MAX_PAGE_LIMIT;
+  return floored;
+}
+
+function normalizeAiPolicySource(value: unknown): DiscoveryIndexAiPolicySource {
+  return value === "AI-USAGE.md" || value === "CONTRIBUTING.md" ? value : "none";
+}
+
+/**
+ * Tolerantly normalize a raw discovery-index request into a canonical {@link DiscoveryIndexRequest}. Never throws:
+ * unknown fields are ignored, malformed scope entries are skipped, and the page limit is clamped, accumulating
+ * warnings. A non-object raw yields an empty query.
+ */
+export function normalizeDiscoveryIndexRequest(raw: unknown): ParsedDiscoveryIndexRequest {
+  const warnings: string[] = [];
+  const record = raw && typeof raw === "object" && !Array.isArray(raw) ? (raw as Record<string, unknown>) : null;
+  if (!record) {
+    warnings.push("DiscoveryIndexRequest must be a mapping; falling back to an empty query.");
+  }
+  const source = record ?? {};
+  const query: DiscoveryIndexQuery = {
+    repos: normalizeStringList(source.repos, normalizeRepoFullName),
+    orgs: normalizeStringList(source.orgs, normalizeOwner),
+    searchTerms: normalizeStringList(source.searchTerms, normalizeSearchTerm),
+    limit: clampLimit(source.limit, warnings),
+    cursor: typeof source.cursor === "string" && source.cursor.trim() ? source.cursor : null,
+  };
+  return { request: { contractVersion: DISCOVERY_INDEX_CONTRACT_VERSION, query }, warnings };
+}
+
+/**
+ * Normalize one raw candidate into a public-safe {@link DiscoveryIndexCandidate}, mirroring opportunity-ranker.js
+ * `normalizeCandidate`. Returns null when required fields are missing/invalid OR when the raw object carries any
+ * forbidden boundary field (a public-safety rejection, not a silent trim).
+ */
+export function normalizeDiscoveryIndexCandidate(raw: unknown): DiscoveryIndexCandidate | null {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  if (discoveryIndexBoundaryViolations(raw).length > 0) return null;
+  const candidate = raw as Record<string, unknown>;
+  const repoFullName = typeof candidate.repoFullName === "string" ? candidate.repoFullName : "";
+  const canonical = normalizeRepoFullName(repoFullName);
+  const issueNumber = candidate.issueNumber;
+  const title = typeof candidate.title === "string" ? candidate.title.trim() : "";
+  if (canonical === null) return null;
+  if (typeof issueNumber !== "number" || !Number.isInteger(issueNumber) || issueNumber <= 0 || !title) return null;
+  // `canonical` is guaranteed `owner/repo` with exactly one slash, so slice yields two non-empty strings.
+  const slashIndex = canonical.indexOf("/");
+  const owner = canonical.slice(0, slashIndex);
+  const repo = canonical.slice(slashIndex + 1);
+  const labels = Array.isArray(candidate.labels)
+    ? candidate.labels.filter((label): label is string => typeof label === "string" && label.trim() !== "").map((label) => label.trim())
+    : [];
+  return {
+    owner,
+    repo,
+    repoFullName: canonical,
+    issueNumber,
+    title,
+    labels,
+    commentsCount: typeof candidate.commentsCount === "number" && Number.isFinite(candidate.commentsCount) ? candidate.commentsCount : 0,
+    createdAt: typeof candidate.createdAt === "string" ? candidate.createdAt : null,
+    updatedAt: typeof candidate.updatedAt === "string" ? candidate.updatedAt : null,
+    htmlUrl: typeof candidate.htmlUrl === "string" ? candidate.htmlUrl : null,
+    aiPolicyAllowed: candidate.aiPolicyAllowed !== false,
+    aiPolicySource: normalizeAiPolicySource(candidate.aiPolicySource),
+  };
+}
+
+/**
+ * Tolerantly normalize a raw discovery-index response: keep only valid, public-safe candidates (invalid or
+ * boundary-violating entries are dropped with a warning), and carry a forward cursor. Never throws.
+ */
+export function normalizeDiscoveryIndexResponse(raw: unknown): ParsedDiscoveryIndexResponse {
+  const warnings: string[] = [];
+  const record = raw && typeof raw === "object" && !Array.isArray(raw) ? (raw as Record<string, unknown>) : null;
+  if (!record) {
+    warnings.push("DiscoveryIndexResponse must be a mapping; falling back to an empty candidate list.");
+  }
+  const rawCandidates = record && Array.isArray(record.candidates) ? record.candidates : [];
+  const candidates: DiscoveryIndexCandidate[] = [];
+  for (const entry of rawCandidates) {
+    const normalized = normalizeDiscoveryIndexCandidate(entry);
+    if (normalized === null) {
+      warnings.push("DiscoveryIndexResponse dropped an invalid or boundary-violating candidate.");
+      continue;
+    }
+    candidates.push(normalized);
+  }
+  const nextCursor = record && typeof record.nextCursor === "string" && record.nextCursor.trim() ? (record.nextCursor as string) : null;
+  return { response: { contractVersion: DISCOVERY_INDEX_CONTRACT_VERSION, candidates, nextCursor }, warnings };
+}

packages/gittensory-engine/src/index.ts

@@ -247,6 +247,21 @@ export {
   type FleetRunManifestRepo,
   type ParsedFleetRunManifest,
 } from "./fleet-run-manifest.js";
+export {
+  DISCOVERY_INDEX_CONTRACT_VERSION,
+  DISCOVERY_INDEX_FORBIDDEN_FIELDS,
+  discoveryIndexBoundaryViolations,
+  normalizeDiscoveryIndexCandidate,
+  normalizeDiscoveryIndexRequest,
+  normalizeDiscoveryIndexResponse,
+  type DiscoveryIndexAiPolicySource,
+  type DiscoveryIndexCandidate,
+  type DiscoveryIndexQuery,
+  type DiscoveryIndexRequest,
+  type DiscoveryIndexResponse,
+  type ParsedDiscoveryIndexRequest,
+  type ParsedDiscoveryIndexResponse,
+} from "./discovery-index-contract.js";
 export {
   computeMetadataLaneFit,
   computeMinerGoalLaneFit,

packages/gittensory-miner/docs/discovery-index-contract.md

@@ -0,0 +1,60 @@
+# Discovery-index API contract
+
+The **discovery-index contract** is the typed request/response shape a miner uses to query the *optional* hosted
+discovery-index service. It is defined in `@jsonbored/gittensory-engine`
+(`packages/gittensory-engine/src/discovery-index-contract.ts`) so both sides — this repo's future server
+implementation ([#4250](https://github.com/JSONbored/gittensory/issues/4250), maintainer-only, explicitly blocked
+on this contract) and any client — build against one shape.
+
+This is **schema/shape only**: no server, no deployed endpoint, no client HTTP implementation (those are #4250 and
+the sibling soft-claim-coordination issue, respectively).
+
+## Why
+
+The plane mitigates the rate-limit incident already fixed once for the review stack
+([#1936](https://github.com/JSONbored/gittensory/issues/1936)): *one* shared GitHub-metadata crawler across the
+miner fleet instead of every miner instance independently hammering the same repos' search/listing endpoints. The
+existing single-instance client pipeline (`packages/gittensory-miner/lib/opportunity-fanout.js` +
+`opportunity-ranker.js`) is what a hosted version must stay compatible with, so the response candidate shape is
+field-for-field compatible with `opportunity-ranker.js`'s `normalizeCandidate` — a miner can swap a local fan-out
+for a hosted query without the ranker changing.
+
+## Boundary (Phase 1)
+
+The contract stays inside the Phase 1 discovery boundary
+([`cross-repo-discovery-phase1.md`](./cross-repo-discovery-phase1.md)): metadata-only, GET/list/search-only, and
+**no raw scores, rewards, wallet/hotkey data, or source contents** cross the public boundary. This is enforced in
+code, not just documented: `discoveryIndexBoundaryViolations` lists any forbidden field on a raw object, and
+`normalizeDiscoveryIndexCandidate` **rejects** (returns `null`) — rather than silently trimming — any candidate
+carrying one, so a misbehaving server cannot smuggle economic/identity/source data past the contract.
+`DISCOVERY_INDEX_FORBIDDEN_FIELDS` is the fragment list (`score`, `reward`, `wallet`, `hotkey`, `coldkey`,
+`mnemonic`, `payout`, `ranking`, `rawtrust`, `trustscore`, `sourcecontent`, `diff`, `patch`).
+
+## Request — `DiscoveryIndexQuery`
+
+Every field is optional; a malformed value degrades to a documented default with a warning rather than throwing
+(`normalizeDiscoveryIndexRequest`).
+
+- **`repos`** — canonical `owner/repo` targets. Non-string / non-`owner/repo` entries are skipped; deduplicated;
+  capped at 200. Default: `[]`.
+- **`orgs`** — bare `owner` (org/user) targets. Entries containing `/` are skipped; deduplicated; capped at 200.
+  Default: `[]`.
+- **`searchTerms`** — free-text GitHub issue-search terms. Blank entries skipped; deduplicated; capped at 200.
+  Default: `[]`.
+- **`limit`** — page size, floored and clamped to `[1, 200]`; a non-numeric value warns and falls back. Default:
+  `50`.
+- **`cursor`** — opaque forward pagination cursor from a previous response's `nextCursor`; a blank/non-string value
+  becomes `null`. Default: `null`.
+
+## Response — `DiscoveryIndexResponse`
+
+- **`candidates`** — a list of `DiscoveryIndexCandidate`. `normalizeDiscoveryIndexResponse` keeps only valid,
+  public-safe entries and drops invalid or boundary-violating ones with a warning.
+- **`nextCursor`** — forward cursor for the next page, or `null` when the result set is exhausted.
+- **`contractVersion`** — `DISCOVERY_INDEX_CONTRACT_VERSION` (currently `1`).
+
+### `DiscoveryIndexCandidate`
+
+Metadata-only, field-for-field compatible with `opportunity-ranker.js` `normalizeCandidate`:
+`owner`, `repo`, `repoFullName`, `issueNumber`, `title`, `labels`, `commentsCount`, `createdAt`, `updatedAt`,
+`htmlUrl`, `aiPolicyAllowed`, `aiPolicySource` (`"AI-USAGE.md" | "CONTRIBUTING.md" | "none"`).