fix(discovery-index): treat body-signalled secondary rate limits as retryable (#10249)

isRateLimitStatus only inspected status/retry-after/x-ratelimit-remaining, so
a GitHub secondary rate limit (403 with body-only signalling, no retry-after,
remaining still non-zero) fell through as a permanent failure and silently
truncated the fan-out's candidate set. Port the body-regex check from
src/github/client.ts's isRateLimitedResponse, reading via clone().text() so
callers can still read the response body on the retried-and-succeeded path.

Co-authored-by: bitfathers94 <237535319+bitfathers94@users.noreply.github.com>

Author: bitfathers94

packages/discovery-index/src/github-client.ts

@@ -58,16 +58,22 @@ function defaultSleep(delayMs: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, delayMs));
 }
 
-function isRateLimitStatus(response: Response): boolean {
+async function isRateLimitStatus(response: Response): Promise<boolean> {
   if (response.status === 429) return true;
   if (response.status !== 403) return false;
   if (response.headers.get("retry-after") != null) return true;
   const remaining = response.headers.get("x-ratelimit-remaining");
-  return remaining != null && Number(remaining) === 0;
+  if (remaining != null && Number(remaining) === 0) return true;
+  try {
+    return /secondary rate limit|\babuse\b|api rate limit exceeded/i.test(await response.clone().text());
+    /* v8 ignore next 3 -- defensive: a cloned Response body that fails to read isn't reachable in practice */
+  } catch {
+    return false;
+  }
 }
 
-function isRetryableStatus(response: Response): boolean {
-  return response.status >= 500 || isRateLimitStatus(response);
+async function isRetryableStatus(response: Response): Promise<boolean> {
+  return response.status >= 500 || (await isRateLimitStatus(response));
 }
 
 function retryDelayMs(response: Response, attempt: number, backoffMs: (attempt: number) => number): number {
@@ -169,7 +175,7 @@ export class GitHubClient {
         this.requestTimeoutMs && this.requestTimeoutMs > 0 ? { ...init, signal: AbortSignal.timeout(this.requestTimeoutMs) } : init,
       );
       this.recordRateLimit(response);
-      if (!isRetryableStatus(response) || attempt >= this.maxAttempts) {
+      if (!(await isRetryableStatus(response)) || attempt >= this.maxAttempts) {
         incr("discovery_index_github_requests_total", { outcome: response.ok ? "ok" : "failed" });
         return response;
       }

test/unit/discovery-index/github-client.test.ts

@@ -226,6 +226,66 @@ describe("discovery-index GitHubClient (#7164)", () => {
       expect(calls).toHaveLength(1);
     });
 
+    it("retries a body-signalled secondary rate limit with no Retry-After header and a non-zero x-ratelimit-remaining", async () => {
+      const secondaryLimitBody = JSON.stringify({
+        message: "You have exceeded a secondary rate limit. Please wait a few minutes before you try again.",
+      });
+      const { fetchImpl, calls } = makeFetchStub([
+        new Response(secondaryLimitBody, { status: 403, headers: { "x-ratelimit-remaining": "4980" } }),
+        new Response("[]", { status: 200 }),
+      ]);
+      const client = new GitHubClient({ token: "tok", fetchImpl, sleepFn: vi.fn(), maxAttempts: 2 });
+      const result = await client.fetchRepoIssues("owner/repo");
+      expect(result).toEqual({ issues: [], warnings: [] });
+      expect(calls).toHaveLength(2);
+    });
+
+    it("retries a body-signalled secondary rate limit matching GitHub's older abuse-detection wording", async () => {
+      const { fetchImpl, calls } = makeFetchStub([
+        new Response(JSON.stringify({ message: "You have triggered an abuse detection mechanism." }), {
+          status: 403,
+          headers: { "x-ratelimit-remaining": "4980" },
+        }),
+        new Response("[]", { status: 200 }),
+      ]);
+      const client = new GitHubClient({ token: "tok", fetchImpl, sleepFn: vi.fn(), maxAttempts: 2 });
+      const { warnings } = await client.fetchRepoIssues("owner/repo");
+      expect(warnings).toEqual([]);
+      expect(calls).toHaveLength(2);
+    });
+
+    it("treats a 403 with an unreadable cloned body as not rate limited", async () => {
+      const unreadableResponse = {
+        status: 403,
+        ok: false,
+        headers: new Headers({ "x-ratelimit-remaining": "4980" }),
+        clone: () => ({ text: () => Promise.reject(new Error("body already consumed")) }),
+      } as unknown as Response;
+      const calls: string[] = [];
+      const fetchImpl = (async (url: string | URL) => {
+        calls.push(String(url));
+        return unreadableResponse;
+      }) as unknown as typeof fetch;
+      const client = new GitHubClient({ token: "tok", fetchImpl, sleepFn: vi.fn(), maxAttempts: 3 });
+      const { warnings } = await client.fetchRepoIssues("owner/repo");
+      expect(calls).toHaveLength(1);
+      expect(warnings[0]).toMatch(/403/);
+    });
+
+    it("REGRESSION: a body-signalled secondary rate limit is retried, not surfaced as a truncated result", async () => {
+      const secondaryLimitBody = JSON.stringify({
+        message: "You have exceeded a secondary rate limit. Please wait a few minutes before you try again.",
+      });
+      const { fetchImpl } = makeFetchStub([
+        new Response(secondaryLimitBody, { status: 403, headers: { "x-ratelimit-remaining": "4980" } }),
+        new Response(JSON.stringify([{ number: 1, title: "A" }]), { status: 200 }),
+      ]);
+      const client = new GitHubClient({ token: "tok", fetchImpl, sleepFn: vi.fn(), maxAttempts: 2 });
+      const { issues, warnings } = await client.fetchRepoIssues("owner/repo");
+      expect(issues).toHaveLength(1);
+      expect(warnings).toEqual([]);
+    });
+
     it("applies a per-attempt request timeout signal when configured", async () => {
       const { fetchImpl, calls } = makeFetchStub([new Response("[]", { status: 200 })]);
       const client = new GitHubClient({ token: "tok", fetchImpl, sleepFn: vi.fn(), requestTimeoutMs: 5000 });