feat(growth): explain company capture failures (#1015)

Author: blove

apps/growth-research/README.md

@@ -17,9 +17,19 @@ npx tsx apps/growth-research/scripts/research-pilot.mts acquire --output /absolu
 ```
 
 These commands return UUIDs for immutable JSON files in the selected output directory.
-Acquisition records include complete, partial, empty and failed outcomes. The existing
-fetcher can skip unusable pages, so missing paths have an unknown reason; redirects may
-make the original path indeterminate. Review the captured corpus before model calls:
+Acquisition records include complete, partial, empty and failed outcomes. Each capture's
+`pageDiagnostics` records the original requested path, a bounded outcome code, HTTP
+status and known byte count when available. Outcomes distinguish capture, access denial
+(403), rate limiting (429), other HTTP failures, oversized pages, request timeout,
+transport failure, rejected redirects, missing redirect locations, exhausted redirect
+budget and security rejection. Diagnostics emitted before a security rejection remain
+in the failed capture; caller cancellation still rejects acquisition. Diagnostics contain
+no response bodies, exception messages or redirect URLs. `access_denied` records HTTP
+403; it does not prove bot detection. Missing diagnostic entries can mean a page was
+not attempted or an older injected capture function did not support diagnostics. The
+250 KiB page limit, five-second timeout, three-total-redirect budget, exact-host redirect
+policy and SSRF controls are unchanged. The existing unavailable-path summary uses final URLs and can
+remain indeterminate after redirects. Review the captured corpus before model calls:
 remove personal biography/contact snippets, retain empty cases and failures, and fill
 expected claims/unknowns from the actual captured evidence. Save the reviewed corpus
 under a new name/version. Acquisition is preparation, not a human quality label.

apps/growth-research/src/pilot/acquisition.ts

@@ -1,4 +1,8 @@
-import { fetchCompanyEvidence } from '../../../lifecycle/src/enrichment/company-fetch.js';
+import {
+  fetchCompanyEvidence,
+  type CompanyFetchOverrides,
+  type CompanyPageDiagnostic,
+} from '../../../lifecycle/src/enrichment/company-fetch.js';
 import type { CompanyPageEvidence } from '../../../lifecycle/src/enrichment/schema.js';
 
 const expectedPaths = ['/', '/about', '/pricing'];
@@ -7,7 +11,8 @@ export async function acquireCompanies(
   signal: AbortSignal,
   capture: (
     domain: string,
-    signal: AbortSignal
+    signal: AbortSignal,
+    options?: Pick<CompanyFetchOverrides, 'onDiagnostic'>
   ) => Promise<CompanyPageEvidence[]> = fetchCompanyEvidence
 ) {
   if (
@@ -36,14 +41,18 @@ export async function acquireCompanies(
     reason: 'unavailable' | 'capture_failed' | null;
     redirectedPathsIndeterminate: boolean;
     filteredIdentityItems: number;
+    pageDiagnostics: CompanyPageDiagnostic[];
   }[] = [];
   for (const [index, domain] of domains.entries()) {
     signal.throwIfAborted();
     const id = `public-${index + 1}`;
     let pages: CompanyPageEvidence[] = [],
       failed = false;
+    const pageDiagnostics: CompanyPageDiagnostic[] = [];
     try {
-      pages = await capture(domain, signal);
+      pages = await capture(domain, signal, {
+        onDiagnostic: (diagnostic) => pageDiagnostics.push(diagnostic),
+      });
     } catch {
       signal.throwIfAborted();
       failed = true;
@@ -92,6 +101,7 @@ export async function acquireCompanies(
         (path) => !expectedPaths.includes(path)
       ),
       filteredIdentityItems,
+      pageDiagnostics,
     });
   }
   return {

apps/growth-research/test/pilot-acquisition.spec.ts

@@ -1,5 +1,34 @@
 import { expect, it } from 'vitest';
 import { acquireCompanies } from '../src/pilot/acquisition.js';
+// Exercise the same internal capture dependency used by pilot acquisition.
+// eslint-disable-next-line @nx/enforce-module-boundaries
+import { fetchCompanyEvidence } from '../../lifecycle/src/enrichment/company-fetch.js';
+
+it('retains partial diagnostics when a later page rejects for security', async () => {
+  const result = await acquireCompanies(
+    ['atlas.example'],
+    new AbortController().signal,
+    (domain, signal, options) =>
+      fetchCompanyEvidence(domain, signal, {
+        ...options,
+        resolve: async () => ['93.184.216.34'],
+        fetch: async (url) =>
+          url.pathname === '/'
+            ? new Response('<title>Atlas</title>')
+            : new Response(null, {
+                status: 302,
+                headers: { location: 'https://unsafe.example/?secret=private' },
+              }),
+      })
+  );
+  expect(result.captures[0].status).toBe('failed');
+  expect(result.cases[0].pages).toEqual([]);
+  expect(result.captures[0].pageDiagnostics).toEqual([
+    { requestedPath: '/', outcome: 'captured', status: 200, bytes: 20 },
+    { requestedPath: '/about', outcome: 'redirect_rejected', status: 302 },
+  ]);
+  expect(JSON.stringify(result)).not.toContain('private');
+});
 
 it('keeps partial, empty, and failed company captures visible', async () => {
   const result = await acquireCompanies(

apps/lifecycle/src/enrichment/company-fetch.spec.ts

@@ -9,11 +9,157 @@ import {
   fetchCompanyEvidence,
   resolveWithNodeDns,
   type CompanyFetchDependencies,
+  type CompanyPageDiagnostic,
   type CompanyRequestInit,
 } from './company-fetch.js';
 
 const NOW = new Date('2026-09-01T12:00:00.000Z');
 
+describe('page diagnostics', () => {
+  it.each([
+    [403, 'access_denied'],
+    [429, 'rate_limited'],
+    [503, 'http_error'],
+  ] as const)(
+    'reports HTTP %s without response content',
+    async (status, outcome) => {
+      const diagnostics: CompanyPageDiagnostic[] = [];
+      await expect(
+        fetchCompanyEvidence('example.com', new AbortController().signal, {
+          ...dependencies({
+            fetch: async () => new Response('private body', { status }),
+          }),
+          onDiagnostic: (diagnostic) => diagnostics.push(diagnostic),
+        })
+      ).resolves.toEqual([]);
+      expect(diagnostics).toEqual(
+        ['/', '/about', '/pricing'].map((requestedPath) => ({
+          requestedPath,
+          outcome,
+          status,
+        }))
+      );
+    }
+  );
+
+  it('records requested paths for redirected captures and isolates observer exceptions', async () => {
+    const diagnostics: CompanyPageDiagnostic[] = [];
+    const pages = await fetchCompanyEvidence(
+      'example.com',
+      new AbortController().signal,
+      {
+        ...dependencies({
+          fetch: async (url) =>
+            url.pathname === '/about'
+              ? new Response(null, {
+                  status: 302,
+                  headers: { location: '/company?token=private' },
+                })
+              : okPage(),
+        }),
+        onDiagnostic: (diagnostic) => {
+          diagnostics.push(diagnostic);
+          throw new Error('observer');
+        },
+      }
+    );
+    expect(pages).toHaveLength(3);
+    expect(diagnostics.map((d) => d.requestedPath)).toEqual([
+      '/',
+      '/about',
+      '/pricing',
+    ]);
+    expect(
+      diagnostics.every(
+        (d) =>
+          d.outcome === 'captured' && d.status === 200 && (d.bytes ?? 0) > 0
+      )
+    ).toBe(true);
+    expect(JSON.stringify(diagnostics)).not.toContain('private');
+  });
+
+  it.each([
+    'missing_location',
+    'redirect_limit',
+    'redirect_rejected',
+    'security_rejected',
+    'page_too_large',
+    'transport_failure',
+    'timeout',
+  ] as const)('classifies %s without weakening policy', async (outcome) => {
+    const diagnostics: CompanyPageDiagnostic[] = [];
+    const ownTimeout = AbortSignal.abort(new Error('private timeout'));
+    const operation = fetchCompanyEvidence(
+      'example.com',
+      new AbortController().signal,
+      {
+        ...dependencies({
+          ...(outcome === 'timeout'
+            ? {
+                createTimeoutSignal: () => ({
+                  signal: ownTimeout,
+                  clear: () => undefined,
+                }),
+              }
+            : {}),
+          resolve: async () => [
+            outcome === 'security_rejected' ? '127.0.0.1' : '93.184.216.34',
+          ],
+          fetch: async () => {
+            if (outcome === 'transport_failure')
+              throw new Error('private transport');
+            if (outcome === 'page_too_large')
+              return new Response('x', {
+                headers: { 'content-length': '256001' },
+              });
+            return new Response(null, {
+              status: 302,
+              headers:
+                outcome === 'missing_location'
+                  ? {}
+                  : {
+                      location:
+                        outcome === 'redirect_rejected'
+                          ? 'https://other.example/?secret=private'
+                          : '/loop',
+                    },
+            });
+          },
+        }),
+        onDiagnostic: (diagnostic) => {
+          diagnostics.push(diagnostic);
+          throw new Error('observer');
+        },
+      }
+    );
+    if (outcome === 'security_rejected' || outcome === 'redirect_rejected')
+      await expect(operation).rejects.toThrow(/unsafe/iu);
+    else await expect(operation).resolves.toEqual([]);
+    expect(diagnostics[0]).toMatchObject({ requestedPath: '/', outcome });
+    expect(JSON.stringify(diagnostics)).not.toContain('private');
+  });
+
+  it('does not classify caller cancellation as a timeout or let an observer mask it', async () => {
+    const controller = new AbortController();
+    const reason = new Error('caller stopped');
+    const observer = vi.fn(() => {
+      throw new Error('observer');
+    });
+    await expect(
+      fetchCompanyEvidence('example.com', controller.signal, {
+        ...dependencies({
+          fetch: async () => {
+            controller.abort(reason);
+            throw reason;
+          },
+        }),
+        onDiagnostic: observer,
+      })
+    ).rejects.toBe(reason);
+    expect(observer).not.toHaveBeenCalled();
+  });
+});
+
 function dependencies(
   overrides: Partial<CompanyFetchDependencies> = {}
 ): CompanyFetchDependencies {