feat(api): fetch full JDs at discovery through SSRF-hardened fetcher + extend local prerank (#263)

apps/api/src/lib/analysis-core.test.ts

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
-import { groundingFromParsed, type ParsedJobOutput } from './analysis-core';
+import { extractKeywords, groundingFromParsed, keywordCatalog, type ParsedJobOutput } from './analysis-core';
 
 const fallback = {
   requiredSkills: ['Python'],
@@ -39,3 +39,46 @@ test('groundingFromParsed falls back to the stored analysis when the parse is in
   assert.deepEqual(grounding.preferredSkills, ['SQL']);
   assert.deepEqual(grounding.atsKeywords, ['Python', 'SQL']);
 });
+
+test('keywordCatalog has no duplicates and contains at least 50 entries', () => {
+  assert.ok(keywordCatalog.length >= 50, `Expected keywordCatalog to have >= 50 items, but got ${keywordCatalog.length}`);
+  const uniqueKeywords = new Set(keywordCatalog.map((k) => k.toLowerCase()));
+  assert.equal(uniqueKeywords.size, keywordCatalog.length, 'keywordCatalog contains duplicate keywords (case-insensitive)');
+});
+
+test('extractKeywords matches short skill names on token boundaries and ignores substring false positives', () => {
+  // "JavaScript" should match JavaScript, not Java
+  const jsOnly = extractKeywords('We use JavaScript everyday.');
+  assert.ok(jsOnly.includes('JavaScript'));
+  assert.ok(!jsOnly.includes('Java'));
+
+  // Standalone "Java" matches Java
+  const javaOnly = extractKeywords('We use Java for backend services.');
+  assert.ok(javaOnly.includes('Java'));
+
+  // "Google Cloud", "MongoDB", "Django", "ongoing" should NOT match "Go"
+  const noGo = extractKeywords('We use Google Cloud, MongoDB, and Django for ongoing operations.');
+  assert.ok(!noGo.includes('Go'));
+  assert.ok(noGo.includes('Google Cloud'));
+  assert.ok(noGo.includes('MongoDB'));
+  assert.ok(noGo.includes('Django'));
+
+  // Standalone "Go" matches Go
+  const goOnly = extractKeywords('We write Go and Python services.');
+  assert.ok(goOnly.includes('Go'));
+  assert.ok(goOnly.includes('Python'));
+
+  // "trust" does NOT match "Rust"
+  const noRust = extractKeywords('We trust the process and learn quickly.');
+  assert.ok(!noRust.includes('Rust'));
+
+  // Standalone "Rust" matches
+  const rustOnly = extractKeywords('High-performance Rust systems.');
+  assert.ok(rustOnly.includes('Rust'));
+
+  // Symbols in skills like C#, C++, .NET
+  const symbols = extractKeywords('Skilled in C#, C++, and .NET Core.');
+  assert.ok(symbols.includes('C#'));
+  assert.ok(symbols.includes('C++'));
+  assert.ok(symbols.includes('.NET'));
+});

apps/api/src/lib/analysis-core.ts

@@ -26,7 +26,7 @@ export interface FitScoreOutput {
   model_used: string;
 }
 
-const keywordCatalog = [
+export const keywordCatalog = [
   'TypeScript',
   'JavaScript',
   'React',
@@ -47,6 +47,61 @@ const keywordCatalog = [
   'Workflow automation',
   'CRM',
   'Analytics',
+  'Java',
+  'C#',
+  '.NET',
+  'Go',
+  'Rust',
+  'C++',
+  'Ruby',
+  'PHP',
+  'Kotlin',
+  'Swift',
+  'Kubernetes',
+  'Docker',
+  'Terraform',
+  'AWS',
+  'GCP',
+  'Google Cloud',
+  'Azure',
+  'Linux',
+  'Git',
+  'CI/CD',
+  'GraphQL',
+  'REST API',
+  'gRPC',
+  'Redis',
+  'Kafka',
+  'RabbitMQ',
+  'MongoDB',
+  'MySQL',
+  'Elasticsearch',
+  'Snowflake',
+  'dbt',
+  'Airflow',
+  'Spark',
+  'ETL',
+  'Machine Learning',
+  'Deep Learning',
+  'PyTorch',
+  'TensorFlow',
+  'LangChain',
+  'RAG',
+  'Prompt Engineering',
+  'FastAPI',
+  'Django',
+  'Flask',
+  'Spring',
+  'Vue',
+  'Angular',
+  'Tailwind',
+  'Playwright',
+  'Jest',
+  'Grafana',
+  'Prometheus',
+  'Datadog',
+  'Agile',
+  'Scrum',
 ] as const;
 
 function clamp(value: number, min: number, max: number) {
@@ -57,8 +112,22 @@ function unique(values: string[]) {
   return [...new Set(values.map((value) => value.trim()).filter(Boolean))];
 }
 
+function escapeRegex(str: string): string {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+const keywordRegexes = keywordCatalog.map((keyword) => {
+  const escaped = escapeRegex(keyword);
+  // Match keyword bounded by non-alphanumeric characters or string ends,
+  // treating # and + as word-like characters so C# and C++ aren't truncated.
+  const regex = new RegExp(`(^|[^a-zA-Z0-9#+])(${escaped})($|[^a-zA-Z0-9#+])`, 'i');
+  return { keyword, regex };
+});
+
 export function extractKeywords(text: string): string[] {
-  return keywordCatalog.filter((keyword) => text.toLowerCase().includes(keyword.toLowerCase()));
+  return keywordRegexes
+    .filter(({ regex }) => regex.test(text))
+    .map(({ keyword }) => keyword);
 }
 
 function inferCompany(description: string) {

apps/api/src/lib/discovery.test.ts

@@ -370,3 +370,95 @@ test('passes sponsorLikelihood to createJob when lookupSponsor returns a known s
     denials: 1,
   });
 });
+
+test('with DISCOVERY_JD_FETCH_CAP unset, calls upgradeJd stub exactly 25 times for 30 eligible snippet jobs', async () => {
+  const originalEnv = process.env.DISCOVERY_JD_FETCH_CAP;
+  delete process.env.DISCOVERY_JD_FETCH_CAP;
+
+  try {
+    const snippetJobs = Array.from({ length: 30 }, (_, i) =>
+      sourced(`https://example.com/job/${i}`, {
+        company: `Company ${i}`,
+        title: `Engineer ${i}`,
+        descriptionText: 'Short snippet description',
+      }),
+    );
+
+    let upgradeCalls = 0;
+    const { deps } = makeDeps(snippetJobs, []);
+    deps.upgradeJd = async (job) => {
+      upgradeCalls += 1;
+      return { job, upgraded: false };
+    };
+
+    await runDiscoveryForUser('u', deps);
+
+    assert.equal(upgradeCalls, 25);
+  } finally {
+    if (originalEnv !== undefined) {
+      process.env.DISCOVERY_JD_FETCH_CAP = originalEnv;
+    } else {
+      delete process.env.DISCOVERY_JD_FETCH_CAP;
+    }
+  }
+});
+
+test('prerank runs on upgraded text so saveAnalysis receives non-null fitScore', async () => {
+  // Original snippet has no catalog keywords -> prerank would produce null score.
+  const snippetJob = sourced('https://example.com/job/prerank', {
+    company: 'Upgraded Tech',
+    title: 'Software Engineer',
+    descriptionText: 'Just a brief snippet.',
+  });
+
+  // Resume has TypeScript, React, and Node.js
+  const resume = 'Senior engineer with TypeScript, React, and Node.js experience.';
+  const { deps, analyses } = makeDeps([snippetJob], [], resume);
+
+  deps.upgradeJd = async (job) => {
+    return {
+      job: {
+        ...job,
+        descriptionText: 'Full JD: We use TypeScript, React, and Node.js daily to build products.',
+      },
+      upgraded: true,
+    };
+  };
+
+  await runDiscoveryForUser('u', deps);
+
+  assert.equal(analyses.length, 1);
+  assert.notEqual(analyses[0]?.fitScore, null);
+  assert.equal(analyses[0]?.fitScore, 100);
+});
+
+test('runDiscoveryForUser stops attempting JD upgrades when upgrade time budget is exhausted', async () => {
+  const originalBudget = process.env.DISCOVERY_JD_UPGRADE_BUDGET_MS;
+  process.env.DISCOVERY_JD_UPGRADE_BUDGET_MS = '0';
+
+  try {
+    const jobs: SourcedJob[] = Array.from({ length: 5 }, (_, i) => ({
+      source: 'adzuna',
+      company: `Company ${i}`,
+      title: `Software Engineer ${i}`,
+      descriptionText: 'Snippet',
+      jobUrl: `https://example.com/job/${i}`,
+    }));
+
+    let upgradeCalls = 0;
+    const { deps } = makeDeps(jobs, []);
+    deps.upgradeJd = async (job) => {
+      upgradeCalls += 1;
+      return { job, upgraded: false };
+    };
+
+    await runDiscoveryForUser('u', deps);
+    assert.equal(upgradeCalls, 0);
+  } finally {
+    if (originalBudget !== undefined) {
+      process.env.DISCOVERY_JD_UPGRADE_BUDGET_MS = originalBudget;
+    } else {
+      delete process.env.DISCOVERY_JD_UPGRADE_BUDGET_MS;
+    }
+  }
+});

apps/api/src/lib/discovery.ts

@@ -8,6 +8,7 @@ import { prerankAnalysis } from '@/lib/local-fit';
 import type { JobSource } from '@/lib/job-sources';
 import { dedupKey, fingerprintKey, type SourcedJob } from '@/lib/job-sources/normalize';
 import { fetchTargetCompanyBoards } from '@/lib/job-sources/boards';
+import { FULL_JD_MIN_CHARS, type upgradeToFullJd } from '@/lib/jd-upgrade';
 import type { SponsorLikelihood, TargetCompany } from '@/types';
 
 export interface DiscoveryResult {
@@ -27,6 +28,7 @@ export interface DiscoveryDeps {
   listTargetCompanies?: (userId: string) => Promise<TargetCompany[]>;
   fetchBoards?: typeof fetchTargetCompanyBoards;
   lookupSponsor?: (company: string) => Promise<SponsorLikelihood | null>;
+  upgradeJd?: typeof upgradeToFullJd;
 }
 
 /**
@@ -60,6 +62,9 @@ function isDuplicateKeyError(error: unknown): boolean {
  * the run). A single failing search is skipped rather than aborting the run.
  */
 export async function runDiscoveryForUser(userId: string, deps: DiscoveryDeps): Promise<DiscoveryResult> {
+  const JD_FETCH_CAP = Number(process.env.DISCOVERY_JD_FETCH_CAP ?? 25);
+  const JD_UPGRADE_TIME_BUDGET_MS = Number(process.env.DISCOVERY_JD_UPGRADE_BUDGET_MS ?? 15_000);
+  const jdUpgradeDeadline = Date.now() + JD_UPGRADE_TIME_BUDGET_MS;
   const searches = await deps.listSavedSearches(userId);
   const seen = new Set((await deps.listJobs(userId)).flatMap(keysFor));
   const resume = await deps.getResume(userId);
@@ -72,6 +77,7 @@ export async function runDiscoveryForUser(userId: string, deps: DiscoveryDeps):
 
   let inserted = 0;
   let skipped = 0;
+  let jdFetchAttempts = 0;
   const contributingSources = new Set<string>();
 
   async function insertIfNew(job: SourcedJob): Promise<void> {
@@ -84,6 +90,22 @@ export async function runDiscoveryForUser(userId: string, deps: DiscoveryDeps):
     // copy in the same run is recognised as a duplicate.
     for (const k of keysFor(job)) seen.add(k);
     try {
+      const currentDesc = job.descriptionText ?? '';
+      const remainingBudgetMs = jdUpgradeDeadline - Date.now();
+      if (
+        deps.upgradeJd &&
+        job.jobUrl &&
+        currentDesc.length < FULL_JD_MIN_CHARS &&
+        jdFetchAttempts < JD_FETCH_CAP &&
+        remainingBudgetMs > 500
+      ) {
+        jdFetchAttempts += 1;
+        const { job: upgraded } = await deps.upgradeJd(job, {}, {
+          timeoutMs: Math.min(8_000, remainingBudgetMs),
+        });
+        job = upgraded;
+      }
+
       const sponsor = deps.lookupSponsor ? await deps.lookupSponsor(job.company) : null;
       const createdJob = await deps.createJob(userId, {
         ...job,