fix: translate scanned PDFs by recovering hidden OCR text with pdf.js

Native extraction returns only an image for scanned pages, so only
footers were translated. Re-extract such pages with pdf.js, normalize
OCR words into lines, weight body size by characters, ship pdfjs-dist
on every platform, and bump the layout cache schema to 4.

Author: reycn

electron/build/production-stage.mjs

@@ -1,4 +1,13 @@
-import { cp, mkdir, mkdtemp, readFile, writeFile, readdir, stat } from 'node:fs/promises';
+import {
+  cp,
+  mkdir,
+  mkdtemp,
+  readFile,
+  writeFile,
+  readdir,
+  stat,
+  rm as removePath,
+} from 'node:fs/promises';
 import { createRequire } from 'node:module';
 import { dirname, join, resolve, relative } from 'node:path';
 import { tmpdir } from 'node:os';
@@ -103,9 +112,25 @@ export async function stageApplication({
   if (!names.includes('onnxruntime-node')) names.push('onnxruntime-node');
   if (test && !names.includes('pdf-lib')) names.push('pdf-lib');
   const fallback = !hasNativeInspector(platform, arch);
-  if (fallback && !names.includes('pdfjs-dist')) names.push('pdfjs-dist');
-  if (fallback && !names.includes('@thednp/dommatrix')) names.push('@thednp/dommatrix');
+  // PDF.js is the whole extractor on fallback platforms and recovers OCR text layers
+  // that the native inspector drops on scanned pages elsewhere.
+  if (!names.includes('pdfjs-dist')) names.push('pdfjs-dist');
+  if (!names.includes('@thednp/dommatrix')) names.push('@thednp/dommatrix');
   const copied = await copyDependencies(root, stage, names, fallback);
+  const pdfjsRoot = join(stage, 'node_modules/pdfjs-dist');
+  for (const entry of await readdir(pdfjsRoot))
+    if (
+      !['package.json', 'LICENSE', 'legacy', 'cmaps', 'standard_fonts', 'wasm', 'iccs'].includes(
+        entry,
+      )
+    )
+      await removePath(join(pdfjsRoot, entry), { recursive: true, force: true });
+  for (const entry of await readdir(join(pdfjsRoot, 'legacy')))
+    if (entry !== 'build')
+      await removePath(join(pdfjsRoot, 'legacy', entry), { recursive: true, force: true });
+  for (const entry of await readdir(join(pdfjsRoot, 'legacy/build')))
+    if (!['pdf.mjs', 'pdf.worker.mjs'].includes(entry))
+      await removePath(join(pdfjsRoot, 'legacy/build', entry), { force: true });
   // Ship only the target CPU runtime; other architectures multiply bundle size.
   const onnxBin = join(stage, 'node_modules/onnxruntime-node/bin/napi-v6');
   const { rm: removeRuntime } = await import('node:fs/promises');

server/documents/layout.mjs

@@ -39,9 +39,20 @@ export function paragraphs(items, height) {
       });
   }
   lines.sort((a, b) => a.top - b.top || a.x - b.x);
-  const bodySize =
-    runs.map((r) => r.fontSize || r.height).sort((a, b) => a - b)[Math.floor(runs.length / 2)] ||
-    12;
+  // Weight by characters so many short runs (footnotes, superscripts) cannot outvote the body.
+  const sizes = runs
+    .map((r) => [r.fontSize || r.height, r.text.trim().length])
+    .sort((a, b) => a[0] - b[0]);
+  let bodySize = 12;
+  for (
+    let i = 0, seen = 0, half = sizes.reduce((sum, [, n]) => sum + n, 0) / 2;
+    i < sizes.length;
+    i++
+  )
+    if ((seen += sizes[i][1]) >= half) {
+      bodySize = sizes[i][0];
+      break;
+    }
   // Learn normal leading from nearby lines of the same size, independently of columns.
   const gaps = new Map();
   for (let i = 0; i < lines.length; i++) {

server/documents/pdf-extractor.mjs

@@ -10,7 +10,7 @@ export function hasNativeInspector(platform = process.platform, arch = process.a
       : platform === 'linux' && ['x64', 'arm64'].includes(arch);
 }
 
-let native, pdfjs;
+let native, pdfjs, recoveryWarned;
 export async function extractPdfJsPositions(bytes, pages) {
   pdfjs ??= (async () => {
     if (!globalThis.DOMMatrix) globalThis.DOMMatrix = (await import('@thednp/dommatrix')).default;
@@ -66,8 +66,90 @@ export async function extractPdfJsPositions(bytes, pages) {
   }
 }
 
-export async function extractTextWithPositionsAsync(...args) {
-  if (!hasNativeInspector()) return extractPdfJsPositions(...args);
+const textLength = (items) =>
+  items.reduce(
+    (sum, item) => sum + (item.itemType === 'Image' ? 0 : item.text?.trim().length || 0),
+    0,
+  );
+
+// OCR words jitter in baseline, size and spacing. Merge each baseline into runs and
+// snap near-body sizes to the page's body size so layout grouping sees whole lines.
+export function normalizeOcrItems(items) {
+  const groups = [];
+  for (const item of [...items].sort((a, b) => b.y - a.y)) {
+    const group = groups.at(-1);
+    if (group && Math.abs(group[0].y - item.y) < Math.min(group[0].fontSize, item.fontSize) * 0.3)
+      group.push(item);
+    else groups.push([item]);
+  }
+  const runs = groups.flatMap((group) => {
+    const size = group.map((item) => item.fontSize).sort((a, b) => a - b)[group.length >> 1],
+      merged = [];
+    for (const item of group.sort((a, b) => a.x - b.x)) {
+      const run = merged.at(-1);
+      if (run && item.x - (run.x + run.width) < size * 1.6) {
+        run.text += ' ' + item.text;
+        run.width = Math.max(run.width, item.x + item.width - run.x);
+        run.isBold ||= item.isBold;
+      } else merged.push({ ...item, y: group[0].y, fontSize: size, height: size });
+    }
+    return merged;
+  });
+  const sizes = runs
+      .flatMap((run) => Array(run.text.length).fill(run.fontSize))
+      .sort((a, b) => a - b),
+    body = sizes[sizes.length >> 1];
+  for (const run of runs)
+    if (run.fontSize >= body * 0.8 && run.fontSize <= body * 1.3) run.fontSize = run.height = body;
+  return runs;
+}
+
+// The native inspector drops invisible OCR text layers on scanned pages, leaving only
+// the page image plus any real overlay text (e.g. a JSTOR footer). Re-read those pages
+// with pdf.js and keep its text when it finds substantially more.
+export async function recoverScannedPages(bytes, items, pdfJs = extractPdfJsPositions) {
+  const byPage = new Map();
+  for (const item of items) {
+    if (!byPage.has(item.page)) byPage.set(item.page, []);
+    byPage.get(item.page).push(item);
+  }
+  const candidates = [...byPage]
+    .filter(([, pageItems]) => pageItems.some((item) => item.itemType === 'Image'))
+    .map(([page]) => page);
+  if (!candidates.length) return items;
+  const recovered = new Map();
+  for (const item of await pdfJs(bytes, candidates)) {
+    if (!recovered.has(item.page)) recovered.set(item.page, []);
+    recovered.get(item.page).push(item);
+  }
+  for (const [page, pageItems] of recovered) recovered.set(page, normalizeOcrItems(pageItems));
+  const replaced = new Set(
+    candidates.filter((page) => {
+      const before = textLength(byPage.get(page)),
+        after = textLength(recovered.get(page) || []);
+      return after >= 200 && after > before * 2;
+    }),
+  );
+  if (!replaced.size) return items;
+  const result = [];
+  for (const [page, pageItems] of byPage)
+    result.push(
+      ...(replaced.has(page)
+        ? [...pageItems.filter((item) => item.itemType === 'Image'), ...recovered.get(page)]
+        : pageItems),
+    );
+  return result;
+}
+
+export async function extractTextWithPositionsAsync(bytes, ...args) {
+  if (!hasNativeInspector()) return extractPdfJsPositions(bytes, ...args);
   native ??= import('@firecrawl/pdf-inspector');
-  return (await native).extractTextWithPositionsAsync(...args);
+  const items = await (await native).extractTextWithPositionsAsync(bytes, ...args);
+  try {
+    return await recoverScannedPages(bytes, items);
+  } catch (error) {
+    if (!recoveryWarned) console.warn('Scanned-page text recovery unavailable:', error?.message);
+    recoveryWarned = true;
+    return items;
+  }
 }

server/http/document-routes.mjs

@@ -4,7 +4,7 @@ import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { paragraphs } from '../documents/layout.mjs';
 
-const LAYOUT_CACHE_SCHEMA = 2;
+const LAYOUT_CACHE_SCHEMA = 4;
 const DOCUMENT_HASH_PATTERN = /^[0-9a-f]{64}$/;
 const CACHE_GENERATION_PATTERN = /^[a-z0-9-]+$/i;
 

tests/server/documents/pdf-extractor.test.mjs

@@ -4,7 +4,9 @@ import { PDFDocument, StandardFonts, degrees } from 'pdf-lib';
 import {
   hasNativeInspector,
   extractPdfJsPositions,
+  recoverScannedPages,
 } from '../../../server/documents/pdf-extractor.mjs';
+import { paragraphs } from '../../../server/documents/layout.mjs';
 
 test('portable extraction preserves selected pages, display-frame positions and font sizes', async () => {
   const pdf = await PDFDocument.create(),
@@ -28,3 +30,82 @@ test('portable extraction preserves selected pages, display-frame positions and
   assert.ok(Math.abs(rotated[0].rotation) === 90);
   await assert.rejects(extractPdfJsPositions(bytes, [3]));
 });
+
+test('scanned pages recover the hidden OCR layer the native inspector drops', async () => {
+  const image = (page) => ({
+      page,
+      text: '[Image]',
+      itemType: 'Image',
+      x: 0,
+      y: 0,
+      width: 400,
+      height: 600,
+    }),
+    footer = (page) => ({
+      page,
+      text: 'All use subject to terms',
+      itemType: 'Text',
+      x: 100,
+      y: 10,
+      width: 100,
+      height: 6,
+      fontSize: 6,
+    }),
+    native = [
+      image(1),
+      footer(1),
+      {
+        page: 2,
+        text: 'Born digital page',
+        itemType: 'Text',
+        x: 40,
+        y: 500,
+        width: 120,
+        height: 10,
+        fontSize: 10,
+      },
+    ],
+    word = (text, x, y, fontSize = 10) => ({
+      page: 1,
+      text,
+      itemType: 'Text',
+      x,
+      y,
+      width: text.length * 5,
+      height: fontSize,
+      fontSize,
+      isBold: false,
+      rotation: 0,
+    });
+  const ocr = [];
+  // Words jitter in baseline and size and sit just over a 1.2-em gap apart.
+  for (let line = 0; line < 8; line++)
+    for (let column = 0; column < 4; column++)
+      ocr.push(
+        word(
+          'sociology',
+          60 + column * 57.5,
+          500 - line * 12 + (column % 2) * 0.4,
+          column === 1 ? 8 : 10,
+        ),
+      );
+  ocr.push(footer(1));
+  const requested = [];
+  const items = await recoverScannedPages(new Uint8Array(), native, async (_bytes, pages) => {
+    requested.push(...pages);
+    return ocr;
+  });
+  assert.deepEqual(requested, [1]);
+  assert.equal(items.filter((item) => item.page === 2).length, 1);
+  assert.ok(items.some((item) => item.page === 1 && item.itemType === 'Image'));
+  const result = paragraphs(
+    items.filter((item) => item.page === 1),
+    600,
+  );
+  assert.equal(result[0].text, Array(32).fill('sociology').join(' '));
+  assert.equal(result.at(-1).text, 'All use subject to terms');
+  assert.equal(
+    await recoverScannedPages(new Uint8Array(), native, async () => [footer(1)]),
+    native,
+  );
+});

tests/server/persistent-layout-cache.test.mjs

@@ -82,7 +82,7 @@ async function layout(origin, documentId, height = 792) {
   });
 }
 
-async function cachePath(directory, height = 792, schema = 2) {
+async function cachePath(directory, height = 792, schema = 4) {
   const documentHash = createHash('sha256').update(PDF).digest('hex');
   const generation = (
     await readFile(join(directory, 'documents', `${documentHash}.generation`), 'utf8')

tests/server/production-stage.test.mjs

@@ -16,6 +16,9 @@ test('production bundle serves and extracts PDFs without an external Express ins
       await access(join(stage, 'server/kernels/python', name));
     await access(join(stage, 'server/platform/macos/local-translation.swift'));
     await access(join(stage, 'server/formula/formula-ocr-worker.mjs'));
+    // Scanned pages recover their OCR text layer through PDF.js on every platform.
+    await access(join(stage, 'node_modules/pdfjs-dist/legacy/build/pdf.worker.mjs'));
+    await access(join(stage, 'node_modules/pdfjs-dist/standard_fonts'));
     await access(
       join(stage, 'node_modules/onnxruntime-node/bin/napi-v6', process.platform, process.arch),
     );