feat: Evernote v10 crawler + cross-cutting infra (sortKey, attachment OCR, subdirConfig)

AGENTS.md

@@ -323,7 +323,7 @@ The `fink publish` command uses the `wrangler` CLI for all Cloudflare operations
 
 Three CF resources are created per deployment:
 
-1. **D1 Database** (`{workerName}-db`) — entities, tags, links, FTS5 search index, R2 manifest
+1. **D1 Database** (`{workerName}-db`) — `entities` (data JSON includes tags + links), `entities_fts` FTS5 index, `metadata`. Stale R2 files are detected by listing the bucket, not via a D1 table. See `SCHEMA.md` for the full schema.
 2. **R2 Bucket** (`{workerName}-files`) — markdown files, attachments, static UI assets (organized by `{collection}/markdown/...`, `{collection}/attachments/...`, `_ui/...`)
 3. **Worker** (`{workerName}`) — Hono server with D1 + R2 bindings, password auth, REST API, MCP
 
@@ -369,6 +369,79 @@ Skipping the build means the old compiled bundle is deployed — source changes
 
 **Note:** `wrangler` is not globally installed; invoke it via `bunx wrangler` or use the CLI which calls it as a subprocess automatically.
 
+#### Local ↔ worker parity rule
+
+**Whenever a change affects schema, query semantics, sort/ranking, the
+manifest, or the rendered output, the same change MUST land in the same PR
+for both code paths and both deployment scenarios.** The local server
+(`fink serve`) and the published Cloudflare worker are two independent
+implementations of the same API surface that must stay in lockstep. Test
+both in the same PR — local-only smoke tests gave us shipped gaps before.
+
+#### Schema migrations — single source of truth
+
+**All on-disk schema changes go through the migrations module — never
+inline `CREATE TABLE` / `ALTER TABLE` / FTS recreate elsewhere.** The
+two databases (local SQLite, published D1) each have their own ordered
+migration list:
+
+- `packages/core/src/db/migrations/local.ts` — `LOCAL_MIGRATIONS`
+- `packages/core/src/db/migrations/worker.ts` — `WORKER_MIGRATIONS`
+
+The runner records the latest applied id in each DB's `metadata` table
+under `schema.version`. The hot path is O(1) (one SELECT, cached in
+process). Migrations are triggered automatically from:
+
+- `getCollectionDb()` in `core/db/client.ts` — every local DB open
+- `SearchIndexer` constructor — defensive re-check (cached, free)
+- `publishCollections()` in `cli/commands/publish.ts` — every publish /
+  republish, before D1 schema is touched
+
+**When making a schema change**, update all of:
+
+1. The migration list (`local.ts` and/or `worker.ts`) — append a new
+   numbered migration; never edit a published one
+2. `SCHEMA.md` at the repo root — document what changed and why
+3. The PR test plan — include a `fink publish` to a scratch worker
+4. A migration unit test if the change is non-trivial (drop+recreate,
+   backfill loop, etc.)
+
+**Idempotence is non-negotiable.** Every migration body must be safe to
+re-run on a partially-applied DB. Use `CREATE TABLE IF NOT EXISTS`,
+`PRAGMA table_info` checks before drops, etc. The runner doesn't
+transaction-wrap migrations — if one fails halfway and is re-run, the
+second pass must finish what the first started.
+
+**FTS5 has no `ALTER TABLE ADD COLUMN`.** Any change that adds an FTS
+column needs a "detect missing → drop FTS table → recreate" body, plus
+a publish-time signal so unchanged entities still get their FTS rows
+re-INSERTed (`worker.ts` exports `FTS_RESET_FROM_BELOW` for this).
+
+What "the same change" looks like in practice — the checklist below is
+not exhaustive but covers the layers most commonly missed:
+
+| Change category | Local layer to update | Worker layer to update |
+|---|---|---|
+| New column on `entities` | New entry in `LOCAL_MIGRATIONS` + `core/db/collection-schema.ts` Drizzle schema | New entry in `WORKER_MIGRATIONS` + `worker/src/db/client.ts` SELECT, `cli/commands/publish.ts` INSERT prefix |
+| New column on `entities_fts` | New entry in `LOCAL_MIGRATIONS` (drop+recreate + `bm25()` weight) | New entry in `WORKER_MIGRATIONS` (drop+recreate + `FTS_RESET_FROM_BELOW`) + `worker/src/db/search.ts` (`bm25()` weight count, `snippet()` column index), `cli/commands/publish.ts` INSERT prefix |
+| New field inside `entities.data` JSON | `core/db/collection-schema.ts` `EntityData`, hash inclusion in `core/sync/entity-hash.ts` | Verify worker reads it: `worker/src/handlers/api.ts` SELECT `data` and parse it (no D1 schema change since data is opaque JSON) |
+| Search ranking / weights | `core/search/indexer.ts` `bm25()` | `worker/src/db/search.ts` `bm25()` — keep weights identical |
+| Markdown rendering / theme output | `crawlers/src/<name>/theme.ts` | Same theme imported by `worker/src/handlers/{api,mcp}.ts` — re-run worker build, `--worker-only` re-publish |
+| New folder-config field | `core/theme/interface.ts`, `cli/commands/{prepare,sync-engine}.ts` serializer | `worker/src/handlers/api.ts` tree builder reads `_config/<name>-folders.json` — make sure the field round-trips |
+| New entity field that affects the file tree (e.g. `sortKey`) | `serve.ts` `buildFileTree`'s comparator | `worker/src/handlers/api.ts` `buildTreeFromPaths` and `default-file` resolver |
+
+**Test plan must cover both.** A smoke test through `fink serve` is not
+sufficient. Test plans for schema/query/ranking changes must include a
+`fink publish` to a scratch worker and a verification step (e.g. an
+attachment-text-only search that should hit, a sort key that should
+reorder, a new field that should round-trip through clone). Capture the
+test in the PR description.
+
+Pre-existing collections (local or published) must be handled. Either
+the change is backward-compatible (worker reads new field gracefully
+when missing) or the migration runs automatically the next time the DB
+is opened / the next time `fink publish` runs.
+
 ### Build & Test Commands
 
 ```bash

SCHEMA.md

@@ -0,0 +1,120 @@
+# Schema
+
+This document tracks every change to the on-disk SQLite collection database (local) and the published Cloudflare D1 database (worker). It pairs with the migrations registered in `packages/core/src/db/migrations/`.
+
+## Source of truth
+
+- **Local migrations** — `packages/core/src/db/migrations/local.ts` (`LOCAL_MIGRATIONS`)
+- **Worker migrations** — `packages/core/src/db/migrations/worker.ts` (`WORKER_MIGRATIONS`)
+- **Schema version** — stored under the `schema.version` key in each database's `metadata` table
+
+The migration runner records the latest applied version once it finishes, and the runner's in-process cache short-circuits subsequent runs in the same process so the hot path issues zero queries.
+
+## Authoring rules
+
+1. **Every schema change goes through migrations.** No inline `CREATE TABLE` / `ALTER TABLE` / FTS recreate elsewhere in the codebase. The runner is called from `getCollectionDb()` (local) and `publishCollections()` (worker) so anything that opens the DB is covered.
+2. **Append-only — never edit a published migration.** Add a new id and let the runner apply it. Editing in place breaks every existing collection.
+3. **Each migration body must be idempotent.** Re-running it on a partially-applied DB has to be safe. Use `CREATE TABLE IF NOT EXISTS`, `PRAGMA table_info` checks before drops, etc.
+4. **Local and worker stay in lockstep.** Per the parity rule in `AGENTS.md`, every PR that touches schema must update both lists in the same commit. The two physical schemas differ in places (FTS `collection_name`, primary-key autoincrement, etc.) — that's fine; the migration ids don't have to match across the two arrays, but the *intent* of each version should.
+5. **Document below.** Every numbered migration gets an entry under the appropriate section, with what changed and why.
+
+When a new schema change ships, update **all five**:
+
+- The migration list (local and/or worker)
+- This file
+- The smoke-test plan in the PR (publish to a scratch worker, verify the new schema is applied)
+- `AGENTS.md` if the rules themselves change
+- A migration unit test if the change is non-trivial (drop+recreate, backfill, etc.)
+
+## Local schema (`~/.frozenink/collections/<name>/db/data.db`)
+
+Opened via `getCollectionDb()` in `packages/core/src/db/client.ts`. The runner is called automatically; `SearchIndexer` runs the same migrations defensively when it opens its own handle to the same file.
+
+### v1 — Baseline
+
+`entities` table with structured `folder` and `slug` columns plus indexes; `sync_errors` per-entity failure journal; `metadata` key-value store. Includes a one-time, idempotent backfill that populates `folder`/`slug` from the legacy `data.markdown_path` JSON field for collections that predate the column split, then strips the old `markdown_path`/`markdown_mtime`/`markdown_size` keys.
+
+### v2 — Add `entities_fts` virtual table
+
+FTS5 table with columns `collection_name UNINDEXED, entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags`. Detects a pre-existing 6-column FTS (no `collection_name`) via `PRAGMA table_info` and drops + recreates it. SyncEngine re-INSERTs FTS rows on the next sync.
+
+### v3 — Add `attachment_text` column to `entities_fts`
+
+Detect the missing column via `PRAGMA table_info`; drop + recreate the FTS table. FTS5 has no `ALTER TABLE ADD COLUMN`. The local SyncEngine re-INSERTs FTS rows on the next sync, this time including `attachment_text` from each entity's `data.assets[].text`.
+
+### v4 — Schema cleanup to align with the worker schema
+
+Two concurrent changes that minimise the gratuitous local↔worker divergence:
+
+- **Drop `collection_name` from `entities_fts`.** Each local collection has its own SQLite file, so collection scoping is implicit — the column was written but never queried. FTS5 has no DROP COLUMN, so detect via `PRAGMA table_info` then drop+recreate. Final 7-column shape: `entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags, attachment_text` — same as the worker. SyncEngine re-INSERTs FTS rows on the next sync.
+
+- **Drop `AUTOINCREMENT` from `entities.id`.** The historical local table used `INTEGER PRIMARY KEY AUTOINCREMENT` while the worker uses plain `INTEGER PRIMARY KEY`; we never depended on the no-id-reuse guarantee that `AUTOINCREMENT` provides (FTS rows are deleted alongside their entity row before any new INSERT, so collisions can't occur). The migration detects the historical shape via `sqlite_master.sql`, builds a clone table with the new shape, copies all rows preserving their existing ids, drops the original, renames the clone, and rebuilds the indexes. Wrapped in `BEGIN/COMMIT` — partial failure rolls back.
+
+Final FTS bm25 weights used by `SearchIndexer.search()`: `title=10, tags=5, body=1, attachment=0.25` (UNINDEXED columns get weight `1.0` but never match). `snippet()` highlights column index 4 (content). Worker `bm25()` and `snippet()` are kept identical per the parity rule.
+
+## Worker schema (Cloudflare D1)
+
+Created and migrated by `publishCollections()` in `packages/cli/src/commands/publish.ts`. The runner uses an async D1 executor backed by `executeD1Query` / `queryD1Rows` from `wrangler-api.ts`.
+
+After local v4, the **`entities` table and `entities_fts` schema are identical between local and worker**. The only intentional difference left is:
+
+- `sync_errors` (per-entity sync failure journal) — **local-only**. Sync runs locally; the worker never syncs.
+
+Stale-file cleanup on R2 is done by listing the bucket directly (`listR2Objects` in `wrangler-api.ts`) and diffing against what was just uploaded — there is **no `r2_manifest` D1 table**. Tags and links are not separate tables either; they live inside `entities.data` JSON.
+
+Both runtimes share the same `entities` DDL (extracted into `migrations/shared.ts` as `ENTITIES_TABLE_DDL` + `ENTITIES_INDEX_DDL`) and the same FTS column shape. Adding a column to `entities` should mean editing one constant and adding a numbered migration to both lists.
+
+### v1 — Baseline
+
+`entities` table with the same columns/indexes as the local v1; `entities_fts` virtual table with `entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags`.
+
+### v2 — Add `attachment_text` column to `entities_fts`
+
+Same idea as local v3: detect, drop + recreate. After this migration, `publishCollections()` force-includes every entity in the FTS push set — the manifest delta isn't enough on its own, because the FTS table was just emptied. The constant `FTS_RESET_FROM_BELOW = 2` exported from `db/migrations/worker.ts` lets the publish flow distinguish "the migration just ran" from "we were already current".
+
+bm25 weights used by `worker/src/db/search.ts`: `title=10, tags=5, body=1, attachment=0.25` — kept identical to the local indexer (parity rule).
+
+## Adding a new migration
+
+Pseudo-code for a typical schema change. Replace the version numbers and identifiers as appropriate.
+
+```ts
+// packages/core/src/db/migrations/local.ts
+export const LOCAL_MIGRATIONS: SyncMigration[] = [
+  // ...existing migrations, untouched...
+  {
+    id: 4,
+    description: "Short summary of the change.",
+    up: (db) => {
+      // Idempotent body. Read PRAGMAs to detect prior state if needed.
+    },
+  },
+];
+```
+
+```ts
+// packages/core/src/db/migrations/worker.ts
+export const WORKER_MIGRATIONS: AsyncMigration[] = [
+  // ...existing migrations, untouched...
+  {
+    id: 3,
+    description: "Mirror of the local change for the published D1 schema.",
+    up: async (db) => {
+      // Async idempotent body. Use db.query() for PRAGMAs.
+    },
+  },
+];
+```
+
+Then add corresponding sections to the **Local schema** / **Worker schema** lists above explaining the change.
+
+## Resetting (development only)
+
+To force every migration to re-run during development:
+
+```sh
+sqlite3 ~/.frozenink/collections/<name>/db/data.db "DELETE FROM metadata WHERE key='schema.version'"
+# Migrations are idempotent so they're safe to re-apply.
+```
+
+For D1, drop the metadata row via `wrangler d1 execute --remote`. Production should never need this — the migration runner handles version transitions automatically.

packages/cli/src/commands/add.ts

@@ -35,6 +35,8 @@ export const addCommand = new Command("add")
   .option("--max-prs <count>", "Maximum pull requests to sync (for github)", parseInt)
   .option("--open-only", "Only sync open issues/PRs, delete closed ones (for github)")
   .option("--sync-entities <types>", "Comma-separated entity types to sync: issues,pages,users (for mantishub)")
+  .option("--conduit-storage <path>", "Path to Evernote v10 conduit-storage directory (for evernote)")
+  .option("--notebooks <names>", "Comma-separated notebook allowlist (for evernote)")
   .option("--credentials <name>", "Use a named credential set from ~/.frozenink/credentials.yml")
   .addHelpText("after", `
 Examples:
@@ -190,6 +192,21 @@ Examples:
       if (opts.includeDiffs) {
         config.includeDiffs = true;
       }
+    } else if (crawlerType === "evernote") {
+      if (opts.conduitStorage) {
+        const { resolve } = await import("path");
+        const path = resolve(opts.conduitStorage);
+        config.conduitStoragePath = path;
+        if (!useNamedCreds) {
+          (credentials as Record<string, unknown>).conduitStoragePath = path;
+        }
+      }
+      if (opts.notebooks) {
+        config.notebooks = (opts.notebooks as string)
+          .split(",")
+          .map((s: string) => s.trim())
+          .filter(Boolean);
+      }
     } else if (crawlerType === "rss") {
       if (!opts.feedUrl) {
         console.error("RSS crawler requires --feed-url <url>");

packages/cli/src/commands/clone.ts

@@ -224,13 +224,19 @@ Examples:
         // Index inline — content is already in memory, no storage read needed
         const [dbEntity] = colDb.select().from(entities).where(eq(entities.externalId, re.externalId)).all();
         if (dbEntity) {
+          const reAssets = (re.data as Record<string, unknown> | undefined)?.assets as Array<{ text?: string }> | undefined;
+          const attachmentText = (reAssets ?? [])
+            .map((a) => a.text)
+            .filter((t): t is string => Boolean(t && t.trim()))
+            .join("\n");
           indexer.updateIndex({
             id: dbEntity.id,
             externalId: re.externalId,
             entityType: re.entityType,
             title: derivedTitle ?? re.title,
             content: markdown,
             tags: re.tags ?? [],
+            attachmentText,
           });
         }
 

packages/cli/src/commands/daemon.ts

@@ -14,7 +14,7 @@ import {
   spawnDetached,
   resolveCredentials,
 } from "@frozenink/core";
-import { createDefaultRegistry, gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme } from "@frozenink/crawlers";
+import { createDefaultRegistry, gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme, evernoteTheme } from "@frozenink/crawlers";
 
 function getPidPath(): string {
   return join(getFrozenInkHome(), "daemon.pid");
@@ -46,6 +46,7 @@ async function runSyncLoop(intervalMs: number): Promise<void> {
     themeEngine.register(gitTheme);
     themeEngine.register(mantisHubTheme);
     themeEngine.register(rssTheme);
+    themeEngine.register(evernoteTheme);
 
     for (const col of collectionRows) {
       const factory = registry.get(col.crawler);