diff --git a/AGENTS.md b/AGENTS.md index c5f6675..98ddc51 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -323,7 +323,7 @@ The `fink publish` command uses the `wrangler` CLI for all Cloudflare operations Three CF resources are created per deployment: -1. **D1 Database** (`{workerName}-db`) — entities, tags, links, FTS5 search index, R2 manifest +1. **D1 Database** (`{workerName}-db`) — `entities` (data JSON includes tags + links), `entities_fts` FTS5 index, `metadata`. Stale R2 files are detected by listing the bucket, not via a D1 table. See `SCHEMA.md` for the full schema. 2. **R2 Bucket** (`{workerName}-files`) — markdown files, attachments, static UI assets (organized by `{collection}/markdown/...`, `{collection}/attachments/...`, `_ui/...`) 3. **Worker** (`{workerName}`) — Hono server with D1 + R2 bindings, password auth, REST API, MCP @@ -369,6 +369,79 @@ Skipping the build means the old compiled bundle is deployed — source changes **Note:** `wrangler` is not globally installed; invoke it via `bunx wrangler` or use the CLI which calls it as a subprocess automatically. +#### Local ↔ worker parity rule + +**Whenever a change affects schema, query semantics, sort/ranking, the +manifest, or the rendered output, the same change MUST land in the same PR +for both code paths and both deployment scenarios.** The local server +(`fink serve`) and the published Cloudflare worker are two independent +implementations of the same API surface that must stay in lockstep. Test +both in the same PR — local-only smoke tests gave us shipped gaps before. + +#### Schema migrations — single source of truth + +**All on-disk schema changes go through the migrations module — never +inline `CREATE TABLE` / `ALTER TABLE` / FTS recreate elsewhere.** The +two databases (local SQLite, published D1) each have their own ordered +migration list: + +- `packages/core/src/db/migrations/local.ts` — `LOCAL_MIGRATIONS` +- `packages/core/src/db/migrations/worker.ts` — `WORKER_MIGRATIONS` + +The runner records the latest applied id in each DB's `metadata` table +under `schema.version`. The hot path is O(1) (one SELECT, cached in +process). Migrations are triggered automatically from: + +- `getCollectionDb()` in `core/db/client.ts` — every local DB open +- `SearchIndexer` constructor — defensive re-check (cached, free) +- `publishCollections()` in `cli/commands/publish.ts` — every publish / + republish, before D1 schema is touched + +**When making a schema change**, update all of: + +1. The migration list (`local.ts` and/or `worker.ts`) — append a new + numbered migration; never edit a published one +2. `SCHEMA.md` at the repo root — document what changed and why +3. The PR test plan — include a `fink publish` to a scratch worker +4. A migration unit test if the change is non-trivial (drop+recreate, + backfill loop, etc.) + +**Idempotence is non-negotiable.** Every migration body must be safe to +re-run on a partially-applied DB. Use `CREATE TABLE IF NOT EXISTS`, +`PRAGMA table_info` checks before drops, etc. The runner doesn't +transaction-wrap migrations — if one fails halfway and is re-run, the +second pass must finish what the first started. + +**FTS5 has no `ALTER TABLE ADD COLUMN`.** Any change that adds an FTS +column needs a "detect missing → drop FTS table → recreate" body, plus +a publish-time signal so unchanged entities still get their FTS rows +re-INSERTed (`worker.ts` exports `FTS_RESET_FROM_BELOW` for this). + +What "the same change" looks like in practice — the checklist below is +not exhaustive but covers the layers most commonly missed: + +| Change category | Local layer to update | Worker layer to update | +|---|---|---| +| New column on `entities` | New entry in `LOCAL_MIGRATIONS` + `core/db/collection-schema.ts` Drizzle schema | New entry in `WORKER_MIGRATIONS` + `worker/src/db/client.ts` SELECT, `cli/commands/publish.ts` INSERT prefix | +| New column on `entities_fts` | New entry in `LOCAL_MIGRATIONS` (drop+recreate + `bm25()` weight) | New entry in `WORKER_MIGRATIONS` (drop+recreate + `FTS_RESET_FROM_BELOW`) + `worker/src/db/search.ts` (`bm25()` weight count, `snippet()` column index), `cli/commands/publish.ts` INSERT prefix | +| New field inside `entities.data` JSON | `core/db/collection-schema.ts` `EntityData`, hash inclusion in `core/sync/entity-hash.ts` | Verify worker reads it: `worker/src/handlers/api.ts` SELECT `data` and parse it (no D1 schema change since data is opaque JSON) | +| Search ranking / weights | `core/search/indexer.ts` `bm25()` | `worker/src/db/search.ts` `bm25()` — keep weights identical | +| Markdown rendering / theme output | `crawlers/src//theme.ts` | Same theme imported by `worker/src/handlers/{api,mcp}.ts` — re-run worker build, `--worker-only` re-publish | +| New folder-config field | `core/theme/interface.ts`, `cli/commands/{prepare,sync-engine}.ts` serializer | `worker/src/handlers/api.ts` tree builder reads `_config/-folders.json` — make sure the field round-trips | +| New entity field that affects the file tree (e.g. `sortKey`) | `serve.ts` `buildFileTree`'s comparator | `worker/src/handlers/api.ts` `buildTreeFromPaths` and `default-file` resolver | + +**Test plan must cover both.** A smoke test through `fink serve` is not +sufficient. Test plans for schema/query/ranking changes must include a +`fink publish` to a scratch worker and a verification step (e.g. an +attachment-text-only search that should hit, a sort key that should +reorder, a new field that should round-trip through clone). Capture the +test in the PR description. + +Pre-existing collections (local or published) must be handled. Either +the change is backward-compatible (worker reads new field gracefully +when missing) or the migration runs automatically the next time the DB +is opened / the next time `fink publish` runs. + ### Build & Test Commands ```bash diff --git a/SCHEMA.md b/SCHEMA.md new file mode 100644 index 0000000..8bdbcf7 --- /dev/null +++ b/SCHEMA.md @@ -0,0 +1,120 @@ +# Schema + +This document tracks every change to the on-disk SQLite collection database (local) and the published Cloudflare D1 database (worker). It pairs with the migrations registered in `packages/core/src/db/migrations/`. + +## Source of truth + +- **Local migrations** — `packages/core/src/db/migrations/local.ts` (`LOCAL_MIGRATIONS`) +- **Worker migrations** — `packages/core/src/db/migrations/worker.ts` (`WORKER_MIGRATIONS`) +- **Schema version** — stored under the `schema.version` key in each database's `metadata` table + +The migration runner records the latest applied version once it finishes, and the runner's in-process cache short-circuits subsequent runs in the same process so the hot path issues zero queries. + +## Authoring rules + +1. **Every schema change goes through migrations.** No inline `CREATE TABLE` / `ALTER TABLE` / FTS recreate elsewhere in the codebase. The runner is called from `getCollectionDb()` (local) and `publishCollections()` (worker) so anything that opens the DB is covered. +2. **Append-only — never edit a published migration.** Add a new id and let the runner apply it. Editing in place breaks every existing collection. +3. **Each migration body must be idempotent.** Re-running it on a partially-applied DB has to be safe. Use `CREATE TABLE IF NOT EXISTS`, `PRAGMA table_info` checks before drops, etc. +4. **Local and worker stay in lockstep.** Per the parity rule in `AGENTS.md`, every PR that touches schema must update both lists in the same commit. The two physical schemas differ in places (FTS `collection_name`, primary-key autoincrement, etc.) — that's fine; the migration ids don't have to match across the two arrays, but the *intent* of each version should. +5. **Document below.** Every numbered migration gets an entry under the appropriate section, with what changed and why. + +When a new schema change ships, update **all five**: + +- The migration list (local and/or worker) +- This file +- The smoke-test plan in the PR (publish to a scratch worker, verify the new schema is applied) +- `AGENTS.md` if the rules themselves change +- A migration unit test if the change is non-trivial (drop+recreate, backfill, etc.) + +## Local schema (`~/.frozenink/collections//db/data.db`) + +Opened via `getCollectionDb()` in `packages/core/src/db/client.ts`. The runner is called automatically; `SearchIndexer` runs the same migrations defensively when it opens its own handle to the same file. + +### v1 — Baseline + +`entities` table with structured `folder` and `slug` columns plus indexes; `sync_errors` per-entity failure journal; `metadata` key-value store. Includes a one-time, idempotent backfill that populates `folder`/`slug` from the legacy `data.markdown_path` JSON field for collections that predate the column split, then strips the old `markdown_path`/`markdown_mtime`/`markdown_size` keys. + +### v2 — Add `entities_fts` virtual table + +FTS5 table with columns `collection_name UNINDEXED, entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags`. Detects a pre-existing 6-column FTS (no `collection_name`) via `PRAGMA table_info` and drops + recreates it. SyncEngine re-INSERTs FTS rows on the next sync. + +### v3 — Add `attachment_text` column to `entities_fts` + +Detect the missing column via `PRAGMA table_info`; drop + recreate the FTS table. FTS5 has no `ALTER TABLE ADD COLUMN`. The local SyncEngine re-INSERTs FTS rows on the next sync, this time including `attachment_text` from each entity's `data.assets[].text`. + +### v4 — Schema cleanup to align with the worker schema + +Two concurrent changes that minimise the gratuitous local↔worker divergence: + +- **Drop `collection_name` from `entities_fts`.** Each local collection has its own SQLite file, so collection scoping is implicit — the column was written but never queried. FTS5 has no DROP COLUMN, so detect via `PRAGMA table_info` then drop+recreate. Final 7-column shape: `entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags, attachment_text` — same as the worker. SyncEngine re-INSERTs FTS rows on the next sync. + +- **Drop `AUTOINCREMENT` from `entities.id`.** The historical local table used `INTEGER PRIMARY KEY AUTOINCREMENT` while the worker uses plain `INTEGER PRIMARY KEY`; we never depended on the no-id-reuse guarantee that `AUTOINCREMENT` provides (FTS rows are deleted alongside their entity row before any new INSERT, so collisions can't occur). The migration detects the historical shape via `sqlite_master.sql`, builds a clone table with the new shape, copies all rows preserving their existing ids, drops the original, renames the clone, and rebuilds the indexes. Wrapped in `BEGIN/COMMIT` — partial failure rolls back. + +Final FTS bm25 weights used by `SearchIndexer.search()`: `title=10, tags=5, body=1, attachment=0.25` (UNINDEXED columns get weight `1.0` but never match). `snippet()` highlights column index 4 (content). Worker `bm25()` and `snippet()` are kept identical per the parity rule. + +## Worker schema (Cloudflare D1) + +Created and migrated by `publishCollections()` in `packages/cli/src/commands/publish.ts`. The runner uses an async D1 executor backed by `executeD1Query` / `queryD1Rows` from `wrangler-api.ts`. + +After local v4, the **`entities` table and `entities_fts` schema are identical between local and worker**. The only intentional difference left is: + +- `sync_errors` (per-entity sync failure journal) — **local-only**. Sync runs locally; the worker never syncs. + +Stale-file cleanup on R2 is done by listing the bucket directly (`listR2Objects` in `wrangler-api.ts`) and diffing against what was just uploaded — there is **no `r2_manifest` D1 table**. Tags and links are not separate tables either; they live inside `entities.data` JSON. + +Both runtimes share the same `entities` DDL (extracted into `migrations/shared.ts` as `ENTITIES_TABLE_DDL` + `ENTITIES_INDEX_DDL`) and the same FTS column shape. Adding a column to `entities` should mean editing one constant and adding a numbered migration to both lists. + +### v1 — Baseline + +`entities` table with the same columns/indexes as the local v1; `entities_fts` virtual table with `entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags`. + +### v2 — Add `attachment_text` column to `entities_fts` + +Same idea as local v3: detect, drop + recreate. After this migration, `publishCollections()` force-includes every entity in the FTS push set — the manifest delta isn't enough on its own, because the FTS table was just emptied. The constant `FTS_RESET_FROM_BELOW = 2` exported from `db/migrations/worker.ts` lets the publish flow distinguish "the migration just ran" from "we were already current". + +bm25 weights used by `worker/src/db/search.ts`: `title=10, tags=5, body=1, attachment=0.25` — kept identical to the local indexer (parity rule). + +## Adding a new migration + +Pseudo-code for a typical schema change. Replace the version numbers and identifiers as appropriate. + +```ts +// packages/core/src/db/migrations/local.ts +export const LOCAL_MIGRATIONS: SyncMigration[] = [ + // ...existing migrations, untouched... + { + id: 4, + description: "Short summary of the change.", + up: (db) => { + // Idempotent body. Read PRAGMAs to detect prior state if needed. + }, + }, +]; +``` + +```ts +// packages/core/src/db/migrations/worker.ts +export const WORKER_MIGRATIONS: AsyncMigration[] = [ + // ...existing migrations, untouched... + { + id: 3, + description: "Mirror of the local change for the published D1 schema.", + up: async (db) => { + // Async idempotent body. Use db.query() for PRAGMAs. + }, + }, +]; +``` + +Then add corresponding sections to the **Local schema** / **Worker schema** lists above explaining the change. + +## Resetting (development only) + +To force every migration to re-run during development: + +```sh +sqlite3 ~/.frozenink/collections//db/data.db "DELETE FROM metadata WHERE key='schema.version'" +# Migrations are idempotent so they're safe to re-apply. +``` + +For D1, drop the metadata row via `wrangler d1 execute --remote`. Production should never need this — the migration runner handles version transitions automatically. diff --git a/bun.lock b/bun.lock index 96f6ee2..914d18b 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,7 @@ }, "packages/cli": { "name": "@frozenink/cli", - "version": "0.1.0", + "version": "0.10.8", "bin": { "fink": "src/index.ts", }, @@ -56,11 +56,12 @@ "@frozenink/core": "workspace:*", "fast-xml-parser": "^4.5.1", "gemoji": "^8.1.0", + "yjs": "^13.6.30", }, }, "packages/desktop": { "name": "@frozenink/desktop", - "version": "0.1.0", + "version": "0.10.8", "dependencies": { "better-sqlite3": "^11.0.0", "electron-updater": "^6.3.9", @@ -1039,6 +1040,8 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + "isomorphic.js": ["isomorphic.js@0.2.5", "", {}, "sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw=="], + "jackspeak": ["jackspeak@3.4.3", "", { "dependencies": { "@isaacs/cliui": "^8.0.2" }, "optionalDependencies": { "@pkgjs/parseargs": "^0.11.0" } }, "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw=="], "jake": ["jake@10.9.4", "", { "dependencies": { "async": "^3.2.6", "filelist": "^1.0.4", "picocolors": "^1.1.1" }, "bin": { "jake": "bin/cli.js" } }, "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA=="], @@ -1077,6 +1080,8 @@ "lazystream": ["lazystream@1.0.1", "", { "dependencies": { "readable-stream": "^2.0.5" } }, "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw=="], + "lib0": ["lib0@0.2.117", "", { "dependencies": { "isomorphic.js": "^0.2.4" }, "bin": { "0serve": "bin/0serve.js", "0gentesthtml": "bin/gentesthtml.js", "0ecdsa-generate-keypair": "bin/0ecdsa-generate-keypair.js" } }, "sha512-DeXj9X5xDCjgKLU/7RR+/HQEVzuuEUiwldwOGsHK/sfAfELGWEyTcf0x+uOvCvK3O2zPmZePXWL85vtia6GyZw=="], + "linkify-it": ["linkify-it@5.0.0", "", { "dependencies": { "uc.micro": "^2.0.0" } }, "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ=="], "lodash": ["lodash@4.18.1", "", {}, "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q=="], @@ -1689,6 +1694,8 @@ "yauzl": ["yauzl@2.10.0", "", { "dependencies": { "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" } }, "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g=="], + "yjs": ["yjs@13.6.30", "", { "dependencies": { "lib0": "^0.2.99" } }, "sha512-vv/9h42eCMC81ZHDFswuu/MKzkl/vyq1BhaNGfHyOonwlG4CJbQF4oiBBJPvfdeCt/PlVDWh7Nov9D34YY09uQ=="], + "yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], "yoga-layout": ["yoga-layout@3.2.1", "", {}, "sha512-0LPOt3AxKqMdFBZA3HBAt/t/8vIKq7VaQYbuA8WxCgung+p9TVyKRYdpvCb80HcdTN2NkbIKbhNwKUfm3tQywQ=="], diff --git a/packages/cli/src/commands/add.ts b/packages/cli/src/commands/add.ts index cdbbcc2..8e30a17 100644 --- a/packages/cli/src/commands/add.ts +++ b/packages/cli/src/commands/add.ts @@ -35,6 +35,8 @@ export const addCommand = new Command("add") .option("--max-prs ", "Maximum pull requests to sync (for github)", parseInt) .option("--open-only", "Only sync open issues/PRs, delete closed ones (for github)") .option("--sync-entities ", "Comma-separated entity types to sync: issues,pages,users (for mantishub)") + .option("--conduit-storage ", "Path to Evernote v10 conduit-storage directory (for evernote)") + .option("--notebooks ", "Comma-separated notebook allowlist (for evernote)") .option("--credentials ", "Use a named credential set from ~/.frozenink/credentials.yml") .addHelpText("after", ` Examples: @@ -190,6 +192,21 @@ Examples: if (opts.includeDiffs) { config.includeDiffs = true; } + } else if (crawlerType === "evernote") { + if (opts.conduitStorage) { + const { resolve } = await import("path"); + const path = resolve(opts.conduitStorage); + config.conduitStoragePath = path; + if (!useNamedCreds) { + (credentials as Record).conduitStoragePath = path; + } + } + if (opts.notebooks) { + config.notebooks = (opts.notebooks as string) + .split(",") + .map((s: string) => s.trim()) + .filter(Boolean); + } } else if (crawlerType === "rss") { if (!opts.feedUrl) { console.error("RSS crawler requires --feed-url "); diff --git a/packages/cli/src/commands/clone.ts b/packages/cli/src/commands/clone.ts index f8af999..2202b73 100644 --- a/packages/cli/src/commands/clone.ts +++ b/packages/cli/src/commands/clone.ts @@ -224,6 +224,11 @@ Examples: // Index inline — content is already in memory, no storage read needed const [dbEntity] = colDb.select().from(entities).where(eq(entities.externalId, re.externalId)).all(); if (dbEntity) { + const reAssets = (re.data as Record | undefined)?.assets as Array<{ text?: string }> | undefined; + const attachmentText = (reAssets ?? []) + .map((a) => a.text) + .filter((t): t is string => Boolean(t && t.trim())) + .join("\n"); indexer.updateIndex({ id: dbEntity.id, externalId: re.externalId, @@ -231,6 +236,7 @@ Examples: title: derivedTitle ?? re.title, content: markdown, tags: re.tags ?? [], + attachmentText, }); } diff --git a/packages/cli/src/commands/daemon.ts b/packages/cli/src/commands/daemon.ts index de70887..0dc5486 100644 --- a/packages/cli/src/commands/daemon.ts +++ b/packages/cli/src/commands/daemon.ts @@ -14,7 +14,7 @@ import { spawnDetached, resolveCredentials, } from "@frozenink/core"; -import { createDefaultRegistry, gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme } from "@frozenink/crawlers"; +import { createDefaultRegistry, gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme, evernoteTheme } from "@frozenink/crawlers"; function getPidPath(): string { return join(getFrozenInkHome(), "daemon.pid"); @@ -46,6 +46,7 @@ async function runSyncLoop(intervalMs: number): Promise { themeEngine.register(gitTheme); themeEngine.register(mantisHubTheme); themeEngine.register(rssTheme); + themeEngine.register(evernoteTheme); for (const col of collectionRows) { const factory = registry.get(col.crawler); diff --git a/packages/cli/src/commands/generate.ts b/packages/cli/src/commands/generate.ts index 61bc1f6..0914ea7 100644 --- a/packages/cli/src/commands/generate.ts +++ b/packages/cli/src/commands/generate.ts @@ -20,7 +20,7 @@ import { } from "@frozenink/core"; import type { EntityData } from "@frozenink/core"; import { eq } from "drizzle-orm"; -import { gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme } from "@frozenink/crawlers"; +import { gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme, evernoteTheme } from "@frozenink/crawlers"; /** Write .yml config files for folders matching the theme's folderConfigs(). */ async function writeFolderConfigFiles( @@ -72,6 +72,7 @@ export function createGenerateThemeEngine(): ThemeEngine { themeEngine.register(gitTheme); themeEngine.register(mantisHubTheme); themeEngine.register(rssTheme); + themeEngine.register(evernoteTheme); return themeEngine; } @@ -220,7 +221,11 @@ export async function generateCollection( .where(eq(entities.id, entity.id)) .run(); - // Rebuild FTS index + // Rebuild FTS index — preserve any previously-extracted attachment text. + const attachmentText = (entityData.assets ?? []) + .map((a) => a.text) + .filter((t): t is string => Boolean(t && t.trim())) + .join("\n"); indexer.updateIndex({ id: entity.id, externalId: entity.externalId, @@ -228,6 +233,7 @@ export async function generateCollection( title, content: markdown, tags: entityData.tags ?? [], + attachmentText, }); // Rebuild entity out_links in data diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 16cdede..4e65a9a 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -84,6 +84,10 @@ Examples: const markdown = readFileSync(filePath, "utf-8"); + const attachmentText = (entityData.assets ?? []) + .map((a) => a.text) + .filter((t): t is string => Boolean(t && t.trim())) + .join("\n"); indexer.updateIndex({ id: entity.id, externalId: entity.externalId, @@ -91,6 +95,7 @@ Examples: title: entity.title, content: markdown, tags: entityData.tags ?? [], + attachmentText, }); indexed++; diff --git a/packages/cli/src/commands/management-api.ts b/packages/cli/src/commands/management-api.ts index 960142b..53b0709 100644 --- a/packages/cli/src/commands/management-api.ts +++ b/packages/cli/src/commands/management-api.ts @@ -38,6 +38,7 @@ import { gitTheme, mantisHubTheme, rssTheme, + evernoteTheme, } from "@frozenink/crawlers"; import { prepareCollection } from "./prepare"; import { createGenerateThemeEngine } from "./generate"; @@ -236,6 +237,7 @@ function createThemeEngine(): ThemeEngine { engine.register(gitTheme); engine.register(mantisHubTheme); engine.register(rssTheme); + engine.register(evernoteTheme); return engine; } @@ -671,6 +673,26 @@ export function handleManagementRequest(req: Request): Response | null { }); } + // --- Crawler-specific helpers --- + + // GET /api/crawlers/evernote/notebooks?path=... + if (path === "/api/crawlers/evernote/notebooks" && method === "GET") { + return handleAsync(async () => { + try { + const { listEvernoteNotebooks } = await import("@frozenink/crawlers"); + const url = new URL(req.url); + const conduitPath = url.searchParams.get("path") || undefined; + const notebooks = await listEvernoteNotebooks(conduitPath); + return jsonResponse({ notebooks }); + } catch (err) { + return jsonResponse( + { notebooks: [], error: err instanceof Error ? err.message : String(err) }, + 200, + ); + } + }); + } + // --- MCP tool links --- if (path === "/api/mcp/status" && method === "GET") { diff --git a/packages/cli/src/commands/prepare.ts b/packages/cli/src/commands/prepare.ts index c6d5044..b39a17d 100644 --- a/packages/cli/src/commands/prepare.ts +++ b/packages/cli/src/commands/prepare.ts @@ -87,6 +87,7 @@ function serializeFolderConfig(config: FolderConfig): string { lines.push(`hide: [${config.hide.join(", ")}]`); } if (config.showCount === true) lines.push("showCount: true"); + if (config.created_at_prefix === true) lines.push("created_at_prefix: true"); return lines.join("\n") + "\n"; } @@ -127,6 +128,30 @@ async function writeFolderConfigFiles( await storage.write(ymlPath, ymlContent); wrote = true; } + + // subdirConfig propagation — see SyncEngine.writeFolderConfigFiles for + // the full rationale. Serialises the parent's subdirConfig into every + // direct child's yml. + for (const dirPath of allDirs) { + const folderName = dirPath.split("/").pop()!; + const cfg = configs[folderName]; + if (!cfg?.subdirConfig) continue; + const desired = serializeFolderConfig(cfg.subdirConfig); + const prefix = `${dirPath}/`; + for (const childPath of allDirs) { + if (!childPath.startsWith(prefix)) continue; + const rel = childPath.slice(prefix.length); + if (!rel || rel.includes("/")) continue; + if (rel in configs) continue; + const ymlPath = `${childPath}/${rel}.yml`; + try { + const existing = await storage.read(ymlPath); + if (existing === desired) continue; + } catch { /* file doesn't exist yet */ } + await storage.write(ymlPath, desired); + wrote = true; + } + } } // Write source-derived subdirectory configs (e.g. from vault .folder.yml files). diff --git a/packages/cli/src/commands/publish.ts b/packages/cli/src/commands/publish.ts index 1838229..31d9bc3 100644 --- a/packages/cli/src/commands/publish.ts +++ b/packages/cli/src/commands/publish.ts @@ -304,6 +304,7 @@ export async function publishCollections( checkWranglerAuth, createD1, executeD1Query, + queryD1Rows, createR2Bucket, putR2Object, @@ -629,26 +630,53 @@ export async function publishCollections( if (!full && storedLocalManifestHash && storedLocalManifestHash === newLocalManifestHash) { onProgress("d1-build", "Local manifest unchanged since last publish — skipping D1 rebuild"); } else { - // Ensure schema exists. Idempotent so first publish creates tables and - // re-publish is a no-op. No DROPs — the manifest-driven delta below - // brings D1 in sync without wiping it. - const schemaInit: string[] = []; - schemaInit.push(`CREATE TABLE IF NOT EXISTS entities ( - id INTEGER PRIMARY KEY, - external_id TEXT NOT NULL, entity_type TEXT NOT NULL, - title TEXT NOT NULL, data TEXT NOT NULL DEFAULT '{}', - content_hash TEXT, - folder TEXT, - slug TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) -);`); - schemaInit.push("CREATE INDEX IF NOT EXISTS idx_entities_external ON entities(external_id);"); - schemaInit.push("CREATE INDEX IF NOT EXISTS idx_entities_folder ON entities(folder, slug);"); - schemaInit.push("CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(entity_type);"); - schemaInit.push("CREATE INDEX IF NOT EXISTS idx_entities_updated ON entities(updated_at);"); - schemaInit.push("CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags);"); - await executeD1Query(d1DatabaseId, schemaInit.join("\n")); + // Schema is owned by the migration runner — see + // `packages/core/src/db/migrations/worker.ts` and the SCHEMA.md + // entries that describe each version. The runner records the + // applied version in the `metadata` table; subsequent republishes + // see the version is current and short-circuit. + // + // Track the pre-migration version so we know whether the FTS table + // was just dropped + recreated. When it was, every FTS row needs + // to re-INSERT regardless of the manifest delta — the FTS table + // itself is empty. + const { runAsyncMigrations, WORKER_MIGRATIONS, FTS_RESET_FROM_BELOW } = + await import("@frozenink/core"); + const d1Executor = { + exec: (sql: string) => executeD1Query(d1DatabaseId, sql), + query: (sql: string) => queryD1Rows(d1DatabaseId, sql), + }; + // Read the version BEFORE the runner writes the new one so we can + // tell whether a reset migration ran in this call. + let preMigrationVersion = 0; + try { + const rows = await queryD1Rows<{ value: string }>( + d1DatabaseId, + "SELECT value FROM metadata WHERE key = 'schema.version'", + ); + preMigrationVersion = rows[0] ? Number.parseInt(rows[0].value, 10) || 0 : 0; + } catch { + // Metadata table missing on first publish — version stays 0. + } + const migrationResult = await runAsyncMigrations( + d1Executor, + WORKER_MIGRATIONS, + `d1:${d1DatabaseId}`, + ); + if (migrationResult.applied.length > 0) { + onProgress( + "d1-build", + `Applied D1 migrations: ${migrationResult.applied.join(", ")} (now at v${migrationResult.current})`, + ); + } + // FTS_RESET_FROM_BELOW is the version that drops + recreates + // entities_fts. When the pre-migration version sits below it and + // the current version is at or above, the FTS table was just + // emptied by a migration in this run and we need to re-INSERT + // every FTS row regardless of manifest delta. (Fresh DBs hit this + // path too — push is just the empty entity list, harmless.) + const needsFtsRebuild = preMigrationVersion < FTS_RESET_FROM_BELOW + && migrationResult.current >= FTS_RESET_FROM_BELOW; // Fetch remote manifest (empty on first publish). Manifest endpoint is // behind authMiddleware when PASSWORD_HASH is set; pass the plaintext @@ -682,7 +710,10 @@ export async function publishCollections( // prune: remote-only → DELETE on remote // --full forces every local entity into the push set so folder/slug // (and other non-content-hash) changes propagate to D1. - const pushExtIds = full + // When the FTS table was rebuilt (schema migration above), every + // FTS row needs to be re-INSERTed regardless of whether the entity's + // content_hash changed — D1 just dropped the table. + const pushExtIds = (full || needsFtsRebuild) ? localRows.map((r) => r.entity.externalId) : [ ...plan.entities.delete, @@ -733,6 +764,13 @@ export async function publishCollections( const entityDataParsed: EntityData = typeof entity.data === "object" ? entity.data as EntityData : JSON.parse(entity.data as string); const entityTagNames = (entityDataParsed.tags ?? []).join(" "); + // Concatenate every asset's OCR / extracted text so attachment + // contents become searchable on the published worker — same way + // SyncEngine builds the local FTS row. + const attachmentText = (entityDataParsed.assets ?? []) + .map((a) => a.text) + .filter((t): t is string => Boolean(t && t.trim())) + .join("\n"); let ftsContent = ""; const hasMarkdown = entity.folder != null && entity.slug != null; if (hasMarkdown && themeEngine.has(colCrawler)) { @@ -754,7 +792,7 @@ export async function publishCollections( if (ftsContent.length > MAX_FTS_CONTENT) ftsContent = ftsContent.slice(0, MAX_FTS_CONTENT); ftsValueRows.push( - `(${ftsEntityIdFor(entity.externalId)}, '${escapeSQL(entity.externalId)}', '${escapeSQL(entity.entityType)}', '${escapeSQL(entity.title)}', '${escapeSQL(ftsContent)}', '${escapeSQL(entityTagNames)}')`, + `(${ftsEntityIdFor(entity.externalId)}, '${escapeSQL(entity.externalId)}', '${escapeSQL(entity.entityType)}', '${escapeSQL(entity.title)}', '${escapeSQL(ftsContent)}', '${escapeSQL(entityTagNames)}', '${escapeSQL(attachmentText)}')`, ); } @@ -762,7 +800,7 @@ export async function publishCollections( // Bounded by byte size per INSERT to stay under D1's statement limit. const MAX_INSERT_BYTES = 80 * 1024; const entityPrefix = "INSERT OR REPLACE INTO entities (external_id, entity_type, title, data, content_hash, folder, slug, created_at, updated_at) VALUES "; - const ftsPrefix = "INSERT INTO entities_fts (entity_id, external_id, entity_type, title, content, tags) VALUES "; + const ftsPrefix = "INSERT INTO entities_fts (entity_id, external_id, entity_type, title, content, tags, attachment_text) VALUES "; pushMultiRowInserts(insertSql, entityPrefix, entityValueRows, MAX_INSERT_BYTES); pushMultiRowInserts(insertSql, ftsPrefix, ftsValueRows, MAX_INSERT_BYTES); } diff --git a/packages/cli/src/commands/pull.ts b/packages/cli/src/commands/pull.ts index f183cb2..cf51801 100644 --- a/packages/cli/src/commands/pull.ts +++ b/packages/cli/src/commands/pull.ts @@ -290,13 +290,20 @@ export async function pullCollection(collectionName: string, opts: PullCollectio content = await storage.read(`content/${re.markdownPath}`); } catch { /* ok */ } } + const reData = re.data as Record | undefined; + const reAssets = reData?.assets as Array<{ text?: string }> | undefined; + const attachmentText = (reAssets ?? []) + .map((a) => a.text) + .filter((t): t is string => Boolean(t && t.trim())) + .join("\n"); indexer.updateIndex({ id: dbEntity.id, externalId: re.externalId, entityType: re.entityType, title: re.title, content, - tags: ((re.data as Record)?.tags as string[] | undefined) ?? [], + tags: (reData?.tags as string[] | undefined) ?? [], + attachmentText, }); } for (const entityId of deletedEntityIds) { diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index af80bdc..7c6fa59 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -27,6 +27,7 @@ import { gitTheme, mantisHubTheme, rssTheme, + evernoteTheme, } from "@frozenink/crawlers"; import { startStdioServer } from "@frozenink/mcp"; import { eq, desc, inArray, and } from "drizzle-orm"; @@ -42,6 +43,7 @@ function createThemeEngine(): ThemeEngine { themeEngine.register(gitTheme); themeEngine.register(mantisHubTheme); themeEngine.register(rssTheme); + themeEngine.register(evernoteTheme); return themeEngine; } @@ -105,14 +107,14 @@ function matchesHidePattern(patterns: string[], filename: string): boolean { }); } -/** Parse a folder yml file (visible/sort/hide/showCount fields). */ -function readFolderConfig(dirPath: string): { visible?: boolean; sort?: "ASC" | "DESC"; hide?: string[]; showCount?: boolean } { +/** Parse a folder yml file (visible/sort/hide/showCount/created_at_prefix fields). */ +function readFolderConfig(dirPath: string): { visible?: boolean; sort?: "ASC" | "DESC"; hide?: string[]; showCount?: boolean; created_at_prefix?: boolean } { const folderName = basename(dirPath); const ymlPath = join(dirPath, `${folderName}.yml`); if (!existsSync(ymlPath)) return {}; try { const content = readFileSync(ymlPath, "utf-8"); - const config: { visible?: boolean; sort?: "ASC" | "DESC"; hide?: string[]; showCount?: boolean } = {}; + const config: { visible?: boolean; sort?: "ASC" | "DESC"; hide?: string[]; showCount?: boolean; created_at_prefix?: boolean } = {}; for (const line of content.split("\n")) { const m = line.match(/^(\w+):\s*(.+)$/); if (!m) continue; @@ -120,6 +122,7 @@ function readFolderConfig(dirPath: string): { visible?: boolean; sort?: "ASC" | if (key === "visible") config.visible = val.trim() !== "false"; if (key === "sort") config.sort = val.trim() === "DESC" ? "DESC" : "ASC"; if (key === "showCount") config.showCount = val.trim() === "true"; + if (key === "created_at_prefix") config.created_at_prefix = val.trim() === "true"; if (key === "hide") { // Parse inline YAML array: [item1, item2] or ["item1", "item2"] const arrayMatch = val.trim().match(/^\[(.+)\]$/); @@ -241,6 +244,7 @@ function buildFileTree( themeConfigs: Record = {}, themeRootConfig: FolderConfig = {}, labelFilesWithTitle: boolean = true, + sortKeyByPath: Map = new Map(), ): object[] { if (!existsSync(dirPath)) return []; @@ -269,7 +273,7 @@ function buildFileTree( const childYmlCfg = readFolderConfig(childDirPath); const childMerged: FolderConfig = { ...childThemeCfg, ...childYmlCfg }; if (childMerged.visible === false) continue; - const children = buildFileTree(childDirPath, titleByPath, relativePath, themeConfigs, themeRootConfig, labelFilesWithTitle); + const children = buildFileTree(childDirPath, titleByPath, relativePath, themeConfigs, themeRootConfig, labelFilesWithTitle, sortKeyByPath); // Hide directories with no (visible) files in their subtree so stale // empty folders on disk don't leak into the tree. if (countFiles(children) === 0) continue; @@ -306,9 +310,20 @@ function buildFileTree( } } - // Apply directory/file ordering by this folder's sort config. + // Apply directory/file ordering by this folder's sort config. Files sort + // by their effective sortKey (entity data → fall back to filename), so + // crawlers can opt notes into chronological order without baking the + // date into the filename. const sortedDirs = sortOrder === "DESC" ? dirs.slice().reverse() : dirs; - const sortedFiles = sortOrder === "DESC" ? files.slice().reverse() : files; + const fileEffectiveKey = (f: object): string => { + const node = f as { path: string; name: string }; + return sortKeyByPath.get(node.path) ?? node.name; + }; + const sortedFiles = files.slice().sort((a, b) => { + const ka = fileEffectiveKey(a); + const kb = fileEffectiveKey(b); + return sortOrder === "DESC" ? kb.localeCompare(ka) : ka.localeCompare(kb); + }); // Mark the first N expanded, rest explicitly collapsed. The UI defaults to // "expanded" when the flag is absent, so both states must be set. @@ -415,16 +430,20 @@ export function createApiServer( const contentDir = join(home, "collections", name, "content"); const titleByPath = new Map(); + const sortKeyByPath = new Map(); const dbPath = getCollectionDbPath(name); if (existsSync(dbPath)) { const colDb = getCollectionDb(dbPath); const rows = colDb - .select({ folder: entities.folder, slug: entities.slug, title: entities.title }) + .select({ folder: entities.folder, slug: entities.slug, title: entities.title, data: entities.data }) .from(entities) .all(); for (const row of rows) { const mdPath = entityMarkdownPath(row.folder, row.slug); - if (mdPath && row.title) titleByPath.set(mdPath, row.title); + if (!mdPath) continue; + if (row.title) titleByPath.set(mdPath, row.title); + const sortKey = (row.data as EntityData)?.sortKey; + if (typeof sortKey === "string" && sortKey) sortKeyByPath.set(mdPath, sortKey); } } @@ -432,7 +451,7 @@ export function createApiServer( const themeConfigs = themeEngine.getFolderConfigs(crawlerType); const themeRootConfig = themeEngine.getRootConfig(crawlerType); const labelWithTitle = themeEngine.labelFilesWithTitle(crawlerType); - const tree = buildFileTree(contentDir, titleByPath, "", themeConfigs, themeRootConfig, labelWithTitle); + const tree = buildFileTree(contentDir, titleByPath, "", themeConfigs, themeRootConfig, labelWithTitle, sortKeyByPath); return jsonResponse(tree); } diff --git a/packages/cli/src/commands/sync.ts b/packages/cli/src/commands/sync.ts index 0020cf1..389974f 100644 --- a/packages/cli/src/commands/sync.ts +++ b/packages/cli/src/commands/sync.ts @@ -16,7 +16,7 @@ import { resolveCredentials, } from "@frozenink/core"; import { sql } from "drizzle-orm"; -import { createDefaultRegistry, gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme } from "@frozenink/crawlers"; +import { createDefaultRegistry, gitHubTheme, obsidianTheme, gitTheme, mantisHubTheme, rssTheme, evernoteTheme } from "@frozenink/crawlers"; import { prepareCollection } from "./prepare"; import { createGenerateThemeEngine } from "./generate"; import { pullCollection } from "./pull"; @@ -80,6 +80,7 @@ Examples: themeEngine.register(gitTheme); themeEngine.register(mantisHubTheme); themeEngine.register(rssTheme); + themeEngine.register(evernoteTheme); const prepareThemeEngine = createGenerateThemeEngine(); diff --git a/packages/cli/src/commands/wrangler-api.ts b/packages/cli/src/commands/wrangler-api.ts index 9b28150..9e7b76e 100644 --- a/packages/cli/src/commands/wrangler-api.ts +++ b/packages/cli/src/commands/wrangler-api.ts @@ -221,6 +221,42 @@ export async function executeD1Query(databaseId: string, sql: string): Promise>( + databaseId: string, + sql: string, +): Promise { + const res = await fetchCloudflareApi(({ apiToken, accountId }) => ({ + url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/d1/database/${databaseId}/query`, + init: { + method: "POST", + headers: { + Authorization: `Bearer ${apiToken}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ sql }), + }, + })); + if (!res.ok) { + const text = await res.text(); + throw new Error(`D1 query failed: ${res.status} ${text.slice(0, 500)}`); + } + const data = (await res.json()) as { + success: boolean; + errors?: Array<{ code?: number; message: string }>; + result?: Array<{ results?: T[] }>; + }; + if (!data.success) { + const errs = (data.errors ?? []).map((e) => `[${e.code ?? "?"}] ${e.message}`).join("; "); + throw new Error(`D1 query failed: ${errs || "unknown error"}`); + } + return data.result?.[0]?.results ?? []; +} + export async function executeD1Command(dbName: string, sql: string): Promise { const { stdout } = await runWrangler(["d1", "execute", dbName, "--remote", "--command", sql, "--json", "--yes"]); return stdout; diff --git a/packages/cli/src/tui/components/CollectionList.tsx b/packages/cli/src/tui/components/CollectionList.tsx index c660fbf..f46e55c 100644 --- a/packages/cli/src/tui/components/CollectionList.tsx +++ b/packages/cli/src/tui/components/CollectionList.tsx @@ -28,6 +28,7 @@ import { gitTheme, mantisHubTheme, rssTheme, + evernoteTheme, } from "@frozenink/crawlers"; import { sql } from "drizzle-orm"; import { TextInput } from "./TextInput.js"; @@ -678,6 +679,7 @@ export function CollectionList({ themeEngine.register(gitTheme); themeEngine.register(mantisHubTheme); themeEngine.register(rssTheme); + themeEngine.register(evernoteTheme); const factory = registry.get(col.crawler); if (!factory) { setSyncProgress((p) => [...p, `No crawler for ${col.crawler}`]); setSyncStartTime(null); setMode("list"); return; } const crawler = factory(); diff --git a/packages/cli/src/tui/components/ExportView.tsx b/packages/cli/src/tui/components/ExportView.tsx index c78841f..2a8b32d 100644 --- a/packages/cli/src/tui/components/ExportView.tsx +++ b/packages/cli/src/tui/components/ExportView.tsx @@ -12,6 +12,7 @@ import { gitTheme, mantisHubTheme, rssTheme, + evernoteTheme, } from "@frozenink/crawlers"; import { TextInput } from "./TextInput.js"; @@ -99,6 +100,7 @@ export function ExportView({ themeEngine.register(gitTheme); themeEngine.register(mantisHubTheme); themeEngine.register(rssTheme); + themeEngine.register(evernoteTheme); } await exportStaticSite({ diff --git a/packages/cli/src/tui/sync-jobs.ts b/packages/cli/src/tui/sync-jobs.ts index 44e8fe3..02f65e5 100644 --- a/packages/cli/src/tui/sync-jobs.ts +++ b/packages/cli/src/tui/sync-jobs.ts @@ -22,6 +22,7 @@ import { gitTheme, mantisHubTheme, rssTheme, + evernoteTheme, } from "@frozenink/crawlers"; import { pullCollection } from "../commands/pull.js"; @@ -102,6 +103,7 @@ function createThemeEngine(): ThemeEngine { engine.register(gitTheme); engine.register(mantisHubTheme); engine.register(rssTheme); + engine.register(evernoteTheme); return engine; } diff --git a/packages/core/src/crawler/interface.ts b/packages/core/src/crawler/interface.ts index 1270077..8fd1a6b 100644 --- a/packages/core/src/crawler/interface.ts +++ b/packages/core/src/crawler/interface.ts @@ -17,11 +17,25 @@ export interface CrawlerEntityData { contentHash?: string; url?: string; tags?: string[]; + /** + * Optional sort key recorded on the entity. When the file-tree's effective + * folder config sorts by this key, entries are ordered lexicographically + * by sortKey (then by title as a tiebreaker). Crawlers without a natural + * order can leave this unset — callers fall back to the entity title. + */ + sortKey?: string; attachments?: { filename: string; mimeType: string; content: Buffer | Uint8Array; storagePath?: string; + /** + * Optional extracted / OCR text for this attachment. Stored on + * `EntityData.assets[i].text` and indexed in the FTS `attachment_text` + * column so contents inside images and PDFs become searchable without + * polluting the rendered note body. + */ + text?: string; }[]; relations?: { targetExternalId: string; diff --git a/packages/core/src/crawler/sync-engine.ts b/packages/core/src/crawler/sync-engine.ts index f0c89aa..92d0572 100644 --- a/packages/core/src/crawler/sync-engine.ts +++ b/packages/core/src/crawler/sync-engine.ts @@ -21,6 +21,7 @@ function serializeFolderConfig(config: FolderConfig): string { lines.push(`hide: [${config.hide.join(", ")}]`); } if (config.showCount === true) lines.push("showCount: true"); + if (config.created_at_prefix === true) lines.push("created_at_prefix: true"); return lines.join("\n") + "\n"; } @@ -33,6 +34,16 @@ function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } +/** Concatenate per-attachment OCR/extracted text for the FTS attachment_text column. */ +function collectAttachmentText(entityData: Pick): string { + if (!entityData.attachments?.length) return ""; + const parts: string[] = []; + for (const att of entityData.attachments) { + if (att.text && att.text.trim()) parts.push(att.text); + } + return parts.join("\n"); +} + function withEntityContext(err: unknown, entity: Pick): Error { const context = `entity type=${entity.entityType} id=${entity.externalId}`; const message = errorMessage(err); @@ -91,8 +102,16 @@ export function extractWikilinks(markdown: string, sourceFilePath?: string): str /** Empty = no extension filter (all file types allowed). */ const DEFAULT_ASSET_EXTENSIONS: string[] = []; -/** Default max asset size: 10 MB in KB. */ -const DEFAULT_ASSET_MAX_SIZE_KB = 10240; +/** + * Default max asset size: 100 MB in KB. PDFs and image-heavy documents + * routinely exceed the previous 10 MB cap, especially for sources like + * Evernote where users attach scanned bills, mortgage statements, etc. + * Skipped attachments leave a broken `![…](…)` reference in the rendered + * markdown, so a too-conservative cap silently corrupts the output. + * Override per collection by setting `assets.maxSize` (KB) in the + * collection's yml. + */ +const DEFAULT_ASSET_MAX_SIZE_KB = 100 * 1024; /** * Per-entity retry cap. Once an entity has failed this many times across @@ -574,6 +593,7 @@ export class SyncEngine { source: entityData.data as Record, url: entityData.url ?? null, tags: entityData.tags ?? [], + ...(entityData.sortKey !== undefined ? { sortKey: entityData.sortKey } : {}), }; // Update entity @@ -615,6 +635,7 @@ export class SyncEngine { title: entityData.title, content: markdown, tags: entityData.tags ?? [], + attachmentText: collectAttachmentText(entityData), }); return { created: 0, updated: 1 }; @@ -636,6 +657,7 @@ export class SyncEngine { source: entityData.data as Record, url: entityData.url ?? null, tags: entityData.tags ?? [], + ...(entityData.sortKey !== undefined ? { sortKey: entityData.sortKey } : {}), }; // Insert entity @@ -682,6 +704,7 @@ export class SyncEngine { title: entityData.title, content: markdown, tags: entityData.tags ?? [], + attachmentText: collectAttachmentText(entityData), }); return { created: 1, updated: 0 }; @@ -714,7 +737,7 @@ export class SyncEngine { return; } - const assetEntries: Array<{ filename: string; mimeType: string; storagePath: string; hash: string }> = []; + const assetEntries: Array<{ filename: string; mimeType: string; storagePath: string; hash: string; text?: string }> = []; for (const att of entityData.attachments) { const storagePath = att.storagePath ?? `${this.markdownBasePath}/${entityData.entityType}/assets/${att.filename}`; @@ -736,6 +759,7 @@ export class SyncEngine { mimeType: att.mimeType, storagePath, hash, + ...(att.text ? { text: att.text } : {}), }); } @@ -975,7 +999,11 @@ export class SyncEngine { this.syncLinks(row.id, markdown, filePath); this.recomputeHash(row.id); - // Update search index + // Update search index — preserve any previously-extracted attachment text. + const storedAssetText = (rowData.assets ?? []) + .map((a) => a.text) + .filter((t): t is string => Boolean(t && t.trim())) + .join("\n"); this.searchIndexer.updateIndex({ id: row.id, externalId: row.externalId, @@ -983,6 +1011,7 @@ export class SyncEngine { title, content: markdown, tags: [], + attachmentText: storedAssetText, }); } catch (err) { throw withEntityContext(err, { entityType: row.entityType, externalId: row.externalId }); @@ -1101,6 +1130,32 @@ export class SyncEngine { await this.storage.write(ymlPath, serializeFolderConfig(configs[folderName])); } + // subdirConfig propagation: for every dir whose theme config carries a + // `subdirConfig`, serialise that config into each direct child's + // `/.yml`. This lets themes with dynamic per-context + // folders (e.g. one subfolder per Evernote notebook) declare a uniform + // default — sort, created_at_prefix, hide, etc. — without needing to + // know the child folder names ahead of time. + for (const dirPath of allDirs) { + const folderName = dirPath.split("/").pop()!; + const cfg = configs[folderName]; + if (!cfg?.subdirConfig) continue; + const desired = serializeFolderConfig(cfg.subdirConfig); + const prefix = `${dirPath}/`; + for (const childPath of allDirs) { + if (!childPath.startsWith(prefix)) continue; + const rel = childPath.slice(prefix.length); + if (!rel || rel.includes("/")) continue; + if (rel in configs) continue; + const ymlPath = `${childPath}/${rel}.yml`; + try { + const existing = await this.storage.read(ymlPath); + if (existing === desired) continue; + } catch { /* file doesn't exist yet */ } + await this.storage.write(ymlPath, desired); + } + } + // Root content.yml is written only by prepare (which merges theme rootConfig // with collection-level hide patterns). The yml survives reconcile (skipped above). } diff --git a/packages/core/src/db/__tests__/migrations.test.ts b/packages/core/src/db/__tests__/migrations.test.ts new file mode 100644 index 0000000..355483b --- /dev/null +++ b/packages/core/src/db/__tests__/migrations.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it, beforeEach } from "bun:test"; +import { Database } from "bun:sqlite"; +import { + LOCAL_MIGRATIONS, + runSyncMigrations, + SCHEMA_VERSION_KEY, + _clearMigrationCacheForTests, +} from "../migrations"; + +function open(): Database { + return new Database(":memory:"); +} + +describe("runSyncMigrations", () => { + beforeEach(() => { + _clearMigrationCacheForTests(); + }); + + it("brings a fresh DB up to the latest version and records it", () => { + const db = open(); + const result = runSyncMigrations(db, LOCAL_MIGRATIONS); + expect(result.current).toBe(LOCAL_MIGRATIONS[LOCAL_MIGRATIONS.length - 1].id); + expect(result.applied).toEqual(LOCAL_MIGRATIONS.map((m) => m.id)); + const row = db + .prepare("SELECT value FROM metadata WHERE key = ?") + .get(SCHEMA_VERSION_KEY) as { value: string }; + expect(row.value).toBe(String(LOCAL_MIGRATIONS[LOCAL_MIGRATIONS.length - 1].id)); + }); + + it("is a no-op when the DB is already at the latest version", () => { + const db = open(); + runSyncMigrations(db, LOCAL_MIGRATIONS); + const second = runSyncMigrations(db, LOCAL_MIGRATIONS); + expect(second.applied).toEqual([]); + }); + + it("re-verifies even when the same cacheKey is reused (no in-process cache)", () => { + // Each call always hits the underlying DB. The cacheKey parameter is + // kept for API symmetry but is intentionally ignored — caching across + // file recreations is a correctness hazard. + const db1 = open(); + runSyncMigrations(db1, LOCAL_MIGRATIONS, "shared"); + + const db2 = open(); + const result = runSyncMigrations(db2, LOCAL_MIGRATIONS, "shared"); + expect(result.applied).toEqual(LOCAL_MIGRATIONS.map((m) => m.id)); + // db2 was a fresh empty DB; the runner created tables on it. + const tables = db2 + .prepare("SELECT name FROM sqlite_master WHERE type='table'") + .all() as Array<{ name: string }>; + expect(tables.find((t) => t.name === "metadata")).toBeDefined(); + expect(tables.find((t) => t.name === "entities")).toBeDefined(); + }); + + it("only applies migrations newer than the recorded version", () => { + const db = open(); + // Pretend we're already at v1: create the metadata table and stamp it. + db.exec("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);"); + db.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)").run( + SCHEMA_VERSION_KEY, + "1", + ); + LOCAL_MIGRATIONS[0].up(db); // simulate the v1 effects + + const result = runSyncMigrations(db, LOCAL_MIGRATIONS); + expect(result.current).toBe(LOCAL_MIGRATIONS[LOCAL_MIGRATIONS.length - 1].id); + expect(result.applied).toEqual(LOCAL_MIGRATIONS.slice(1).map((m) => m.id)); + }); + + it("FTS table reaches the final 7-column shape after all migrations", () => { + const db = open(); + runSyncMigrations(db, LOCAL_MIGRATIONS); + const cols = db + .prepare("PRAGMA table_info(entities_fts)") + .all() as Array<{ name: string }>; + const names = cols.map((c) => c.name); + // collection_name was dropped in v4 — never queried, redundant since + // each collection has its own DB file. + expect(names).not.toContain("collection_name"); + expect(names).toContain("entity_id"); + expect(names).toContain("title"); + expect(names).toContain("content"); + expect(names).toContain("tags"); + expect(names).toContain("attachment_text"); + }); + + it("entities table loses AUTOINCREMENT after v4 (matches worker shape)", () => { + const db = open(); + runSyncMigrations(db, LOCAL_MIGRATIONS); + const row = db + .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='entities'") + .get() as { sql: string }; + expect(row.sql).not.toContain("AUTOINCREMENT"); + expect(row.sql).toContain("INTEGER PRIMARY KEY"); + }); + + it("v4 preserves entity row data (id, columns) when rebuilding the table", () => { + const db = open(); + // Run v1+v2+v3 (skip v4 by simulating an at-v3 DB), insert real data, then run v4. + db.exec("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);"); + LOCAL_MIGRATIONS.slice(0, 3).forEach((m) => m.up(db)); + db.prepare("INSERT INTO metadata (key, value) VALUES ('schema.version', '3')").run(); + + db.prepare( + "INSERT INTO entities (external_id, entity_type, title, data) VALUES (?, ?, ?, ?)", + ).run("foo-1", "note", "Hello", '{"source":{}}'); + db.prepare( + "INSERT INTO entities (external_id, entity_type, title, data) VALUES (?, ?, ?, ?)", + ).run("foo-2", "note", "World", '{"source":{}}'); + + const before = db.prepare("SELECT id, external_id, title FROM entities ORDER BY id").all(); + + runSyncMigrations(db, LOCAL_MIGRATIONS); + + const after = db.prepare("SELECT id, external_id, title FROM entities ORDER BY id").all(); + expect(after).toEqual(before); + // And the indexes survive the rebuild. + const idxs = db + .prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='entities'") + .all() as Array<{ name: string }>; + const idxNames = idxs.map((i) => i.name); + expect(idxNames).toContain("idx_entities_external"); + expect(idxNames).toContain("idx_entities_folder"); + }); +}); diff --git a/packages/core/src/db/client.ts b/packages/core/src/db/client.ts index 85ada52..86e12d9 100644 --- a/packages/core/src/db/client.ts +++ b/packages/core/src/db/client.ts @@ -1,6 +1,7 @@ import { openDatabase } from "../compat/sqlite"; import { isBun } from "../compat/runtime"; import * as collectionSchema from "./collection-schema"; +import { runSyncMigrations, LOCAL_MIGRATIONS } from "./migrations"; const COLLECTION_KEY_RE = /^[a-zA-Z0-9_-]+$/; @@ -18,77 +19,21 @@ function createDrizzle(sqlite: any) { return drizzle(sqlite, { schema: collectionSchema }); } +/** + * Open (or create) a collection's SQLite database and run any pending + * schema migrations from `LOCAL_MIGRATIONS`. The migration runner caches + * the verified version in-memory, so subsequent calls in the same process + * are effectively free. + * + * Schema changes go through migrations — never inline `CREATE TABLE` / + * `ALTER TABLE` here. See `SCHEMA.md` for the migration list. + */ export function getCollectionDb(dbPath: string) { const sqlite = openDatabase(dbPath); sqlite.exec("PRAGMA journal_mode = WAL;"); sqlite.exec("PRAGMA foreign_keys = ON;"); - const db = createDrizzle(sqlite); + runSyncMigrations(sqlite, LOCAL_MIGRATIONS, dbPath); - // Create tables if they don't exist - sqlite.exec(` - CREATE TABLE IF NOT EXISTS entities ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - external_id TEXT NOT NULL, - entity_type TEXT NOT NULL, - title TEXT NOT NULL, - data TEXT NOT NULL DEFAULT '{}', - content_hash TEXT, - folder TEXT, - slug TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - `); - - // Safe schema upgrade for existing DBs - for (const col of ["folder TEXT", "slug TEXT"]) { - try { sqlite.exec(`ALTER TABLE entities ADD COLUMN ${col}`); } catch {} - } - - // Ensure indexes exist (safe on new and existing DBs) - sqlite.exec("CREATE INDEX IF NOT EXISTS idx_entities_external ON entities(external_id);"); - sqlite.exec("CREATE INDEX IF NOT EXISTS idx_entities_folder ON entities(folder, slug);"); - sqlite.exec("CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(entity_type);"); - sqlite.exec("CREATE INDEX IF NOT EXISTS idx_entities_updated ON entities(updated_at);") - - // Per-entity sync failure journal. Created lazily on first DB open so existing - // collections gain it without an explicit migration step. - sqlite.exec(` - CREATE TABLE IF NOT EXISTS sync_errors ( - external_id TEXT PRIMARY KEY, - entity_type TEXT NOT NULL, - error TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 1, - first_seen_at TEXT NOT NULL DEFAULT (datetime('now')), - last_seen_at TEXT NOT NULL DEFAULT (datetime('now')) - ); - `); - - // One-time backfill: populate folder/slug from data.markdown_path for existing rows - const needsBackfill = sqlite.prepare( - "SELECT id, json_extract(data, '$.markdown_path') as mp FROM entities WHERE folder IS NULL AND json_extract(data, '$.markdown_path') IS NOT NULL LIMIT 500", - ); - const updateFolderSlug = sqlite.prepare( - "UPDATE entities SET folder = ?, slug = ? WHERE id = ?", - ); - const stripOldFields = sqlite.prepare( - "UPDATE entities SET data = json_remove(data, '$.markdown_path', '$.markdown_mtime', '$.markdown_size') WHERE json_extract(data, '$.markdown_path') IS NOT NULL OR json_extract(data, '$.markdown_mtime') IS NOT NULL", - ); - - let batch: Array<{ id: number; mp: string }>; - do { - batch = needsBackfill.all() as Array<{ id: number; mp: string }>; - for (const row of batch) { - const lastSlash = row.mp.lastIndexOf("/"); - const folder = lastSlash >= 0 ? row.mp.slice(0, lastSlash) : ""; - const slug = row.mp.slice(lastSlash + 1).replace(/\.md$/, ""); - updateFolderSlug.run(folder, slug, row.id); - } - } while (batch.length > 0); - - // Strip removed fields from data JSON - stripOldFields.run(); - - return db; + return createDrizzle(sqlite); } diff --git a/packages/core/src/db/collection-schema.ts b/packages/core/src/db/collection-schema.ts index 1ff49e2..92655a7 100644 --- a/packages/core/src/db/collection-schema.ts +++ b/packages/core/src/db/collection-schema.ts @@ -5,9 +5,24 @@ export interface EntityData { source: Record; out_links?: string[]; in_links?: string[]; - assets?: Array<{ filename: string; mimeType: string; storagePath: string; hash: string }>; + assets?: Array<{ + filename: string; + mimeType: string; + storagePath: string; + hash: string; + /** Extracted / OCR text used to make attachment contents full-text searchable. */ + text?: string; + }>; url?: string | null; tags?: string[] | null; + /** + * Optional explicit sort key for the file-tree view. When unset, callers + * fall back to the entity's title (lex sort by title). Crawlers that have + * a more meaningful order (e.g. timestamp-based, like Evernote's last- + * updated date) emit a sortable string here so the tree can show notes + * in chronological order without relying on filename hacks. + */ + sortKey?: string; } /** Reconstruct the relative markdown path from folder/slug columns. */ diff --git a/packages/core/src/db/index.ts b/packages/core/src/db/index.ts index 4af57b6..7ef1f3e 100644 --- a/packages/core/src/db/index.ts +++ b/packages/core/src/db/index.ts @@ -7,3 +7,16 @@ export { writeCollectionConfigMirror, } from "./metadata"; export type { SyncStateSnapshot, SyncStateUpdate } from "./metadata"; +export { + runSyncMigrations, + runAsyncMigrations, + LOCAL_MIGRATIONS, + WORKER_MIGRATIONS, + FTS_RESET_FROM_BELOW, + SCHEMA_VERSION_KEY, + type SyncMigration, + type AsyncMigration, + type SyncMigrationDb, + type AsyncMigrationDb, + type MigrationResult, +} from "./migrations"; diff --git a/packages/core/src/db/migrations/index.ts b/packages/core/src/db/migrations/index.ts new file mode 100644 index 0000000..35bf0e4 --- /dev/null +++ b/packages/core/src/db/migrations/index.ts @@ -0,0 +1,15 @@ +export type { + AsyncMigration, + AsyncMigrationDb, + SyncMigration, + SyncMigrationDb, +} from "./types"; +export { SCHEMA_VERSION_KEY } from "./types"; +export { + runSyncMigrations, + runAsyncMigrations, + _clearMigrationCacheForTests, + type MigrationResult, +} from "./runner"; +export { LOCAL_MIGRATIONS } from "./local"; +export { WORKER_MIGRATIONS, FTS_RESET_FROM_BELOW } from "./worker"; diff --git a/packages/core/src/db/migrations/local.ts b/packages/core/src/db/migrations/local.ts new file mode 100644 index 0000000..c4f816f --- /dev/null +++ b/packages/core/src/db/migrations/local.ts @@ -0,0 +1,246 @@ +import type { SyncMigration } from "./types"; +import { ENTITIES_INDEX_DDL } from "./shared"; + +/** + * Schema migrations for the **local** SQLite collection database + * (`~/.frozenink/collections//db/data.db`). Numbered, append-only — + * never edit a published migration; add a new one instead. + * + * Document each change in `SCHEMA.md` at the repo root. Worker (D1) + * migrations live in `./worker.ts`; keep the two in lockstep per the + * "Local ↔ worker parity rule" in AGENTS.md. + */ +export const LOCAL_MIGRATIONS: SyncMigration[] = [ + { + id: 1, + description: + "Baseline: entities table (with folder/slug + indexes), sync_errors journal, metadata table, legacy markdown_path → folder/slug backfill.", + up: (db) => { + db.exec(` + CREATE TABLE IF NOT EXISTS entities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + external_id TEXT NOT NULL, + entity_type TEXT NOT NULL, + title TEXT NOT NULL, + data TEXT NOT NULL DEFAULT '{}', + content_hash TEXT, + folder TEXT, + slug TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `); + + // Pre-baseline DBs predated folder/slug; ALTER ADD COLUMN is idempotent + // via try/catch since SQLite has no `IF NOT EXISTS` for columns. + for (const col of ["folder TEXT", "slug TEXT"]) { + try { + db.exec(`ALTER TABLE entities ADD COLUMN ${col}`); + } catch { + // column already present + } + } + + db.exec( + "CREATE INDEX IF NOT EXISTS idx_entities_external ON entities(external_id);", + ); + db.exec( + "CREATE INDEX IF NOT EXISTS idx_entities_folder ON entities(folder, slug);", + ); + db.exec( + "CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(entity_type);", + ); + db.exec( + "CREATE INDEX IF NOT EXISTS idx_entities_updated ON entities(updated_at);", + ); + + db.exec(` + CREATE TABLE IF NOT EXISTS sync_errors ( + external_id TEXT PRIMARY KEY, + entity_type TEXT NOT NULL, + error TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 1, + first_seen_at TEXT NOT NULL DEFAULT (datetime('now')), + last_seen_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `); + + // One-time backfill: migrate legacy `data.markdown_path` rows into + // structured `folder` + `slug` columns. Idempotent — only touches + // rows that still have markdown_path. Runs in 500-row batches so + // very large legacy DBs don't blow out the prepared-statement cache. + const needsBackfill = db.prepare( + "SELECT id, json_extract(data, '$.markdown_path') as mp FROM entities WHERE folder IS NULL AND json_extract(data, '$.markdown_path') IS NOT NULL LIMIT 500", + ); + const updateFolderSlug = db.prepare( + "UPDATE entities SET folder = ?, slug = ? WHERE id = ?", + ); + const stripOldFields = db.prepare( + "UPDATE entities SET data = json_remove(data, '$.markdown_path', '$.markdown_mtime', '$.markdown_size') WHERE json_extract(data, '$.markdown_path') IS NOT NULL OR json_extract(data, '$.markdown_mtime') IS NOT NULL", + ); + + let batch: Array<{ id: number; mp: string }>; + do { + batch = needsBackfill.all() as Array<{ id: number; mp: string }>; + for (const row of batch) { + const lastSlash = row.mp.lastIndexOf("/"); + const folder = lastSlash >= 0 ? row.mp.slice(0, lastSlash) : ""; + const slug = row.mp.slice(lastSlash + 1).replace(/\.md$/, ""); + updateFolderSlug.run(folder, slug, row.id); + } + } while (batch.length > 0); + + stripOldFields.run(); + }, + }, + { + id: 2, + description: + "Add `entities_fts` virtual table (FTS5) with collection_name / entity_id / external_id / entity_type / title / content / tags. Includes drop+recreate when an older 6-column FTS (no collection_name) is detected.", + up: (db) => { + // Detect a pre-existing legacy FTS table that lacks the + // collection_name column. FTS5 doesn't support ALTER TABLE ADD + // COLUMN, so the only path is drop + recreate. The SyncEngine + // re-INSERTs FTS rows on the next sync. + try { + const cols = db.prepare("PRAGMA table_info(entities_fts)").all() as Array<{ name: string }>; + const hasCollectionName = cols.some((c) => c.name === "collection_name"); + if (cols.length > 0 && !hasCollectionName) { + db.exec("DROP TABLE IF EXISTS entities_fts"); + } + } catch { + // Table doesn't exist yet — fresh DB. + } + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5( + collection_name UNINDEXED, + entity_id UNINDEXED, + external_id UNINDEXED, + entity_type UNINDEXED, + title, + content, + tags + ); + `); + }, + }, + { + id: 3, + description: + "Add `attachment_text` column to `entities_fts` for OCR'd / extracted text from images and PDFs. Drop+recreate (FTS5 has no ALTER ADD COLUMN); SyncEngine re-INSERTs FTS rows on the next sync.", + up: (db) => { + let needsRebuild = false; + try { + const cols = db.prepare("PRAGMA table_info(entities_fts)").all() as Array<{ name: string }>; + if (cols.length > 0 && !cols.some((c) => c.name === "attachment_text")) { + needsRebuild = true; + } + } catch { + // Table missing — IF NOT EXISTS below creates the new shape. + } + if (needsRebuild) { + db.exec("DROP TABLE IF EXISTS entities_fts"); + } + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5( + collection_name UNINDEXED, + entity_id UNINDEXED, + external_id UNINDEXED, + entity_type UNINDEXED, + title, + content, + tags, + attachment_text + ); + `); + }, + }, + { + id: 4, + description: + "Schema cleanup to align with the worker schema: drop `collection_name` from `entities_fts` (it was never queried), and drop `AUTOINCREMENT` from `entities.id` (worker has plain INTEGER PRIMARY KEY). FTS row repopulation happens on next sync.", + up: (db) => { + // ── Part 1: drop `collection_name` from entities_fts ── + // FTS5 has no DROP COLUMN, so detect + recreate. Idempotent: skip + // if the column is already absent (fresh DB, or migration already + // ran on this DB). + let ftsNeedsRebuild = false; + try { + const cols = db + .prepare("PRAGMA table_info(entities_fts)") + .all() as Array<{ name: string }>; + if (cols.length > 0 && cols.some((c) => c.name === "collection_name")) { + ftsNeedsRebuild = true; + } + } catch { + // Table missing; CREATE IF NOT EXISTS below handles it. + } + if (ftsNeedsRebuild) { + db.exec("DROP TABLE IF EXISTS entities_fts"); + } + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5( + entity_id UNINDEXED, + external_id UNINDEXED, + entity_type UNINDEXED, + title, + content, + tags, + attachment_text + ); + `); + + // ── Part 2: drop AUTOINCREMENT from entities ── + // ALTER TABLE can't change PRIMARY KEY semantics. Detect via + // sqlite_master.sql — if the historical CREATE statement contains + // "AUTOINCREMENT", rebuild the table by copying rows into a + // correctly-shaped clone and renaming. Idempotent: skips when the + // current schema already lacks AUTOINCREMENT (worker-shape DB). + const masterRow = db + .prepare( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='entities'", + ) + .get() as { sql: string } | undefined; + if (masterRow && /AUTOINCREMENT/i.test(masterRow.sql)) { + // Wrap in a transaction so a partial failure doesn't leave the + // DB without an `entities` table. + db.exec("BEGIN"); + try { + db.exec(` + CREATE TABLE entities_v4_new ( + id INTEGER PRIMARY KEY, + external_id TEXT NOT NULL, + entity_type TEXT NOT NULL, + title TEXT NOT NULL, + data TEXT NOT NULL DEFAULT '{}', + content_hash TEXT, + folder TEXT, + slug TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + `); + db.exec(` + INSERT INTO entities_v4_new ( + id, external_id, entity_type, title, data, + content_hash, folder, slug, created_at, updated_at + ) + SELECT id, external_id, entity_type, title, data, + content_hash, folder, slug, created_at, updated_at + FROM entities; + `); + db.exec("DROP TABLE entities"); + db.exec("ALTER TABLE entities_v4_new RENAME TO entities"); + // Rebuild indexes — DROP TABLE removed them along with the table. + for (const sql of ENTITIES_INDEX_DDL) { + db.exec(sql); + } + db.exec("COMMIT"); + } catch (err) { + db.exec("ROLLBACK"); + throw err; + } + } + }, + }, +]; diff --git a/packages/core/src/db/migrations/runner.ts b/packages/core/src/db/migrations/runner.ts new file mode 100644 index 0000000..0639d1d --- /dev/null +++ b/packages/core/src/db/migrations/runner.ts @@ -0,0 +1,132 @@ +import type { + AsyncMigration, + AsyncMigrationDb, + SyncMigration, + SyncMigrationDb, +} from "./types"; +import { SCHEMA_VERSION_KEY } from "./types"; + +/* + * The hot path is just `ensureMetadataTable` + a single primary-key SELECT + * against `metadata` — well under a microsecond on local SQLite and a + * single D1 round-trip on the worker (which only happens at publish time + * anyway). We deliberately don't cache the verified version in-process + * because tests routinely delete + recreate DB files at the same path, + * which a cache would mis-treat as "already migrated". + */ + +function ensureMetadataTableSync(db: SyncMigrationDb): void { + db.exec( + "CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);", + ); +} + +async function ensureMetadataTableAsync(db: AsyncMigrationDb): Promise { + await db.exec( + "CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);", + ); +} + +function readSchemaVersionSync(db: SyncMigrationDb): number { + const row = db.prepare( + `SELECT value FROM metadata WHERE key = '${SCHEMA_VERSION_KEY}'`, + ).get() as { value: string } | undefined; + if (!row) return 0; + const n = Number.parseInt(row.value, 10); + return Number.isFinite(n) ? n : 0; +} + +async function readSchemaVersionAsync(db: AsyncMigrationDb): Promise { + const rows = await db.query<{ value: string }>( + `SELECT value FROM metadata WHERE key = '${SCHEMA_VERSION_KEY}'`, + ); + if (!rows[0]) return 0; + const n = Number.parseInt(rows[0].value, 10); + return Number.isFinite(n) ? n : 0; +} + +function writeSchemaVersionSync(db: SyncMigrationDb, version: number): void { + db.prepare( + `INSERT OR REPLACE INTO metadata (key, value) VALUES ('${SCHEMA_VERSION_KEY}', ?)`, + ).run(String(version)); +} + +async function writeSchemaVersionAsync( + db: AsyncMigrationDb, + version: number, +): Promise { + await db.exec( + `INSERT OR REPLACE INTO metadata (key, value) VALUES ('${SCHEMA_VERSION_KEY}', '${version}');`, + ); +} + +/** Result of a migration pass. */ +export interface MigrationResult { + /** Schema version after the run. */ + current: number; + /** Ids that were applied during this run (empty if already up-to-date). */ + applied: number[]; +} + +/** + * Run every pending migration against a synchronous SQLite handle and + * record the new schema version in the `metadata` table. Cheap (one + * SELECT) when the DB is already current. + * + * `cacheKey` is accepted for API symmetry with `runAsyncMigrations` but + * currently unused; included so callers can pre-thread a key for any + * future opt-in caching. + */ +export function runSyncMigrations( + db: SyncMigrationDb, + migrations: SyncMigration[], + _cacheKey?: string, +): MigrationResult { + if (migrations.length === 0) return { current: 0, applied: [] }; + const latest = migrations[migrations.length - 1].id; + + ensureMetadataTableSync(db); + const current = readSchemaVersionSync(db); + if (current === latest) { + return { current, applied: [] }; + } + + const applied: number[] = []; + for (const m of migrations) { + if (m.id <= current) continue; + m.up(db); + applied.push(m.id); + } + writeSchemaVersionSync(db, latest); + return { current: latest, applied }; +} + +/** Async variant for D1 / remote SQL endpoints. Same semantics. */ +export async function runAsyncMigrations( + db: AsyncMigrationDb, + migrations: AsyncMigration[], + _cacheKey?: string, +): Promise { + if (migrations.length === 0) return { current: 0, applied: [] }; + const latest = migrations[migrations.length - 1].id; + + await ensureMetadataTableAsync(db); + const current = await readSchemaVersionAsync(db); + if (current === latest) { + return { current, applied: [] }; + } + + const applied: number[] = []; + for (const m of migrations) { + if (m.id <= current) continue; + await m.up(db); + applied.push(m.id); + } + await writeSchemaVersionAsync(db, latest); + return { current: latest, applied }; +} + +/** Kept for API stability — no internal cache to clear. */ +export function _clearMigrationCacheForTests(): void { + /* no-op */ +} diff --git a/packages/core/src/db/migrations/shared.ts b/packages/core/src/db/migrations/shared.ts new file mode 100644 index 0000000..d603813 --- /dev/null +++ b/packages/core/src/db/migrations/shared.ts @@ -0,0 +1,38 @@ +/** + * SQL fragments shared between the local and worker schemas. Anything + * here is by definition the SAME on both sides — adding a column or + * index here must be safe for both runtimes. + * + * Per-runtime divergences (currently just `sync_errors`, which is + * local-only) live in each list's own migration body, NOT here. + */ + +/** + * `entities` table DDL — the central row store. Identical on local and + * worker after `LOCAL_MIGRATIONS` v4 (which drops the local-only + * `AUTOINCREMENT` to align with worker). No `AUTOINCREMENT`: SQLite's + * implicit ROWID semantics are sufficient — we never delete entities by + * id, and reused ids would only matter if FTS rows for deleted entities + * weren't being cleaned up (they are, via `DELETE FROM entities_fts` + * before any INSERT). + */ +export const ENTITIES_TABLE_DDL = `CREATE TABLE IF NOT EXISTS entities ( + id INTEGER PRIMARY KEY, + external_id TEXT NOT NULL, + entity_type TEXT NOT NULL, + title TEXT NOT NULL, + data TEXT NOT NULL DEFAULT '{}', + content_hash TEXT, + folder TEXT, + slug TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +);`; + +/** Indexes on `entities` — same on both runtimes. */ +export const ENTITIES_INDEX_DDL: ReadonlyArray = [ + "CREATE INDEX IF NOT EXISTS idx_entities_external ON entities(external_id);", + "CREATE INDEX IF NOT EXISTS idx_entities_folder ON entities(folder, slug);", + "CREATE INDEX IF NOT EXISTS idx_entities_type ON entities(entity_type);", + "CREATE INDEX IF NOT EXISTS idx_entities_updated ON entities(updated_at);", +]; diff --git a/packages/core/src/db/migrations/types.ts b/packages/core/src/db/migrations/types.ts new file mode 100644 index 0000000..740bb6e --- /dev/null +++ b/packages/core/src/db/migrations/types.ts @@ -0,0 +1,56 @@ +/** + * Schema migration framework. + * + * Every change to the on-disk SQLite schema OR the published Cloudflare D1 + * schema must land here as a numbered migration with a matching entry in + * `SCHEMA.md`. The migration runner records the latest applied id in the + * `metadata` table under `schema.version` so subsequent loads are O(1). + * + * Two flavours: the local SQLite handle is synchronous (used inside + * `getCollectionDb`, which is itself sync), and D1 is async. After local + * migration v4 the `entities` and `entities_fts` shapes are identical — + * the only intentional divergence is the local-only `sync_errors` + * journal (the worker never syncs). + * + * **Idempotence rule.** Every migration body must be safe to re-run + * against a partially-applied DB. Use `CREATE TABLE IF NOT EXISTS`, + * `PRAGMA table_info` checks, etc. — never assume the previous migration + * succeeded. This protects against partial failures and lets us run all + * migrations on legacy DBs that don't have a `schema.version` recorded. + */ + +/** Schema version key in the `metadata` table. */ +export const SCHEMA_VERSION_KEY = "schema.version"; + +/** Synchronous DB handle abstraction — matches `bun:sqlite` / `better-sqlite3`. */ +export interface SyncMigrationDb { + exec(sql: string): void; + prepare(sql: string): { + all(...args: unknown[]): unknown[]; + get(...args: unknown[]): unknown; + run(...args: unknown[]): unknown; + }; +} + +/** Async DB handle abstraction for D1 / remote SQL endpoints. */ +export interface AsyncMigrationDb { + exec(sql: string): Promise; + query>(sql: string): Promise; +} + +/** A single sync schema migration. */ +export interface SyncMigration { + /** Sequential, gap-free id starting at 1. */ + id: number; + /** One-line summary; surfaced in logs and SCHEMA.md. */ + description: string; + /** Idempotent body. May read the DB to detect prior state. */ + up(db: SyncMigrationDb): void; +} + +/** A single async schema migration. */ +export interface AsyncMigration { + id: number; + description: string; + up(db: AsyncMigrationDb): Promise; +} diff --git a/packages/core/src/db/migrations/worker.ts b/packages/core/src/db/migrations/worker.ts new file mode 100644 index 0000000..01b1d9d --- /dev/null +++ b/packages/core/src/db/migrations/worker.ts @@ -0,0 +1,67 @@ +import type { AsyncMigration } from "./types"; +import { ENTITIES_TABLE_DDL, ENTITIES_INDEX_DDL } from "./shared"; + +/** + * Schema migrations for the **published Cloudflare D1** database that + * each worker owns. The shape differs from the local DB in a few places: + * + * - No `collection_name` column on `entities_fts` (each worker is a + * single collection) + * - No backfill loops or legacy migrations from before publish existed + * - `entities.id` isn't AUTOINCREMENT (D1 limitations + we use + * `INSERT OR REPLACE` keyed on external_id) + * + * Local migrations live in `./local.ts`; document each change in the + * top-level `SCHEMA.md`. + */ +export const WORKER_MIGRATIONS: AsyncMigration[] = [ + { + id: 1, + description: + "Baseline: entities table + indexes, entities_fts virtual table (entity_id / external_id / entity_type / title / content / tags).", + up: async (db) => { + await db.exec(ENTITIES_TABLE_DDL); + for (const sql of ENTITIES_INDEX_DDL) { + await db.exec(sql); + } + await db.exec( + "CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags);", + ); + }, + }, + { + id: 2, + description: + "Add `attachment_text` column to `entities_fts` for OCR'd attachment text. FTS5 doesn't support ALTER ADD COLUMN, so detect via PRAGMA, drop + recreate, and let the publish path re-INSERT every FTS row in the same run.", + up: async (db) => { + let needsRebuild = false; + try { + const cols = await db.query<{ name: string }>("PRAGMA table_info(entities_fts);"); + if (cols.length > 0 && !cols.some((c) => c.name === "attachment_text")) { + needsRebuild = true; + } + } catch { + // Table missing — IF NOT EXISTS below creates the new shape. + } + if (needsRebuild) { + await db.exec("DROP TABLE IF EXISTS entities_fts;"); + } + await db.exec( + "CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags, attachment_text);", + ); + }, + }, +]; + +/** + * True when the publish path needs to force-include every entity in the + * FTS push set — i.e. one of the migrations dropped + recreated + * `entities_fts`. The publish flow normally relies on a + * content-hash-driven manifest delta; that's not enough when the FTS + * table itself was reset, since unchanged entities still need their FTS + * rows re-INSERTed. + * + * Returns true when the version on disk before the run was below the + * "added attachment_text" migration. + */ +export const FTS_RESET_FROM_BELOW = 2; diff --git a/packages/core/src/search/indexer.ts b/packages/core/src/search/indexer.ts index 0c5fac6..7661f4f 100644 --- a/packages/core/src/search/indexer.ts +++ b/packages/core/src/search/indexer.ts @@ -1,4 +1,5 @@ import { openDatabase } from "../compat/sqlite"; +import { runSyncMigrations, LOCAL_MIGRATIONS } from "../db/migrations"; import { buildFtsQuery } from "./fts-query"; export interface SearchResult { @@ -22,38 +23,27 @@ export interface SearchFilters { limit?: number; } +/** + * FTS5 search over the `entities_fts` virtual table. Schema is owned by + * `LOCAL_MIGRATIONS` (`packages/core/src/db/migrations/local.ts`); the + * runner is invoked here defensively so callers that open this without + * having gone through `getCollectionDb` first still see the latest shape. + * + * Final FTS column order (after migration v4): + * entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, + * title, content, tags, attachment_text + * + * bm25 weights — kept identical to `worker/src/db/search.ts`: + * title 10, tags 5, body 1, attachment 0.25 (UNINDEXED cols get 1.0 + * but never match). See `SCHEMA.md` for the rationale on weights. + */ export class SearchIndexer { private sqlite: any; constructor(dbPath: string) { this.sqlite = openDatabase(dbPath); this.sqlite.exec("PRAGMA journal_mode = WAL;"); - this.createFtsTable(); - } - - private createFtsTable(): void { - // Migrate old FTS schema (without collection_name) by detecting column count - try { - const row = this.sqlite.prepare("PRAGMA table_info(entities_fts)").all(); - const hasCollectionName = (row as any[]).some((r: any) => r.name === "collection_name"); - if (!hasCollectionName && (row as any[]).length > 0) { - this.sqlite.exec("DROP TABLE IF EXISTS entities_fts"); - } - } catch { - // Table may not exist yet - } - - this.sqlite.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5( - collection_name UNINDEXED, - entity_id UNINDEXED, - external_id UNINDEXED, - entity_type UNINDEXED, - title, - content, - tags - ); - `); + runSyncMigrations(this.sqlite, LOCAL_MIGRATIONS, dbPath); } updateIndex(entity: { @@ -63,25 +53,33 @@ export class SearchIndexer { title: string; content: string; tags: string[]; + /** + * Accepted for API symmetry with older callers but no longer + * persisted — the FTS table dropped its `collection_name` column in + * migration v4 (it was never queried). Each collection has its own + * SQLite file, so collection scoping is implicit. + */ collectionName?: string; + /** Concatenated OCR / extracted text from the entity's attachments. */ + attachmentText?: string; }): void { - // Remove existing entry for this entity this.sqlite.exec( `DELETE FROM entities_fts WHERE entity_id = '${entity.id}'`, ); - const stmt = this.sqlite.prepare( - `INSERT INTO entities_fts (collection_name, entity_id, external_id, entity_type, title, content, tags) VALUES (?, ?, ?, ?, ?, ?, ?)`, - ); - stmt.run( - entity.collectionName ?? "", - String(entity.id), - entity.externalId, - entity.entityType, - entity.title, - entity.content, - entity.tags.join(" "), - ); + this.sqlite + .prepare( + `INSERT INTO entities_fts (entity_id, external_id, entity_type, title, content, tags, attachment_text) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + String(entity.id), + entity.externalId, + entity.entityType, + entity.title, + entity.content, + entity.tags.join(" "), + entity.attachmentText ?? "", + ); } /** @@ -109,20 +107,19 @@ export class SearchIndexer { search(query: string, filters?: SearchFilters): SearchResult[] { const ftsQuery = buildFtsQuery(query); - if (!ftsQuery) return []; - // Weight title matches much more heavily than body/tags so an entity - // with the query in its title ranks above one that only mentions the - // query in its content. bm25() weights are positional for every - // declared column (UNINDEXED columns included), not just indexed ones. - // Column order: collection_name, entity_id, external_id, entity_type, - // title, content, tags. - // Lower bm25 score = better match. + // Column order: entity_id, external_id, entity_type (UNINDEXED), then + // title=10, content=1, tags=5, attachment_text=0.25. Snippet uses + // column 3 (title) — wait, snippet uses column 4 (content) in + // 0-indexed terms. After v4 the indices are: 0=entity_id, 1=external_id, + // 2=entity_type, 3=title, 4=content, 5=tags, 6=attachment_text. Snippet + // continues to highlight column 4 (content) so the UI shows where in + // the body the query matched. let sql = ` SELECT entity_id, external_id, entity_type, title, - bm25(entities_fts, 1.0, 1.0, 1.0, 1.0, 10.0, 1.0, 2.0) AS rank, - snippet(entities_fts, 5, '', '', '…', 48) AS snippet + bm25(entities_fts, 1.0, 1.0, 1.0, 10.0, 1.0, 5.0, 0.25) AS rank, + snippet(entities_fts, 4, '', '', '…', 48) AS snippet FROM entities_fts WHERE entities_fts MATCH ? `; diff --git a/packages/core/src/sync/entity-hash.ts b/packages/core/src/sync/entity-hash.ts index e353592..1770dd8 100644 --- a/packages/core/src/sync/entity-hash.ts +++ b/packages/core/src/sync/entity-hash.ts @@ -33,6 +33,7 @@ export function computeEntityHash(entity: HashableEntity): string { assets, url: d.url ?? null, tags, + sortKey: d.sortKey ?? null, }); const hasher = createCryptoHasher("sha256"); diff --git a/packages/core/src/theme/interface.ts b/packages/core/src/theme/interface.ts index eba97b3..39769ef 100644 --- a/packages/core/src/theme/interface.ts +++ b/packages/core/src/theme/interface.ts @@ -3,6 +3,15 @@ export interface FolderConfig { visible?: boolean; /** Sort order for both files and subdirectories in this folder (default: ASC). */ sort?: "ASC" | "DESC"; + /** + * Default config to apply *inside* every direct child subdirectory of + * this folder that doesn't already have its own explicit theme config. + * Useful when a theme creates one subfolder per "context" (e.g. one per + * Evernote notebook) and wants every such folder to share defaults. + * Implemented at config-write time: each matching child gets a generated + * `.yml` serialised from this object. + */ + subdirConfig?: FolderConfig; /** * Glob patterns matching filenames to hide within this folder. * Wildcards: * matches any sequence of characters, ? matches one character. diff --git a/packages/crawlers/package.json b/packages/crawlers/package.json index f5a66ee..ad5b252 100644 --- a/packages/crawlers/package.json +++ b/packages/crawlers/package.json @@ -16,6 +16,7 @@ "dependencies": { "@frozenink/core": "workspace:*", "fast-xml-parser": "^4.5.1", - "gemoji": "^8.1.0" + "gemoji": "^8.1.0", + "yjs": "^13.6.30" } } diff --git a/packages/crawlers/src/evernote/README.md b/packages/crawlers/src/evernote/README.md new file mode 100644 index 0000000..d0b9ba0 --- /dev/null +++ b/packages/crawlers/src/evernote/README.md @@ -0,0 +1,79 @@ +# Evernote Crawler + +Imports notes, notebooks, tags, and attachments from a local **Evernote v10** +install. No tokens, no network — the crawler reads Evernote's local SQLite +database (`conduit-storage`) plus the on-disk attachment cache. + +Auto-detected paths by platform: + +- **macOS**, sandboxed (App Store / direct download): + `~/Library/Containers/com.evernote.Evernote/Data/Library/Application Support/Evernote/conduit-storage/` +- **macOS**, Setapp build: same container path with the Setapp bundle id. +- **Windows**: `%LOCALAPPDATA%\Evernote\Evernote\conduit-storage\` for the + default installer; `%LOCALAPPDATA%\Packages\Evernote*\LocalCache\…` for MSIX + / Microsoft Store builds. +- **Linux**: `~/.config/Evernote/conduit-storage/` or + `~/snap/evernote/current/.config/Evernote/conduit-storage/`. + +Within `conduit-storage/` the actual DB is one level deeper, under a +URL-encoded host directory (`https%3A%2F%2Fwww.evernote.com/UDB-User+RemoteGraph.sql`), +which the crawler discovers automatically. + +## Note body resolution + +v10 does **not** store note bodies as ENML in the SQLite DB. The rich-text +content lives in a custom Yjs/CRDT binary blob under +`/conduit-fs/.../rte/Note/internal_rteDoc///.dat`. +The crawler decodes that blob with a best-effort extractor (`rte.ts`) that +walks printable ASCII runs and filters out structural metadata, hashes, +style declarations, and CRDT identifiers. Body text comes through; layout +and inline attachment placement do not. Resolution order, first non-empty +wins: + +1. **`conduit-fs/.../.dat`** — extracted via `rte.ts`. Most + complete source for normally-edited notes. +2. **`Offline_Search_Note_Content.content`** — Evernote's own pre-stripped + plain text, populated for notes the user has marked for offline access. +3. **`Nodes_Note.snippet`** — short preview, useful for very small notes. +4. **Legacy `content.enml`** on disk — for very old installs. + +Attachments (images, PDFs) always render in their own `## Attachments` +section at the bottom of the markdown, regardless of where they appear in +the original note. + +## Configuration + +```yaml +crawler: evernote +config: + conduitStoragePath: ~/Library/Containers/com.evernote.Evernote/Data/Library/Application\ Support/Evernote/conduit-storage + notebooks: ["Personal", "Work"] # optional allowlist; default = all + snapshot: true # default; copies DB+WAL+SHM before reading +``` + +The default `conduitStoragePath` is auto-detected, so the field can be left +blank in most setups. `snapshot` is on by default and lets the crawler read +safely even while Evernote is running. + +## OCR cascade + +Each image / PDF attachment runs through a three-tier OCR cascade. The first +tier that returns non-empty text wins: + +1. **Evernote-provided** `recognition` XML (free, exact, no work). +2. **Apple Vision** (macOS only) via a small Swift one-shot helper. +3. **Tesseract.js + pdfjs-dist** WASM fallback (lazy-loaded; tier is silently + skipped when those modules aren't installed). + +OCR text is written to `EntityData.assets[i].text` and indexed by FTS5 in the +`attachment_text` column with a 0.5× weight, so a body match still ranks above +an OCR-only match. + +## Sync model + +- USN-driven incremental sync per notebook: only notes whose + `updateSequenceNum` exceeds the last high-water mark are re-emitted. +- Deletions are detected by diffing the previous run's known node ids against + the current snapshot, plus any node whose `active` flag flipped to false. +- A single `sync()` call covers all notebooks; pagination isn't needed because + reads are local. diff --git a/packages/crawlers/src/evernote/__tests__/crawler.test.ts b/packages/crawlers/src/evernote/__tests__/crawler.test.ts new file mode 100644 index 0000000..19e9ac3 --- /dev/null +++ b/packages/crawlers/src/evernote/__tests__/crawler.test.ts @@ -0,0 +1,184 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { openDatabase } from "@frozenink/core"; +import { EvernoteCrawler, listEvernoteNotebooks } from "../crawler"; + +interface Fixture { + conduitDir: string; + cleanup: () => void; +} + +/** + * Build a synthetic conduit-storage tree that mirrors Evernote v10's actual + * schema: per-type Nodes_* tables, a NoteTag junction, an Attachment table, + * and an AttachmentSearchText table for OCR'd attachment text. + */ +function buildFixture(): Fixture { + const root = mkdtempSync(join(tmpdir(), "frozenink-evernote-test-")); + const conduitDir = join(root, "conduit-storage"); + mkdirSync(conduitDir, { recursive: true }); + + const dbPath = join(conduitDir, "UDB-User1+RemoteGraph.sql"); + const db = openDatabase(dbPath); + + db.exec(` + CREATE TABLE Nodes_Notebook (id TEXT PRIMARY KEY, label TEXT, version REAL); + CREATE TABLE Nodes_Note ( + id TEXT PRIMARY KEY, + label TEXT, + parent_Notebook_id TEXT, + content_hash TEXT, + deleted INTEGER, + version REAL, + created INTEGER, + updated INTEGER, + snippet TEXT + ); + CREATE TABLE Nodes_Tag (id TEXT PRIMARY KEY, label TEXT); + CREATE TABLE NoteTag (id TEXT PRIMARY KEY, Note_id TEXT, Tag_id TEXT); + CREATE TABLE Attachment ( + id TEXT PRIMARY KEY, + filename TEXT, + mime TEXT, + isActive INTEGER, + dataHash TEXT, + dataSize INTEGER, + parent_Note_id TEXT + ); + CREATE TABLE AttachmentSearchText (id TEXT PRIMARY KEY, searchText TEXT); + `); + + db.prepare("INSERT INTO Nodes_Notebook(id, label, version) VALUES (?, ?, ?)") + .run("nb-personal", "Personal", 1); + db.prepare("INSERT INTO Nodes_Notebook(id, label, version) VALUES (?, ?, ?)") + .run("nb-work", "Work", 1); + + for (const [id, title, nb, ver] of [ + ["note-1", "Hello note", "nb-personal", 10], + ["note-2", "Work plan", "nb-work", 20], + ["note-3", "Drafted", "nb-personal", 30], + ] as const) { + db.prepare( + "INSERT INTO Nodes_Note(id, label, parent_Notebook_id, content_hash, deleted, version, created, updated, snippet) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ).run(id, title, nb, `hash-${id}`, 0, ver, 0, 0, `Snippet of ${id}`); + } + + db.prepare("INSERT INTO Nodes_Tag(id, label) VALUES (?, ?)").run("tag-a", "TagA"); + db.prepare("INSERT INTO Nodes_Tag(id, label) VALUES (?, ?)").run("tag-b", "TagB"); + db.prepare("INSERT INTO NoteTag(id, Note_id, Tag_id) VALUES (?, ?, ?)").run("nt-1", "note-1", "tag-a"); + db.prepare("INSERT INTO NoteTag(id, Note_id, Tag_id) VALUES (?, ?, ?)").run("nt-2", "note-1", "tag-b"); + + db.prepare( + "INSERT INTO Attachment(id, filename, mime, isActive, dataHash, dataSize, parent_Note_id) VALUES (?, ?, ?, ?, ?, ?, ?)", + ).run("att-1", "pic.png", "image/png", 1, "deadbeef", 4, "note-1"); + db.prepare("INSERT INTO AttachmentSearchText(id, searchText) VALUES (?, ?)") + .run("att-1", "OCR text from picture"); + db.close(); + + const resCacheDir = join(root, "resource-cache", "User1", "note-1"); + mkdirSync(resCacheDir, { recursive: true }); + writeFileSync(join(resCacheDir, "deadbeef"), Buffer.from([1, 2, 3, 4])); + + return { conduitDir, cleanup: () => rmSync(root, { recursive: true, force: true }) }; +} + +describe("EvernoteCrawler", () => { + let fixture: Fixture; + beforeEach(() => { + fixture = buildFixture(); + }); + afterEach(() => { + fixture.cleanup(); + }); + + it("emits one entity per active note with tags from the NoteTag junction table", async () => { + const crawler = new EvernoteCrawler(); + await crawler.initialize({ conduitStoragePath: fixture.conduitDir, snapshot: false }, {}); + try { + const result = await crawler.sync(null); + expect(result.entities).toHaveLength(3); + const note1 = result.entities.find((e) => e.externalId === "note-1")!; + expect(note1.tags).toContain("TagA"); + expect(note1.tags).toContain("TagB"); + // Notebook membership is structured metadata, NOT a tag. + expect(note1.tags).not.toContain("notebook:Personal"); + expect((note1.data as any).notebookName).toBe("Personal"); + expect(note1.attachments?.[0]?.text).toBe("OCR text from picture"); + } finally { + await crawler.dispose(); + } + }); + + it("only re-emits notes whose version advances on the second sync", async () => { + const crawler = new EvernoteCrawler(); + await crawler.initialize({ conduitStoragePath: fixture.conduitDir, snapshot: false }, {}); + try { + const first = await crawler.sync(null); + expect(first.entities).toHaveLength(3); + const second = await crawler.sync(first.nextCursor); + expect(second.entities).toHaveLength(0); + } finally { + await crawler.dispose(); + } + }); + + it("reports deletions when a note disappears or is marked deleted", async () => { + const crawler = new EvernoteCrawler(); + await crawler.initialize({ conduitStoragePath: fixture.conduitDir, snapshot: false }, {}); + try { + const first = await crawler.sync(null); + const dbPath = join(fixture.conduitDir, "UDB-User1+RemoteGraph.sql"); + const db = openDatabase(dbPath); + db.exec("DELETE FROM Nodes_Note WHERE id = 'note-2'"); + db.close(); + await crawler.dispose(); + + const c2 = new EvernoteCrawler(); + await c2.initialize({ conduitStoragePath: fixture.conduitDir, snapshot: false }, {}); + try { + const result = await c2.sync(first.nextCursor); + expect(result.deletedExternalIds).toContain("note-2"); + } finally { + await c2.dispose(); + } + } finally { + // already disposed + } + }); + + it("respects the notebooks allowlist", async () => { + const crawler = new EvernoteCrawler(); + await crawler.initialize( + { conduitStoragePath: fixture.conduitDir, snapshot: false, notebooks: ["Work"] }, + {}, + ); + try { + const result = await crawler.sync(null); + expect(result.entities).toHaveLength(1); + expect(result.entities[0].externalId).toBe("note-2"); + } finally { + await crawler.dispose(); + } + }); +}); + +describe("listEvernoteNotebooks", () => { + let fixture: Fixture; + beforeEach(() => { + fixture = buildFixture(); + }); + afterEach(() => { + fixture.cleanup(); + }); + + it("returns notebooks with note counts and aggregate attachment sizes", async () => { + const summaries = await listEvernoteNotebooks(fixture.conduitDir); + const byName = Object.fromEntries(summaries.map((s) => [s.name, s])); + expect(byName.Personal.noteCount).toBe(2); + expect(byName.Personal.totalBytes).toBe(4); + expect(byName.Work.noteCount).toBe(1); + expect(byName.Work.totalBytes).toBe(0); + }); +}); diff --git a/packages/crawlers/src/evernote/__tests__/enml.test.ts b/packages/crawlers/src/evernote/__tests__/enml.test.ts new file mode 100644 index 0000000..ef54ae8 --- /dev/null +++ b/packages/crawlers/src/evernote/__tests__/enml.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "bun:test"; +import { enmlToMarkdown, parseEvernoteRecognitionXml } from "../enml"; + +const noResources = { resourceByHash: {} }; + +describe("enmlToMarkdown", () => { + it("strips the en-note wrapper and produces clean markdown", () => { + const enml = `

Hello world

`; + expect(enmlToMarkdown(enml, noResources).trim()).toBe("Hello **world**"); + }); + + it("renders en-todo as GFM checkboxes", () => { + const enml = `Done
Pending`; + const md = enmlToMarkdown(enml, noResources); + expect(md).toContain("- [x] Done"); + expect(md).toContain("- [ ] Pending"); + }); + + it("substitutes en-media references with image and link refs", () => { + const enml = ``; + const md = enmlToMarkdown(enml, { + resourceByHash: { + abc: { filename: "pic.png", mimeType: "image/png", assetPath: "attachments/evernote/x/pic.png" }, + def: { filename: "doc.pdf", mimeType: "application/pdf", assetPath: "attachments/evernote/x/doc.pdf" }, + }, + }); + expect(md).toContain("![pic.png](attachments/evernote/x/pic.png)"); + expect(md).toContain("[doc.pdf](attachments/evernote/x/doc.pdf)"); + }); + + it("converts links and headings", () => { + const enml = `

Title

x`; + const md = enmlToMarkdown(enml, noResources); + expect(md).toContain("## Title"); + expect(md).toContain("[x](https://x)"); + }); + + it("encrypted blocks are replaced with a placeholder", () => { + const enml = `…`; + expect(enmlToMarkdown(enml, noResources)).toContain("[encrypted content]"); + }); +}); + +describe("parseEvernoteRecognitionXml", () => { + it("returns an empty string for empty input", () => { + expect(parseEvernoteRecognitionXml("")).toBe(""); + expect(parseEvernoteRecognitionXml("")).toBe(""); + }); + + it("picks the highest-confidence token per item", () => { + const xml = ` + + helohello + wrldworld + `; + expect(parseEvernoteRecognitionXml(xml)).toBe("hello world"); + }); +}); diff --git a/packages/crawlers/src/evernote/__tests__/ocr.test.ts b/packages/crawlers/src/evernote/__tests__/ocr.test.ts new file mode 100644 index 0000000..181a46a --- /dev/null +++ b/packages/crawlers/src/evernote/__tests__/ocr.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "bun:test"; +import { extractAttachmentText } from "../ocr"; + +describe("extractAttachmentText OCR cascade", () => { + it("uses the Evernote-provided recognition XML when available (tier 1)", async () => { + const text = await extractAttachmentText({ + buffer: Buffer.from([0]), + mimeType: "image/png", + filename: "x.png", + evernoteRecognitionXml: `cached`, + }); + expect(text).toBe("cached"); + }); + + it("returns empty for non-image / non-PDF attachments without falling through", async () => { + const text = await extractAttachmentText({ + buffer: Buffer.from("hello"), + mimeType: "text/plain", + filename: "x.txt", + }); + expect(text).toBe(""); + }); + + it("respects the maxBytes filter and skips oversized files", async () => { + const big = Buffer.alloc(10); + const text = await extractAttachmentText({ + buffer: big, + mimeType: "image/png", + filename: "huge.png", + evernoteRecognitionXml: `nope`, + maxBytes: 4, + }); + expect(text).toBe(""); + }); +}); diff --git a/packages/crawlers/src/evernote/__tests__/rte.test.ts b/packages/crawlers/src/evernote/__tests__/rte.test.ts new file mode 100644 index 0000000..0d32927 --- /dev/null +++ b/packages/crawlers/src/evernote/__tests__/rte.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "bun:test"; +import { extractRte, extractTextFromRteBlob, rteDatPath } from "../rte"; + +/** + * Synthesize a buffer that mimics the v10 binary RTE wire format closely + * enough for extraction: arbitrary 1-byte control bytes interleaved with + * length-prefixed UTF-8 strings. The extractor doesn't actually parse the + * length bytes — it just walks printable runs — so this is a fair stand-in. + */ +function buildBlob(strings: string[]): Buffer { + const parts: Buffer[] = []; + for (const s of strings) { + // Real Yjs streams interleave several non-printable control bytes + // (typically < 0x20) between strings — use 0x05 as a guaranteed boundary + // marker so the extractor's printable-run scan terminates between fields. + parts.push(Buffer.from([0x07, 0x01, 0x05])); + parts.push(Buffer.from(s, "utf-8")); + } + return Buffer.concat(parts); +} + +describe("extractTextFromRteBlob", () => { + it("strips structural keys and recovers body content", () => { + const blob = buildBlob([ + "content", "en-note", "div", "meta", "schemaVersion", + "type", "image/png", "hash", "5e47bac687a10827a57aeb600d4f3bbb", + "Paid on 22nd of Apr 2025", + "See receipt attached", + ]); + const text = extractTextFromRteBlob(blob); + expect(text).toContain("Paid on 22nd of Apr 2025"); + expect(text).toContain("See receipt attached"); + expect(text).not.toContain("en-note"); + expect(text).not.toContain("schemaVersion"); + expect(text).not.toContain("5e47bac687a10827a57aeb600d4f3bbb"); + }); + + it("drops style declarations and GUID-like strings", () => { + const blob = buildBlob([ + "--en-naturalWidth:2512; --en-naturalHeight:1450;", + "abc12345-6789-abcd-ef01-23456789abcd", + "Real body text here", + ]); + expect(extractTextFromRteBlob(blob)).toBe("Real body text here"); + }); + + it("returns empty string for empty input", () => { + expect(extractTextFromRteBlob(Buffer.alloc(0))).toBe(""); + }); + + it("drops --en- CSS variables and base64 payloads", () => { + const blob = buildBlob([ + `display:none;--en-chs:"eyJoMSI6eyJmb250RmFtaWx5IjoiaW5oZXJpdCJ9fQ=="`, + "Real body text", + ]); + expect(extractTextFromRteBlob(blob)).toBe("Real body text"); + }); + + it("strips field-name+length-byte prefixes from glued values", () => { + // Real Yjs binary often glues a length byte onto the field name, e.g. + // `title<2>` decodes to `title2` once read as ASCII. We + // strip the prefix so the actual title text reads cleanly. + const blob = buildBlob([ + "title220260331 230 Lake Chelan Reclamation District Bill", + ]); + expect(extractTextFromRteBlob(blob)).toContain("20260331 230 Lake Chelan Reclamation District Bill"); + }); + + it("drops trailing concatenated field runs", () => { + const blob = buildBlob([ + "Reclamation District Bill colorw 0 fontStylew", + ]); + expect(extractTextFromRteBlob(blob)).toBe("Reclamation District Bill"); + }); +}); + +describe("extractRte attachment placeholders", () => { + it("collects en-media references and appends them after body text", () => { + const blob = buildBlob([ + "Some body text", + "en-media", + "hash", + "w 5e47bac687a10827a57aeb600d4f3bbb", + "type", + "image/png", + "More body text", + ]); + const result = extractRte(blob); + expect(result.attachmentHashes).toEqual(["5e47bac687a10827a57aeb600d4f3bbb"]); + // Body lines come first; the placeholder appears after them. The + // binary's CRDT structure isn't decoded, so we can't match exact + // inline position — but the common "short body + attached doc" layout + // matches how Evernote shows these notes. + const bodyIdx = result.text.indexOf("Some body text"); + const placeholderIdx = result.text.indexOf("[[EVERNOTE_ATTACHMENT:"); + expect(bodyIdx).toBeGreaterThanOrEqual(0); + expect(bodyIdx).toBeLessThan(placeholderIdx); + }); +}); + +describe("rteDatPath", () => { + it("shards the GUID into //.dat", () => { + expect(rteDatPath("/root", "000a3dfa-86c4-57e0-a72c-dbfa337905ea")) + .toBe("/root/000/5ea/000a3dfa-86c4-57e0-a72c-dbfa337905ea.dat"); + }); +}); diff --git a/packages/crawlers/src/evernote/crawler.ts b/packages/crawlers/src/evernote/crawler.ts new file mode 100644 index 0000000..56e38be --- /dev/null +++ b/packages/crawlers/src/evernote/crawler.ts @@ -0,0 +1,913 @@ +import { existsSync, readFileSync } from "fs"; +import { homedir, platform } from "os"; +import { join, dirname, basename } from "path"; +import type { + Crawler, + CrawlerMetadata, + CrawlerEntityData, + SyncCursor, + SyncResult, + AssetFilter, +} from "@frozenink/core"; +import { openDatabase, createCryptoHasher } from "@frozenink/core"; +import type { + EvernoteConfig, + EvernoteCredentials, + EvernoteSyncCursor, + EvernoteNoteRow, + EvernoteNotebookRow, + EvernoteResourceRow, + EvernoteTagRow, +} from "./types"; +import { copyConduitStorage, findRemoteGraphDb, isWalDirty } from "./snapshot"; +import { enmlToMarkdown, encodePathForMarkdown } from "./enml"; +import { extractAttachmentText } from "./ocr"; +import { + decodeRteBlob, + extractRte, + rteDatPath, + ATTACHMENT_PLACEHOLDER_PREFIX, + ATTACHMENT_PLACEHOLDER_SUFFIX, +} from "./rte"; + +export interface EvernoteNotebookSummary { + guid: string; + name: string; + noteCount: number; + /** Sum of attachment sizes across notes in this notebook, in bytes. */ + totalBytes: number; +} + +/** + * Read the notebook list (with note count and aggregate attachment size) from + * a conduit-storage directory without instantiating the full crawler. Used by + * the desktop "Add Collection" form so the user can pick which notebooks to + * sync. Reads the live file directly when the WAL is checkpointed; otherwise + * snapshots first to stay safe alongside a running Evernote. + */ +export async function listEvernoteNotebooks( + conduitStorageDir?: string, +): Promise { + const dir = conduitStorageDir || defaultConduitPath(); + if (!existsSync(dir)) { + throw new Error(`Evernote conduit-storage directory not found: ${dir}`); + } + const live = findRemoteGraphDb(dir); + if (!live) throw new Error(`No Evernote RemoteGraph DB found in ${dir}`); + + let dbPath = live; + let cleanup: (() => void) | null = null; + if (isWalDirty(live)) { + const snap = copyConduitStorage(dirname(live)); + dbPath = snap.dbPath; + cleanup = snap.cleanup; + } + + const db = openDatabase(dbPath); + try { + try { db.exec("PRAGMA query_only = 1;"); } catch { /* ok */ } + + const tables = new Set( + (db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as Array<{ name: string }>).map( + (t) => t.name, + ), + ); + if (!tables.has("Nodes_Notebook")) return []; + + type NotebookRow = { id: string; name: string | null }; + const notebookRows = db + .prepare(`SELECT id, label AS name FROM Nodes_Notebook`) + .all() as NotebookRow[]; + + // Per-notebook note count + aggregate attachment size in one shot. + type CountRow = { notebookId: string; n: number }; + const countRows = db + .prepare( + `SELECT parent_Notebook_id AS notebookId, COUNT(*) AS n + FROM Nodes_Note + WHERE parent_Notebook_id IS NOT NULL AND (deleted IS NULL OR deleted = 0) + GROUP BY parent_Notebook_id`, + ) + .all() as CountRow[]; + const counts = new Map(countRows.map((r) => [r.notebookId, r.n] as const)); + + type SizeRow = { notebookId: string; bytes: number }; + let sizes = new Map(); + if (tables.has("Attachment")) { + const sizeRows = db + .prepare( + `SELECT n.parent_Notebook_id AS notebookId, SUM(a.dataSize) AS bytes + FROM Attachment a JOIN Nodes_Note n ON n.id = a.parent_Note_id + WHERE a.isActive = 1 AND n.parent_Notebook_id IS NOT NULL + GROUP BY n.parent_Notebook_id`, + ) + .all() as SizeRow[]; + sizes = new Map(sizeRows.map((r) => [r.notebookId, r.bytes ?? 0] as const)); + } + + return notebookRows + .map((r) => ({ + guid: r.id, + name: r.name ?? "(unnamed)", + noteCount: counts.get(r.id) ?? 0, + totalBytes: sizes.get(r.id) ?? 0, + })) + .sort((a, b) => a.name.localeCompare(b.name)); + } finally { + try { db.close?.(); } catch { /* ignore */ } + if (cleanup) cleanup(); + } +} + +/** + * Recover the Evernote user identifier (e.g. `User1234567`) from the path of + * the RemoteGraph database. Filenames look like `UDB-User1234567+RemoteGraph.sql` + * — the literal `User` segment is what shows up under `resource-cache/`. + */ +function extractUserIdFromDbPath(dbPath: string): string | null { + const m = basename(dbPath).match(/(User\d+)/); + return m ? m[1] : null; +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Wrap a flat OCR string from `AttachmentSearchText` into the minimal + * `` shape that `parseEvernoteRecognitionXml()` consumes — this + * keeps a single tier-1 code path in the OCR cascade. + */ +function recognitionAsXml(plainText: string): string { + const escaped = plainText + .replace(/&/g, "&").replace(//g, ">"); + return `${escaped}`; +} + +/** + * Auto-detect candidate locations for Evernote v10's conduit-storage + * directory. Different platforms and install sources put it in different + * places — we return all the spots worth checking, in priority order. + * + * macOS: + * - Mac App Store / direct download (sandboxed): + * `~/Library/Containers/com.evernote.Evernote/Data/Library/Application Support/Evernote/conduit-storage` + * - Setapp variant: same container path with the Setapp bundle id. + * - Unsandboxed legacy v10 builds occasionally used + * `~/Library/Application Support/Evernote/conduit-storage`. + * + * Windows: + * - Default installer: `%LOCALAPPDATA%\Evernote\Evernote\conduit-storage` + * - Microsoft Store (MSIX) builds: under `%LOCALAPPDATA%\Packages\\LocalCache\...` + * — we glob the Packages directory for any folder whose name starts + * with `Evernote` and contains `conduit-storage`. + * + * Linux: + * - AppImage / Snap / .deb: `~/.config/Evernote/conduit-storage` + * - Snap confined: `~/snap/evernote/current/.config/Evernote/conduit-storage` + * + * The legacy non-v10 client stored data under a totally different layout + * (`~/Library/Application Support/Evernote/accounts/...` on macOS); we + * don't support that and simply won't find a RemoteGraph DB there. + */ +function defaultConduitCandidates(): string[] { + const home = homedir(); + const out: string[] = []; + if (platform() === "darwin") { + out.push(join(home, "Library/Containers/com.evernote.Evernote/Data/Library/Application Support/Evernote/conduit-storage")); + // Setapp build + out.push(join(home, "Library/Containers/com.evernote.EvernoteSetapp/Data/Library/Application Support/Evernote/conduit-storage")); + // Unsandboxed + out.push(join(home, "Library/Application Support/Evernote/conduit-storage")); + } else if (platform() === "win32") { + const local = process.env.LOCALAPPDATA || join(home, "AppData", "Local"); + const roaming = process.env.APPDATA || join(home, "AppData", "Roaming"); + out.push(join(local, "Evernote", "Evernote", "conduit-storage")); + out.push(join(local, "Programs", "Evernote", "conduit-storage")); + out.push(join(roaming, "Evernote", "conduit-storage")); + // MSIX / Microsoft Store: scan the Packages directory. + try { + const packagesDir = join(local, "Packages"); + const { readdirSync } = require("fs") as typeof import("fs"); + for (const entry of readdirSync(packagesDir)) { + if (entry.toLowerCase().startsWith("evernote")) { + out.push(join(packagesDir, entry, "LocalCache", "Roaming", "Evernote", "conduit-storage")); + out.push(join(packagesDir, entry, "LocalCache", "Local", "Evernote", "conduit-storage")); + } + } + } catch { + // packages dir may not exist + } + } else { + // Linux + other Unixes + const xdg = process.env.XDG_CONFIG_HOME || join(home, ".config"); + out.push(join(xdg, "Evernote", "conduit-storage")); + out.push(join(home, "snap", "evernote", "current", ".config", "Evernote", "conduit-storage")); + } + return out; +} + +/** First candidate that actually exists, or the canonical macOS path as a last-resort label. */ +function autoDetectConduitPath(): string { + for (const candidate of defaultConduitCandidates()) { + if (existsSync(candidate)) return candidate; + } + return defaultConduitCandidates()[0]; +} + +function defaultConduitPath(): string { + return autoDetectConduitPath(); +} + +const SYNC_CONTEXT_KEY = "default"; + +interface InternalState { + conduitDir: string; + /** Root of Evernote's app-support directory (parent of `conduit-storage/`). */ + evernoteRoot: string; + /** Derived from the DB filename (e.g. `UDB-User1234+RemoteGraph.sql` → `User1234`). */ + userId: string | null; + dbPath: string; + livePath: string; + cleanup: (() => void) | null; + db: any; + notebookFilter: Set | null; +} + +export class EvernoteCrawler implements Crawler { + metadata: CrawlerMetadata = { + type: "evernote", + displayName: "Evernote", + description: + "Imports notes, attachments, and OCR text from the local Evernote v10 macOS database", + // Bumped on each enml/encoding fix to trigger SyncEngine to re-render + // every existing entity from its stored data (see AGENTS.md — minor + // bump = re-render markdown). 1.1 added Yjs decoding + image-syntax + // PDFs; 1.2 percent-encodes markdown-special chars (`*`, `(`, `)`) + // in attachment URLs so filenames like `11292025_*000779*…` stop + // tripping markdown's emphasis parser. + version: "1.2", + configSchema: { + conduitStoragePath: { + type: "string", + required: false, + description: "Override path to Evernote's conduit-storage directory", + }, + notebooks: { + type: "array", + required: false, + description: "Optional notebook allowlist (names or GUIDs)", + }, + snapshot: { + type: "boolean", + required: false, + description: "Snapshot the DB+WAL+SHM into a temp dir before reading (default true)", + }, + }, + credentialFields: ["conduitStoragePath"], + }; + + private state: InternalState | null = null; + private snapshotMode = true; + private assetFilter: AssetFilter | null = null; + private progressCallback: ((message: string) => void) | null = null; + + setAssetFilter(filter: AssetFilter): void { + this.assetFilter = filter; + } + + setProgressCallback(callback: (message: string) => void): void { + this.progressCallback = callback; + } + + async initialize( + config: Record, + credentials: Record, + ): Promise { + const cfg = config as unknown as EvernoteConfig; + const creds = credentials as unknown as EvernoteCredentials; + const conduitDir = + creds.conduitStoragePath || cfg.conduitStoragePath || defaultConduitPath(); + if (!existsSync(conduitDir)) { + throw new Error(`Evernote conduit-storage directory not found: ${conduitDir}`); + } + this.snapshotMode = cfg.snapshot !== false; + + let dbPath: string; + let cleanup: (() => void) | null = null; + const live = findRemoteGraphDb(conduitDir); + if (!live) throw new Error(`No Evernote RemoteGraph DB found in ${conduitDir}`); + + // Snapshot is on by default, but if the WAL is empty we know Evernote + // has checkpointed and quit — reading the live file is safe and + // dramatically cheaper than copying gigabytes of attachments metadata. + if (this.snapshotMode && isWalDirty(live)) { + const snap = copyConduitStorage(dirname(live)); + dbPath = snap.dbPath; + cleanup = snap.cleanup; + } else { + dbPath = live; + } + + const db = openDatabase(dbPath); + try { + db.exec("PRAGMA query_only = 1;"); + } catch { + // older sqlite builds may not support it; safe to ignore for read-only access patterns + } + + this.state = { + conduitDir, + // app-support root sits one level above conduit-storage + evernoteRoot: dirname(conduitDir), + userId: extractUserIdFromDbPath(live), + dbPath, + livePath: live, + cleanup, + db, + notebookFilter: + cfg.notebooks && cfg.notebooks.length > 0 ? new Set(cfg.notebooks) : null, + }; + } + + async sync(cursor: SyncCursor | null): Promise { + if (!this.state) throw new Error("Evernote crawler not initialized"); + const c = (cursor as EvernoteSyncCursor | null) ?? {}; + const highWaterByContext = { ...(c.highWaterByContext ?? {}) }; + const previousKnown = c.knownNodes ?? {}; + const high = highWaterByContext[SYNC_CONTEXT_KEY] ?? 0; + + this.report("Reading notebooks"); + const notebooks = this.readNotebooks(); + const notebookById = new Map(notebooks.map((n) => [n.guid, n])); + + this.report("Reading tags"); + const tags = this.readTags(); + const tagById = new Map(tags.map((t) => [t.guid, t])); + + this.report("Reading notes"); + const notes = this.readNotes(); + const resourcesByNote = this.readResources(); + const offlineBodyById = this.readOfflineSearchContent(); + // Some v10 builds keep tag→note links in a separate Edges table rather + // than embedded in the note's NodeFields. Build a supplemental index. + const edgeTagsByNote = this.readTagEdges(new Set(tagById.keys())); + + const filtered = notes.filter((n) => this.matchesNotebookFilter(n, notebookById)); + const newKnownNodes: Record = {}; + for (const n of filtered) { + newKnownNodes[n.guid] = n.updateSequenceNum; + } + + // Tombstones: ids known in the previous run but absent now, plus any + // node whose `active` flag flipped to false in this snapshot. + const deletedExternalIds: string[] = []; + for (const guid of Object.keys(previousKnown)) { + if (!newKnownNodes[guid]) deletedExternalIds.push(guid); + } + for (const n of filtered) { + if (!n.active) deletedExternalIds.push(n.guid); + } + + let newHigh = high; + const entities: CrawlerEntityData[] = []; + + let processed = 0; + for (const note of filtered) { + processed++; + if (note.updateSequenceNum <= high) continue; + if (!note.active) continue; // already counted as deletion above + + this.report(`Building note ${processed}/${filtered.length}: ${note.title}`); + const notebook = note.notebookId ? notebookById.get(note.notebookId) : undefined; + const resources = resourcesByNote.get(note.guid) ?? []; + + // Resolve the note body, in priority order: + // 1. Decoded text from the v10 binary RTE blob (`conduit-fs/.../.dat`). + // This is the most complete source — contains the full note body + // Evernote actually renders. Best-effort text extraction; loses + // formatting but recovers all the searchable content. + // 2. `Offline_Search_Note_Content.content` — Evernote's own + // pre-stripped plain text, populated for offline-cached notes. + // 3. The `Nodes_Note.snippet` preview (loaded as `note.enml` upstream). + // 4. Legacy on-disk `content.enml` for very old installs. + // Body resolution. Prefer the Yjs decoder — it gives back real ENML + // in original document order, with `` tags positioned where + // the user actually placed them. If decoding fails, fall back to the + // byte-walking extractor (which emits placeholder tokens) and + // finally to plain-text sources. + let enml = ""; + // Hashes seen by the byte-walker only — used to drive placeholder + // substitution. With proper Yjs decoding we don't need this; the + // ENML's `` tags already reference attachments by hash. + let inlinedAttachmentHashes: string[] = []; + const rte = this.readRteBody(note.guid); + if (rte?.kind === "enml") { + enml = rte.enml; + inlinedAttachmentHashes = rte.attachmentHashes; + } else if (rte?.kind === "fallback") { + enml = rte.text; + inlinedAttachmentHashes = rte.attachmentHashes; + } + if (!enml && offlineBodyById.has(note.guid)) { + enml = offlineBodyById.get(note.guid) ?? ""; + } + if (!enml && note.enml) enml = note.enml; + if (!enml) { + for (const candidate of this.legacyEnmlPaths(note.guid)) { + try { + if (existsSync(candidate)) { + enml = readFileSync(candidate, "utf-8"); + break; + } + } catch { + // try next + } + } + } + + const resourceByHash: Record< + string, + { filename: string; mimeType: string; assetPath: string } + > = {}; + const attachments: NonNullable = []; + + for (const r of resources) { + const filename = r.filename || `${r.hash}`; + const mime = r.mime || "application/octet-stream"; + const storagePath = `attachments/evernote/${note.guid}/${filename}`; + resourceByHash[r.hash] = { + filename, + mimeType: mime, + assetPath: storagePath, + }; + + const blob = this.readResourceBlob(note.guid, r.hash); + if (!blob) continue; + + const recognitionXml = + r.recognition || note.recognitionByResourceHash?.[r.hash]; + const text = await extractAttachmentText({ + buffer: blob, + mimeType: mime, + filename, + evernoteRecognitionXml: recognitionXml, + maxBytes: this.assetFilter?.maxSizeBytes, + }); + + attachments.push({ + filename, + mimeType: mime, + content: blob, + storagePath, + ...(text ? { text } : {}), + }); + } + + let markdown = enml ? enmlToMarkdown(enml, { resourceByHash }) : ""; + + // The byte-walking fallback occasionally recovers the note's title as + // a quoted prefix glued onto the body (e.g. `"G Paid on…`). + // Real ENML from the Yjs decoder doesn't have this problem — the + // title is stored separately — so this scrub only runs for fallback + // output. + if (rte?.kind === "fallback" && markdown && note.title) { + const titleEsc = escapeRegExp(note.title); + const titleStripRe = new RegExp( + `^["']?\\s*${titleEsc}\\s*["']?[a-zA-Z0-9]?\\s*\\n?`, + ); + markdown = markdown.replace(titleStripRe, ""); + markdown = markdown + .split("\n") + .filter((line) => { + const trimmed = line.trim(); + return trimmed !== note.title + && trimmed !== `"${note.title}"` + && trimmed !== `'${note.title}'`; + }) + .join("\n"); + } + + // Substitute `[[EVERNOTE_ATTACHMENT:<hash>]]` placeholders that the + // fallback extractor leaves in its output. Yjs-decoded ENML doesn't + // contain placeholders — `<en-media>` tags are already substituted + // by enmlToMarkdown above. + if (rte?.kind === "fallback" && markdown && inlinedAttachmentHashes.length > 0) { + const placeholderRe = new RegExp( + `${escapeRegExp(ATTACHMENT_PLACEHOLDER_PREFIX)}([a-f0-9]+)${escapeRegExp(ATTACHMENT_PLACEHOLDER_SUFFIX)}`, + "gi", + ); + markdown = markdown.replace(placeholderRe, (_match, rawHash: string) => { + const hash = rawHash.toLowerCase(); + let res = resourceByHash[hash]; + if (!res) { + const fallback = Object.keys(resourceByHash).find((k) => k.endsWith(hash) || hash.endsWith(k)); + if (fallback) res = resourceByHash[fallback]; + } + if (!res) return ""; + const url = encodePathForMarkdown(res.assetPath); + const isImage = res.mimeType.startsWith("image/"); + const isPdf = res.mimeType === "application/pdf" || /\.pdf$/i.test(res.filename); + if (isImage || isPdf) return `\n\n![${res.filename}](${url})\n\n`; + return `\n\n[${res.filename}](${url})\n\n`; + }); + } + + // Detect which attachments ended up referenced inline so the theme + // can drop them from the trailing Attachments section. Works for + // both ENML- and placeholder-driven substitution paths since both + // emit the same `attachments/evernote/…` storage path. + const inlinedHashesUsed = new Set<string>(); + for (const r of resources) { + const storagePath = `attachments/evernote/${note.guid}/${r.filename || r.hash}`; + const encoded = encodePathForMarkdown(storagePath); + if (markdown.includes(storagePath) || markdown.includes(encoded)) { + inlinedHashesUsed.add(storagePath); + } + } + + // Merge tag GUIDs from the note's NodeFields with any links discovered + // via the Edges table, then resolve to display names. Dedupe so a tag + // referenced both inline and via an edge only appears once. Notebook + // membership is intentionally NOT pushed into tags — it's structured + // metadata, surfaced separately by the theme. + const tagGuidSet = new Set<string>(note.tagGuids ?? []); + for (const g of edgeTagsByNote.get(note.guid) ?? []) tagGuidSet.add(g); + const tagNames: string[] = []; + const seenNames = new Set<string>(); + for (const g of tagGuidSet) { + const name = tagById.get(g)?.name; + if (name && !seenNames.has(name)) { + seenNames.add(name); + tagNames.push(name); + } + } + + // Pre-build a serialisable attachment list the theme can render + // directly — without it the theme has no way to find the assets that + // belong to a note (CrawlerEntityData.attachments doesn't make it onto + // ThemeRenderContext.entity). `inline: true` flags attachments that + // are already substituted into the body so the theme can omit them + // from the trailing Attachments section. + const dataAttachments = attachments.map((a) => { + const storagePath = a.storagePath ?? ""; + return { + filename: a.filename, + mimeType: a.mimeType, + storagePath, + inline: inlinedHashesUsed.has(storagePath), + }; + }); + + const hasher = createCryptoHasher("sha256"); + hasher.update(note.contentHash || ""); + hasher.update(String(note.updateSequenceNum)); + hasher.update(JSON.stringify(tagNames)); + const contentHash = hasher.digest("hex"); + + // Sort key: ISO-8601-ish timestamp of the note's last update so a lex + // sort matches chronological order. Falls back to creation time, then + // to the title — `~` is sentinel that pushes undated notes to the + // bottom under DESC (any real ISO date sorts before `~`). + const sortStamp = (() => { + const ts = note.updated ?? note.created; + if (!ts || !Number.isFinite(ts)) return `~${note.title}`; + const d = new Date(ts); + return Number.isNaN(d.getTime()) ? `~${note.title}` : d.toISOString(); + })(); + + entities.push({ + externalId: note.guid, + entityType: "note", + title: note.title || "(untitled)", + contentHash, + sortKey: sortStamp, + url: `evernote:///view/0/${note.guid}/${note.guid}/`, + tags: tagNames, + data: { + notebookId: note.notebookId, + notebookName: notebook?.name, + created: note.created, + updated: note.updated, + usn: note.updateSequenceNum, + markdown, + attachments: dataAttachments, + }, + attachments: attachments.length > 0 ? attachments : undefined, + }); + + if (note.updateSequenceNum > newHigh) newHigh = note.updateSequenceNum; + } + + highWaterByContext[SYNC_CONTEXT_KEY] = newHigh; + + return { + entities, + nextCursor: { highWaterByContext, knownNodes: newKnownNodes }, + hasMore: false, + deletedExternalIds, + }; + } + + async validateCredentials(credentials: Record<string, unknown>): Promise<boolean> { + const creds = credentials as unknown as EvernoteCredentials; + const dir = creds.conduitStoragePath || defaultConduitPath(); + if (!existsSync(dir)) return false; + return Boolean(findRemoteGraphDb(dir)); + } + + async dispose(): Promise<void> { + if (!this.state) return; + try { + this.state.db?.close?.(); + } catch { + // ignore + } + if (this.state.cleanup) { + try { this.state.cleanup(); } catch { /* ignore */ } + } + this.state = null; + } + + private report(msg: string): void { + this.progressCallback?.(msg); + } + + private matchesNotebookFilter( + note: EvernoteNoteRow, + notebookById: Map<string, EvernoteNotebookRow>, + ): boolean { + if (!this.state?.notebookFilter) return true; + if (!note.notebookId) return false; + const f = this.state.notebookFilter; + if (f.has(note.notebookId)) return true; + const nb = notebookById.get(note.notebookId); + return Boolean(nb && f.has(nb.name)); + } + + /** + * Return paths to try when looking for an ENML file on disk for a note. + * Modern v10 builds embed ENML in NodeFields, so this is only consulted as + * a fallback for older / migrated installs. + */ + private legacyEnmlPaths(noteGuid: string): string[] { + const out: string[] = []; + const userId = this.state?.userId; + if (!userId) return out; + const root = this.state!.evernoteRoot; + out.push(join(root, "accounts", "www.evernote.com", userId, "content", `c${noteGuid}`, "content.enml")); + out.push(join(root, "content", `c${noteGuid}`, "content.enml")); + return out; + } + + /** + * Decode the v10 binary RTE blob for a note and return a best-effort + * plain-text body. Returns an empty string when the file isn't on disk + * (e.g. the note hasn't been opened in the desktop app) or contains no + * recoverable text. + * + * The conduit-fs root is one level deep under + * `<app-support>/conduit-fs/<urlencoded-host>/<account>/rte/Note/internal_rteDoc/`. + * We scan for the account directory once and cache the resolved root. + */ + private cachedRteRoot: string | null | undefined = undefined; + private resolveRteRoot(): string | null { + if (this.cachedRteRoot !== undefined) return this.cachedRteRoot; + const root = this.state!.evernoteRoot; + const conduitFs = `${root}/conduit-fs`; + if (!existsSync(conduitFs)) { + this.cachedRteRoot = null; + return null; + } + const { readdirSync, statSync } = require("fs") as typeof import("fs"); + try { + for (const host of readdirSync(conduitFs)) { + const hostDir = `${conduitFs}/${host}`; + if (!statSync(hostDir).isDirectory()) continue; + for (const account of readdirSync(hostDir)) { + const candidate = `${hostDir}/${account}/rte/Note/internal_rteDoc`; + if (existsSync(candidate)) { + this.cachedRteRoot = candidate; + return candidate; + } + } + } + } catch { + // fall through + } + this.cachedRteRoot = null; + return null; + } + + /** + * Returns either real ENML decoded from the Yjs blob (preferred — preserves + * original document order) or, on failure, the byte-walking extractor's + * placeholder-style output. The caller distinguishes via the returned + * `kind`. + */ + private readRteBody(noteGuid: string): + | { kind: "enml"; enml: string; attachmentHashes: string[] } + | { kind: "fallback"; text: string; attachmentHashes: string[] } + | null { + const root = this.resolveRteRoot(); + if (!root) return null; + const path = rteDatPath(root, noteGuid); + try { + if (!existsSync(path)) return null; + const buf = readFileSync(path); + const decoded = decodeRteBlob(buf); + if (decoded) { + return { kind: "enml", enml: decoded.enml, attachmentHashes: decoded.attachmentHashes }; + } + const fallback = extractRte(buf); + if (fallback.text || fallback.attachmentHashes.length > 0) { + return { kind: "fallback", text: fallback.text, attachmentHashes: fallback.attachmentHashes }; + } + return null; + } catch { + return null; + } + } + + /** + * Read the offline-cached plain text body Evernote stores for notes the + * user has marked for offline access. Only a subset of notes have this, + * but where present it's a clean stripped-text version that we can use as + * a fallback when the RTE blob isn't available. + */ + private readOfflineSearchContent(): Map<string, string> { + const out = new Map<string, string>(); + const db = this.state!.db; + try { + const rows = db + .prepare(`SELECT id, content FROM Offline_Search_Note_Content WHERE content IS NOT NULL AND content != ''`) + .all() as Array<{ id: string; content: string }>; + for (const r of rows) out.set(r.id, r.content); + } catch { + // Table absent on older builds. + } + return out; + } + + /** + * Resolve a resource blob by note guid + content hash. v10 stores them in + * `Evernote/resource-cache/<userId>/<noteGuid>/<hash>` (no extension); some + * installs also keep a copy in the legacy `accounts/.../content/c<guid>/resource/<hash>` + * tree. Try both. + */ + private readResourceBlob(noteGuid: string, hash: string): Buffer | null { + const userId = this.state?.userId; + const root = this.state!.evernoteRoot; + const candidates: string[] = []; + if (userId) { + candidates.push(join(root, "resource-cache", userId, noteGuid, hash)); + candidates.push(join(root, "accounts", "www.evernote.com", userId, "content", `c${noteGuid}`, "resource", hash)); + } + candidates.push(join(root, "resource-cache", noteGuid, hash)); + for (const p of candidates) { + try { + if (existsSync(p)) return readFileSync(p); + } catch { + // try next + } + } + return null; + } + + // ── DB readers ────────────────────────────────────────────────────── + // + // Evernote v10 (Conduit) writes one SQLite table per node type + // (`Nodes_Note`, `Nodes_Notebook`, `Nodes_Tag`, …). Note↔tag links live in + // a `NoteTag` junction table, attachments in `Attachment`, and pre-OCR'd + // attachment text in `AttachmentSearchText`. The note body itself is NOT + // stored in this DB — it lives in a binary RTE blob under + // `<app-support>/conduit-fs/.../rte/Note/internal_rteDoc/<aaa>/<bbb>/<guid>.dat` + // and is not yet parseable by the crawler. We extract everything we can + // from SQL (title, notebook, tags, attachments, OCR text, snippet) and + // leave the body empty — searchable via tags + attachment OCR. + + private readNotes(): EvernoteNoteRow[] { + const db = this.state!.db; + const stmt = db.prepare( + `SELECT id, label AS title, parent_Notebook_id AS notebookId, + content_hash AS contentHash, deleted, version, created, updated, + snippet + FROM Nodes_Note`, + ); + type Row = { + id: string; + title: string | null; + notebookId: string | null; + contentHash: string | null; + deleted: number | null; + version: number; + created: number | null; + updated: number | null; + snippet: string | null; + }; + return (stmt.all() as Row[]).map((r) => ({ + guid: r.id, + title: r.title ?? "", + notebookId: r.notebookId ?? undefined, + contentHash: r.contentHash ?? undefined, + active: !r.deleted, + created: r.created ?? undefined, + updated: r.updated ?? undefined, + // v10 exposes the Evernote `version` field — equivalent to USN for + // change tracking. Falls back to 0 for unsynced local notes. + updateSequenceNum: Math.floor(r.version ?? 0), + tagGuids: undefined, + recognitionByResourceHash: undefined, + enml: r.snippet ?? undefined, + })); + } + + private readNotebooks(): EvernoteNotebookRow[] { + const db = this.state!.db; + type Row = { id: string; name: string | null; version: number | null }; + const rows = db + .prepare(`SELECT id, label AS name, version FROM Nodes_Notebook`) + .all() as Row[]; + return rows.map((r) => ({ + guid: r.id, + name: r.name ?? "(unnamed)", + updateSequenceNum: Math.floor(r.version ?? 0), + })); + } + + private readTags(): EvernoteTagRow[] { + const db = this.state!.db; + type Row = { id: string; name: string | null }; + const rows = db.prepare(`SELECT id, label AS name FROM Nodes_Tag`).all() as Row[]; + return rows.map((r) => ({ guid: r.id, name: r.name ?? "" })); + } + + /** Build the noteGuid → tagGuid[] index from the `NoteTag` junction table. */ + private readTagEdges(_knownTagIds: Set<string>): Map<string, string[]> { + const out = new Map<string, string[]>(); + const db = this.state!.db; + type Row = { Note_id: string; Tag_id: string }; + const rows = db.prepare(`SELECT Note_id, Tag_id FROM NoteTag`).all() as Row[]; + for (const row of rows) { + const list = out.get(row.Note_id) ?? []; + if (!list.includes(row.Tag_id)) list.push(row.Tag_id); + out.set(row.Note_id, list); + } + return out; + } + + private readResources(): Map<string, EvernoteResourceRow[]> { + const out = new Map<string, EvernoteResourceRow[]>(); + const db = this.state!.db; + type Row = { + id: string; + noteGuid: string; + hash: string; + mime: string | null; + filename: string | null; + size: number | null; + }; + const rows = db + .prepare( + `SELECT id, parent_Note_id AS noteGuid, dataHash AS hash, + mime, filename, dataSize AS size + FROM Attachment WHERE isActive = 1`, + ) + .all() as Row[]; + + // Pre-OCR'd searchable text for attachments. Keyed by attachment id. + const recognitionById = new Map<string, string>(); + try { + const recRows = db + .prepare(`SELECT id, searchText FROM AttachmentSearchText`) + .all() as Array<{ id: string; searchText: string | null }>; + for (const r of recRows) { + if (r.searchText) recognitionById.set(r.id, r.searchText); + } + } catch { + // Older builds may lack this table — OCR cascade will fall through. + } + + for (const r of rows) { + const row: EvernoteResourceRow = { + guid: r.id, + noteGuid: r.noteGuid, + hash: r.hash, + mime: r.mime ?? undefined, + filename: r.filename ?? undefined, + size: r.size ?? undefined, + // Evernote's pre-OCR'd text is plain (not the raw recoIndex XML), so + // we wrap it in a minimal recoIndex doc that parseEvernoteRecognitionXml + // already understands. + recognition: recognitionById.has(r.id) + ? recognitionAsXml(recognitionById.get(r.id)!) + : undefined, + }; + const list = out.get(r.noteGuid) ?? []; + list.push(row); + out.set(r.noteGuid, list); + } + return out; + } +} + diff --git a/packages/crawlers/src/evernote/enml.ts b/packages/crawlers/src/evernote/enml.ts new file mode 100644 index 0000000..1f0820a --- /dev/null +++ b/packages/crawlers/src/evernote/enml.ts @@ -0,0 +1,194 @@ +/** + * Convert ENML (Evernote's XHTML-ish note format) into Markdown. + * + * ENML wraps note content in `<en-note>…</en-note>` and uses three custom + * elements: + * <en-media hash="..." type="image/png"/> — embedded resources + * <en-todo checked="true|false"/> — checklist items + * <en-crypt>...</en-crypt> — encrypted blocks + * + * Beyond those, ENML is roughly a subset of XHTML. We do a single-pass + * tag-aware transform without pulling in a full HTML parser, which keeps + * the conversion deterministic and trivially testable. For unknown tags we + * strip the markup but keep the inner text. + */ + +export interface EnmlConvertOptions { + /** + * Map from resource hash → asset path used in the rendered markdown. + * Used to convert `<en-media hash="abc">` into `![](attachments/.../foo.png)` + * or `[foo.pdf](attachments/.../foo.pdf)` depending on MIME type. + */ + resourceByHash: Record< + string, + { filename: string; mimeType: string; assetPath: string } + >; +} + +export function enmlToMarkdown(enml: string, opts: EnmlConvertOptions): string { + // 1. Strip XML/DOCTYPE preamble and <en-note> wrapper. + let body = enml.replace(/<\?xml[\s\S]*?\?>/g, ""); + body = body.replace(/<!DOCTYPE[\s\S]*?>/g, ""); + body = body.replace(/<en-note[^>]*>/i, "").replace(/<\/en-note>/i, ""); + + // 2. <en-crypt> — emit a placeholder; we cannot decrypt without the password. + body = body.replace( + /<en-crypt[\s\S]*?<\/en-crypt>/gi, + "*[encrypted content]*", + ); + + // 3. <en-todo> — render as a GFM checkbox. + body = body.replace( + /<en-todo([^>]*)\/?>(?:<\/en-todo>)?/gi, + (_match, attrs: string) => { + const checked = /checked\s*=\s*["']true["']/i.test(attrs); + return checked ? "- [x] " : "- [ ] "; + }, + ); + + // 4. <en-media> — replace with an inline reference. Images and PDFs use + // the `![label](url)` image syntax so the UI's `<img>` override can pick + // them up: PDFs become an inline `<object>` viewer; everything else + // renders as a normal `<img>`. URLs are percent-encoded segment-by- + // segment because Evernote filenames frequently contain spaces, which + // would otherwise break markdown link parsing. + body = body.replace(/<en-media([^>]*)\/?>(?:<\/en-media>)?/gi, (_m, attrs: string) => { + const hash = matchAttr(attrs, "hash"); + const declaredMime = matchAttr(attrs, "type") ?? ""; + if (!hash) return ""; + const res = opts.resourceByHash[hash]; + if (!res) return ""; + const url = encodePathForMarkdown(res.assetPath); + const mime = res.mimeType || declaredMime; + const isImage = mime.startsWith("image/"); + const isPdf = mime === "application/pdf" || /\.pdf$/i.test(res.filename); + if (isImage || isPdf) { + // Surround with blank lines so the embed/img isn't rendered inside an + // inline paragraph (which suppresses block-level styling like the + // full-width PDF viewer). + return `\n\n![${res.filename}](${url})\n\n`; + } + return `[${res.filename}](${url})`; + }); + + // 5. Block-level constructs. Order matters: process tables/lists/headings + // before the generic tag stripper. + body = body.replace(/<br\s*\/?>(?!\n)/gi, "\n"); + body = body.replace(/<p[^>]*>/gi, "").replace(/<\/p>/gi, "\n\n"); + // <div> is the default block wrapper Evernote v10 uses for paragraphs. + // Treat the opening tag as a hint to start a block and the closing tag + // as a paragraph break so adjacent <div>...<en-media>...<div> chains + // don't run together on a single line. + body = body.replace(/<div[^>]*>/gi, "\n").replace(/<\/div>/gi, "\n"); + + for (let level = 6; level >= 1; level--) { + const re = new RegExp(`<h${level}[^>]*>([\\s\\S]*?)<\\/h${level}>`, "gi"); + const hashes = "#".repeat(level); + body = body.replace(re, (_m, inner: string) => `\n\n${hashes} ${stripTags(inner).trim()}\n\n`); + } + + body = body.replace(/<strong>([\s\S]*?)<\/strong>/gi, "**$1**"); + body = body.replace(/<b>([\s\S]*?)<\/b>/gi, "**$1**"); + body = body.replace(/<em>([\s\S]*?)<\/em>/gi, "*$1*"); + body = body.replace(/<i>([\s\S]*?)<\/i>/gi, "*$1*"); + body = body.replace(/<code>([\s\S]*?)<\/code>/gi, "`$1`"); + body = body.replace(/<a([^>]*)>([\s\S]*?)<\/a>/gi, (_m, attrs: string, inner: string) => { + const href = matchAttr(attrs, "href"); + const label = stripTags(inner).trim(); + if (!href) return label; + return `[${label || href}](${href})`; + }); + + // Lists — ENML uses standard <ul>/<ol>/<li>. + body = body.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_m, inner: string) => { + return `- ${stripTags(inner).trim()}\n`; + }); + body = body.replace(/<\/?(?:ul|ol)[^>]*>/gi, "\n"); + + // Tables — collapse rows to pipe-delimited markdown rows; not perfect but + // searchable. + body = body.replace(/<tr[^>]*>([\s\S]*?)<\/tr>/gi, (_m, inner: string) => { + const cells = Array.from(inner.matchAll(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi)).map( + (c) => stripTags(c[1]).trim(), + ); + if (cells.length === 0) return ""; + return `| ${cells.join(" | ")} |\n`; + }); + body = body.replace(/<\/?(?:table|thead|tbody|tfoot)[^>]*>/gi, ""); + + // 6. Strip remaining tags but keep their text content. + body = stripTags(body); + + // 7. Decode core HTML entities. + body = decodeEntities(body); + + // 8. Collapse runs of blank lines. + body = body.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim(); + return body + (body.endsWith("\n") ? "" : "\n"); +} + +function matchAttr(attrs: string, name: string): string | undefined { + const m = attrs.match(new RegExp(`${name}\\s*=\\s*["']([^"']*)["']`, "i")); + return m ? m[1] : undefined; +} + +function stripTags(html: string): string { + return html.replace(/<[^>]+>/g, ""); +} + +function decodeEntities(s: string): string { + return s + .replace(/ /g, " ") + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/'/g, "'") + .replace(/&#(\d+);/g, (_m, code: string) => String.fromCodePoint(Number(code))); +} + +/** + * Parse the OCR `<recoIndex>` blob Evernote stores for an image resource. + * The XML looks like + * <recoIndex …><item …><t w="80">Hello</t><t w="60">World</t></item>…</recoIndex> + * We pick the highest-confidence token (`w` weight) per `<item>` and + * concatenate. Returns an empty string if the blob is empty or unparseable. + */ +export function parseEvernoteRecognitionXml(xml: string): string { + if (!xml || !xml.includes("<t")) return ""; + const tokens: string[] = []; + const itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi; + let item: RegExpExecArray | null; + while ((item = itemRegex.exec(xml)) !== null) { + const inner = item[1]; + let best = { weight: -1, text: "" }; + const tRegex = /<t(?:\s+w=["'](\d+)["'])?[^>]*>([\s\S]*?)<\/t>/gi; + let t: RegExpExecArray | null; + while ((t = tRegex.exec(inner)) !== null) { + const weight = t[1] ? Number(t[1]) : 0; + const text = decodeEntities(t[2]).trim(); + if (text && weight > best.weight) best = { weight, text }; + } + if (best.text) tokens.push(best.text); + } + return tokens.join(" "); +} + +/** + * Percent-encode a path segment-by-segment for use inside a markdown + * `[label](url)` or `![label](url)` link. `encodeURIComponent` leaves a few + * unreserved characters alone (`!*'()`) — three of those (`*`, `(`, `)`) + * are markdown special characters that wreck link parsing when they + * appear in the URL portion. Encode them too so filenames like + * `11292025_*000779*…` don't get mis-parsed as italic markers mid-URL. + */ +export function encodePathForMarkdown(path: string): string { + return path + .split("/") + .map((seg) => encodeURIComponent(seg).replace( + /[!*'()]/g, + (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`, + )) + .join("/"); +} diff --git a/packages/crawlers/src/evernote/ocr.ts b/packages/crawlers/src/evernote/ocr.ts new file mode 100644 index 0000000..0097205 --- /dev/null +++ b/packages/crawlers/src/evernote/ocr.ts @@ -0,0 +1,169 @@ +import { spawnSync } from "child_process"; +import { writeFileSync, mkdtempSync, rmSync } from "fs"; +import { tmpdir, platform } from "os"; +import { join } from "path"; +import { parseEvernoteRecognitionXml } from "./enml"; + +/** + * Three-tier OCR cascade. The first tier that returns non-empty text wins. + * + * 1. Evernote-provided `recognition` XML — free, exact, no work. + * 2. Apple Vision (macOS only) via a small Swift helper invoked through + * `child_process.execFile`. Handles images natively; PDFs are rasterized + * with `pdftoppm` if it's installed, otherwise skipped at this tier. + * 3. Tesseract.js (+ pdfjs-dist for PDFs) WASM fallback. These are dynamic + * imports — if the modules aren't installed the tier is skipped. + * + * The cascade is best-effort: any exception inside a tier collapses to an + * empty string and falls through to the next tier. + */ +export interface OcrInput { + /** Raw bytes of the attachment. */ + buffer: Buffer; + /** MIME type as reported by Evernote (e.g. `image/png`, `application/pdf`). */ + mimeType: string; + /** Original filename, used as a hint for extension-based heuristics. */ + filename: string; + /** Optional Evernote-supplied OCR XML for this resource. */ + evernoteRecognitionXml?: string; + /** Per-attachment cap; oversized files are skipped entirely. */ + maxBytes?: number; +} + +export async function extractAttachmentText(input: OcrInput): Promise<string> { + if (input.maxBytes && input.buffer.length > input.maxBytes) return ""; + + const isImage = input.mimeType.startsWith("image/"); + const isPdf = + input.mimeType === "application/pdf" || + input.filename.toLowerCase().endsWith(".pdf"); + if (!isImage && !isPdf) return ""; + + // Tier 1 — Evernote recognition. + if (input.evernoteRecognitionXml) { + const text = parseEvernoteRecognitionXml(input.evernoteRecognitionXml); + if (text.trim()) return text; + } + + // Tier 2 — Apple Vision on macOS. Only attempted when the buffer looks + // like a real image (magic-byte sniff) so we don't fork/exec for every + // junk byte sequence we encounter — particularly important in tests. + if (platform() === "darwin" && looksLikeImage(input.buffer)) { + try { + const text = await runAppleVision(input.buffer, input.mimeType, input.filename); + if (text.trim()) return text; + } catch { + // fall through + } + } + + // Tier 3 — Tesseract / pdfjs fallback (lazy, optional). + try { + const text = await runTesseract(input); + if (text.trim()) return text; + } catch { + // give up + } + return ""; +} + +function looksLikeImage(buf: Buffer): boolean { + if (buf.length < 8) return false; + // PNG: 89 50 4E 47 0D 0A 1A 0A + if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4e && buf[3] === 0x47) return true; + // JPEG: FF D8 FF + if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return true; + // GIF: GIF87a / GIF89a + if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) return true; + // BMP: BM + if (buf[0] === 0x42 && buf[1] === 0x4d) return true; + // WebP: RIFF....WEBP + if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46) return true; + return false; +} + +async function runAppleVision( + buffer: Buffer, + mimeType: string, + filename: string, +): Promise<string> { + // The helper writes the input to a temp file then runs a one-shot Swift + // VNRecognizeTextRequest. Image-only — PDFs need rasterization first. + if (!mimeType.startsWith("image/")) return ""; + const tmp = mkdtempSync(join(tmpdir(), "frozenink-vision-")); + const inPath = join(tmp, filename || "img"); + try { + writeFileSync(inPath, buffer); + const swift = ` +import Foundation +import Vision +import AppKit + +let url = URL(fileURLWithPath: CommandLine.arguments[1]) +guard let img = NSImage(contentsOf: url), + let cg = img.cgImage(forProposedRect: nil, context: nil, hints: nil) else { + exit(0) +} +let req = VNRecognizeTextRequest() +req.recognitionLevel = .accurate +let handler = VNImageRequestHandler(cgImage: cg, options: [:]) +do { try handler.perform([req]) } catch { exit(0) } +let lines = (req.results ?? []).compactMap { $0.topCandidates(1).first?.string } +print(lines.joined(separator: "\\n")) +`; + const scriptPath = join(tmp, "ocr.swift"); + writeFileSync(scriptPath, swift); + const res = spawnSync("swift", [scriptPath, inPath], { + encoding: "utf-8", + timeout: 30_000, + }); + if (res.status !== 0) return ""; + return res.stdout?.trim() ?? ""; + } finally { + try { rmSync(tmp, { recursive: true, force: true }); } catch { /* ignore */ } + } +} + +async function runTesseract(input: OcrInput): Promise<string> { + // Both deps are optional; if they aren't installed we silently no-op so + // the crawler keeps working without the WASM dependency footprint. + type TesseractModule = { + recognize: (img: Buffer | Uint8Array, lang: string) => Promise<{ data: { text: string } }>; + }; + let tesseract: TesseractModule | null = null; + try { + tesseract = (await import( + /* @vite-ignore */ "tesseract.js" as string + )) as TesseractModule; + } catch { + return ""; + } + if (!tesseract) return ""; + const recognize = tesseract.recognize; + const isPdf = + input.mimeType === "application/pdf" || + input.filename.toLowerCase().endsWith(".pdf"); + + if (!isPdf) { + const result = await recognize(input.buffer, "eng"); + return result.data?.text?.trim() ?? ""; + } + + // PDF: try pdfjs-dist first for the embedded text layer; rasterize pages + // that come back empty and run Tesseract on them. + let pdfjs: any = null; + try { + pdfjs = await import(/* @vite-ignore */ "pdfjs-dist/legacy/build/pdf.mjs" as string); + } catch { + return ""; + } + const doc = await pdfjs.getDocument({ data: new Uint8Array(input.buffer) }).promise; + const out: string[] = []; + for (let i = 1; i <= doc.numPages; i++) { + const page = await doc.getPage(i); + const tc = await page.getTextContent(); + const pageText = (tc.items as Array<{ str: string }>).map((it) => it.str).join(" ").trim(); + if (pageText) out.push(pageText); + } + return out.join("\n"); +} diff --git a/packages/crawlers/src/evernote/rte.ts b/packages/crawlers/src/evernote/rte.ts new file mode 100644 index 0000000..3e8274f --- /dev/null +++ b/packages/crawlers/src/evernote/rte.ts @@ -0,0 +1,400 @@ +/** + * Decoder for Evernote v10's binary RTE (`*.dat`) files. + * + * v10 stores note bodies under + * `<app-support>/conduit-fs/.../rte/Note/internal_rteDoc/<aaa>/<bbb>/<noteGuid>.dat` + * as Yjs CRDT updates. The document is a Y.Doc with shared types named + * `content` (Y.XmlFragment with the ENML tree), `title` (Y.Text), + * `customNoteStyles` / `meta` / `resources` (Y.Map metadata). Decoding via + * the `yjs` library reconstructs the full XML tree in document order, so + * the body comes back as proper ENML — text, `<en-media>` references, + * checklists, and tables all in their original positions. + * + * A best-effort byte-walking fallback (`extractRte`) is kept for cases + * where Yjs decoding fails (corrupted file, unknown variant, etc.). + */ +import * as Y from "yjs"; + +const MIN_STRING_LEN = 2; +const MAX_STRING_LEN = 4096; + +/** Document-structure keys, tag names, and CSS-style values that are not body content. */ +const STRUCTURAL_TOKENS = new Set([ + "content", "en-note", "en-media", "en-todo", "en-crypt", "en-codeblock", + "div", "span", "p", "br", "hr", "ul", "ol", "li", "table", "tr", "td", "th", + "tbody", "thead", "tfoot", "h1", "h2", "h3", "h4", "h5", "h6", + "strong", "b", "em", "i", "u", "s", "code", "a", "img", + "hash", "type", "style", "title", "alt", "src", "href", "width", "height", + "meta", "resources", "customNoteStyles", "headingStyles", + "taskGroupOrder", "calendarEventIds", "lastENMLNormalizationVersion", + "schemaVersion", "taskIdClocksMap", + "checked", "true", "false", + // Common style-attribute *values* the Yjs document carries inline. Each of + // these surfaces as its own length-prefixed string in the binary stream + // and would otherwise leak into the body as e.g. + // "isEmpty fontFamilyw inherit colorw 0 fontStylew". + "isEmpty", "inherit", "initial", "unset", "normal", "auto", "none", + "default", "bold", "italic", "underline", "transparent", "left", "right", + "center", "justify", + // Common field names — also enumerated in KNOWN_FIELD_PREFIXES below for + // the prefix-stripping pass. Listing them here as well makes + // startsWithStructuralKey pick up `colorw`, `paddingw`, etc. — field + // names with a leaked single-char length byte glued to the end. + "color", "background", "padding", "margin", "border", + "fontFamily", "fontSize", "fontWeight", "fontStyle", "textDecoration", + "image/png", "image/jpeg", "image/gif", "image/webp", "image/svg+xml", + "application/pdf", "application/octet-stream", +]); + +/** + * Walk the buffer and pull out every plausible string. Strings are runs of + * printable ASCII bytes (0x20–0x7E plus tab/LF/CR); high bytes (>= 0x80) + * are treated as separators because Yjs uses them for varint length + * continuations and CRDT identifiers — not as UTF-8 multi-byte content + * (real Evernote notes are overwhelmingly ASCII; the few notes with true + * non-ASCII characters lose those characters but stay searchable in the + * surrounding ASCII). + */ +function* extractStrings(buf: Buffer): Generator<string> { + let i = 0; + while (i < buf.length) { + while (i < buf.length && !isPrintableAscii(buf[i])) i++; + const start = i; + while (i < buf.length && isPrintableAscii(buf[i])) i++; + if (i - start < MIN_STRING_LEN) continue; + const len = Math.min(i - start, MAX_STRING_LEN); + yield buf.subarray(start, start + len).toString("ascii"); + } +} + +function isPrintableAscii(b: number): boolean { + if (b === 0x09 || b === 0x0a || b === 0x0d) return true; + return b >= 0x20 && b <= 0x7e; +} + +/** + * True when a string looks like a content-addressable hash. We accept 16+ + * hex chars because varint length bytes often chop a leading character off + * a 32-byte hash, leaving 30/31-char fragments. Body text rarely contains + * hex runs that long without intervening spaces or vowels. + */ +function isHashLike(s: string): boolean { + const trimmed = s.replace(/^[^a-f0-9]+|[^a-f0-9]+$/gi, ""); + return /^[a-f0-9]{16,}$/i.test(trimmed); +} + +/** + * True when the string starts with a known structural key, even if extra + * characters are concatenated to the end (e.g. `customNoteStylesheadingStyles`, + * which Yjs sometimes emits as a single run). + */ +function startsWithStructuralKey(s: string): boolean { + for (const k of STRUCTURAL_TOKENS) { + if (!s.startsWith(k)) continue; + if (s.length === k.length) return true; + const next = s[k.length]; + if (next === next.toUpperCase() && /[A-Z]/.test(next)) return true; // CamelCase concat + if (/[\W]/.test(next)) return true; + // Field name + single trailing character — almost always a length byte + // (e.g. `colorw`, `paddingw`, `titleq`). Real body text doesn't end + // mid-field-name plus one stray letter. + if (s.length === k.length + 1) return true; + } + return false; +} + +/** True when a string is obviously a CSS style declaration block, not body text. */ +function isStyleLike(s: string): boolean { + if (s.includes("--en-")) return true; + if (s.startsWith("--")) return true; + // Anything starting with a CSS property declaration (`display:none;…`, + // `font-family:foo;`). The block can carry a `--en-chs:"<base64>"` payload + // afterwards, which the original regex missed because it required the + // *entire* string to be CSS-shaped. + if (/^[a-z-]+\s*:\s*[^;]+;/i.test(s)) return true; + return /^(?:[a-z-]+\s*:\s*[^;]+;\s*)+$/i.test(s); +} + +/** + * True when the string contains a long unbroken alphanumeric/base64 run + * (60+ chars). Evernote serialises per-heading style configs as + * `--en-chs:"<base64-encoded JSON>"` and the base64 payload is several + * hundred characters of pure noise from the user's perspective. + */ +function containsLongOpaqueRun(s: string): boolean { + return /[A-Za-z0-9+/=]{60,}/.test(s); +} + +/** + * Field-name prefixes that the binary stream often pastes onto the front + * of the field's value when the length byte happens to be printable ASCII. + * `title<2><body>` becomes `title2<body>` once decoded as plain text. + * Strip the prefix so the actual value reads cleanly. + */ +const KNOWN_FIELD_PREFIXES = [ + "title", "color", "alt", "src", "href", "type", "hash", "width", "height", + "fontFamily", "fontSize", "fontWeight", "fontStyle", "textDecoration", + "background", "padding", "margin", "border", +]; + +/** + * Strings that contain a known field-name as a *suffix* are usually the + * remnant of two concatenated fields; trim everything from the field-name + * onward. e.g. `Reclamation District Bill colorw 0 fontStylew` becomes + * `Reclamation District Bill`. + */ +function stripTrailingFieldRun(s: string): string { + const re = new RegExp( + `\\s+(?:${KNOWN_FIELD_PREFIXES.join("|")})[a-zA-Z]?\\b.*$`, + ); + return s.replace(re, ""); +} + +/** + * True when a string is a single camelCase identifier (no whitespace, no + * punctuation) — overwhelmingly a Yjs/CRDT field name like `fontFamily`, + * `textDecoration`, `fontWeightw` (with a leaked length byte). Real body + * text contains spaces, sentences, or punctuation; legitimate single + * identifiers in note bodies are rare enough to accept the trade-off. + */ +function isCamelCaseIdentifier(s: string): boolean { + if (s.length > 40) return false; + return /^[a-z][a-z0-9]*[A-Z][a-zA-Z0-9]*$/.test(s); +} + +/** + * Yjs frequently emits short noise tokens like `w 5e47…` (a 1-char field + * key followed by a hex resource id) immediately before each en-media + * reference. Strip those — they're never user-authored content. + */ +function isResourceRefLike(s: string): boolean { + return /^[a-z][\s\W]+[a-f0-9]{16,}\b/i.test(s); +} + +/** + * Some short fragments are pure protocol noise: a single ASCII byte plus + * a separator, two-letter tags, etc. Drop them. + */ +function isLikelyNoise(s: string): boolean { + if (s.length <= 2) return true; + // Standalone CSS-ish leftovers like ");" or "}; " + if (/^[\s\W]+$/.test(s)) return true; + // CSS dimension values — `25px`, `16em`, `100%`, `2rem`, etc. show up as + // their own length-prefixed entries when style attributes get serialised. + if (/^\d+(\.\d+)?(px|em|rem|pt|pc|%|vh|vw|deg)$/i.test(s)) return true; + // High punctuation density usually means a binary stub leaked through. + const wordChars = (s.match(/[a-zA-Z]/g) ?? []).length; + if (wordChars < 2) return true; + if (wordChars / s.length < 0.3) return true; + return false; +} + +/** + * Strip a single leading "key character" followed by whitespace, e.g. + * `w foo` → `foo`, `( bar` → `bar`. Yjs leaves these markers in front of + * many strings; trimming them keeps the body clean. + */ +function trimLeadingNoise(s: string): string { + let out = s; + // Strip a known field-name + length-byte prefix when the value is glued + // onto the field key (e.g. `title220260331 230 Lake…` → + // `20260331 230 Lake…`). Length bytes that are printable ASCII letters + // get absorbed too — match up to two trailing characters between the + // field name and the value. + // Length bytes are exactly one byte; when they fall in the printable + // range they get pasted onto the field name as a single character. + // Match at most one trailing char so we don't eat into the actual value. + const fieldPrefixRe = new RegExp(`^(?:${KNOWN_FIELD_PREFIXES.join("|")})[a-zA-Z0-9]?(?=[A-Z0-9"'])`); + out = out.replace(fieldPrefixRe, ""); + // Strip a single ASCII byte (letter, digit, or symbol) followed by space + // — Yjs leaves a 1-char field key in front of many strings. + out = out.replace(/^[A-Za-z0-9\W]\s+/, ""); + // Strip leading single non-letter, non-digit, non-quote bytes that + // aren't separated by whitespace (e.g. "(hello"). Digits are preserved + // here so legitimate leading dates like `20260331 …` survive — `0--en-…` + // junk is caught later by isStyleLike's `--en-` includes check. + while (/^[^a-zA-Z0-9"'\s]\S/.test(out)) out = out.slice(1); + // Strip trailing field-separator bytes like `(`, `'`, `"`. + out = out.replace(/[\s(){}\[\]'"`]+$/, ""); + // Strip a single trailing uppercase letter that immediately follows a + // lowercase letter (e.g. `Bulk Create IssuesG`). That's almost always + // the next field's length-byte leaking — real English text doesn't end + // mid-camelcase. Repeat to handle `…PageGn` style two-byte tails. + while (/[a-z][A-Z]$/.test(out)) out = out.slice(0, -1); + // Cut off concatenated trailing field runs. + out = stripTrailingFieldRun(out); + return out.trim(); +} + +/** Placeholder emitted in the body where an en-media reference appears. */ +export const ATTACHMENT_PLACEHOLDER_PREFIX = "[[EVERNOTE_ATTACHMENT:"; +export const ATTACHMENT_PLACEHOLDER_SUFFIX = "]]"; + +export interface YjsDecodeResult { + /** Raw ENML reconstructed from the Yjs document, in original order. */ + enml: string; + /** Hashes of every `<en-media>` reference inside the body, in order. */ + attachmentHashes: string[]; + /** Title stored on the document (separate Y.Text shared type). */ + title?: string; +} + +/** + * Decode the binary RTE blob via the Yjs library. Returns full ENML in + * the document's original order, which `enmlToMarkdown` can then convert + * to clean markdown. Returns null on any decode error so callers can fall + * back to the byte-walking extractor. + */ +export function decodeRteBlob(buf: Buffer): YjsDecodeResult | null { + let doc: Y.Doc; + try { + doc = new Y.Doc(); + Y.applyUpdate(doc, new Uint8Array(buf)); + } catch { + return null; + } + + let enml: string; + try { + enml = doc.getXmlFragment("content").toString(); + } catch { + return null; + } + if (!enml || !enml.includes("<en-note")) return null; + + // Walk for ordered en-media hashes — useful so the crawler knows which + // attachments were referenced inline (vs. orphans to dump at the end). + const attachmentHashes: string[] = []; + const seen = new Set<string>(); + const re = /<en-media[^>]*\bhash\s*=\s*"([a-f0-9]{16,})"/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(enml)) !== null) { + const hash = m[1].toLowerCase(); + if (!seen.has(hash)) { + seen.add(hash); + attachmentHashes.push(hash); + } + } + + let title: string | undefined; + try { + const t = doc.getText("title").toString(); + if (t) title = t; + } catch { + // optional + } + + return { enml, attachmentHashes, title }; +} + +export interface RteExtractResult { + /** + * Recovered body text. May contain inline placeholders of the form + * `[[EVERNOTE_ATTACHMENT:<hash>]]` at the positions where an `en-media` + * marker appeared in the binary stream — callers substitute these for + * the matching attachment ref so images/PDFs render at their original + * location instead of being grouped at the bottom of the note. + */ + text: string; + /** Distinct attachment hashes encountered in the binary, in order. */ + attachmentHashes: string[]; +} + +/** + * Extract a plain-text body plus a list of inline attachment hashes from a + * note's `.dat` file. Returns an empty result for empty or unrecognised + * files. + * + * Ordering: en-media blocks live at the top of the binary in the document's + * structural metadata, while the body text appears later. The CRDT tree + * that defines where each attachment renders within the body isn't + * decoded — we don't have a Yjs parser. The pragmatic compromise is to + * emit body text first, then append attachment placeholders. This matches + * the common Evernote pattern of "short note body + attached document(s) + * below" and renders the way the user's screenshots show: Evernote-style + * note opens with text, PDFs follow. + */ +export function extractRte(buf: Buffer): RteExtractResult { + const bodyParts: string[] = []; + const seenText = new Set<string>(); + const attachmentHashes: string[] = []; + const seenAttachment = new Set<string>(); + + const stringList = [...extractStrings(buf)]; + + for (let i = 0; i < stringList.length; i++) { + const raw = stringList[i]; + const trimmed = raw.trim(); + + // en-media boundary: scan ahead for the resource hash that follows. + if (trimmed === "en-media" || /^en-media\b/.test(trimmed)) { + const hash = findHashNear(stringList, i + 1); + if (hash && !seenAttachment.has(hash)) { + seenAttachment.add(hash); + attachmentHashes.push(hash); + } + continue; + } + + const s = trimLeadingNoise(trimmed); + if (s.length < MIN_STRING_LEN) continue; + if (STRUCTURAL_TOKENS.has(s)) continue; + if (startsWithStructuralKey(s)) continue; + if (isHashLike(s)) continue; + if (isStyleLike(s)) continue; + if (containsLongOpaqueRun(s)) continue; + if (isResourceRefLike(s)) continue; + if (isCamelCaseIdentifier(s)) continue; + if (isLikelyNoise(s)) continue; + if (/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(s)) continue; + if (seenText.has(s)) continue; + seenText.add(s); + bodyParts.push(s); + } + + const placeholders = attachmentHashes.map( + (hash) => `${ATTACHMENT_PLACEHOLDER_PREFIX}${hash}${ATTACHMENT_PLACEHOLDER_SUFFIX}`, + ); + const text = [bodyParts.join("\n"), placeholders.join("\n")] + .filter((p) => p.length > 0) + .join("\n\n") + .trim(); + + return { text, attachmentHashes }; +} + +/** Find the next 30–32-char hex run in the next few strings; returns lowercase normalised hash. */ +function findHashNear(strings: string[], from: number): string | null { + for (let j = from; j < Math.min(from + 6, strings.length); j++) { + const m = strings[j].match(/[a-f0-9]{30,32}/i); + if (!m) continue; + let hash = m[0].toLowerCase(); + // Length bytes occasionally chop the leading hex char off a 32-byte + // hash — keep the tail and let the crawler match it loosely. + if (hash.length === 32) return hash; + return hash; + } + return null; +} + +/** Backwards-compatible wrapper that returns just the text body. */ +export function extractTextFromRteBlob(buf: Buffer): string { + return extractRte(buf).text; +} + +/** + * Resolve the on-disk path to a note's `.dat` file given the conduit-fs + * root and the note's GUID. Returns null if no matching file is found. + * + * The path layout is sharded by GUID: + * `<rteRoot>/<first-3-hex>/<last-3-hex>/<guid>.dat` + * where first-3 / last-3 are taken from the un-hyphenated GUID's first and + * last three characters (NOT bytes — characters of the original guid). + */ +export function rteDatPath(rteRoot: string, noteGuid: string): string { + const first = noteGuid.slice(0, 3); + const last = noteGuid.slice(-3); + // join() lives in node:path but importers may not have it; do it by hand + // to keep this module dependency-free for downstream tests. + return `${rteRoot}/${first}/${last}/${noteGuid}.dat`; +} diff --git a/packages/crawlers/src/evernote/snapshot.ts b/packages/crawlers/src/evernote/snapshot.ts new file mode 100644 index 0000000..32aef62 --- /dev/null +++ b/packages/crawlers/src/evernote/snapshot.ts @@ -0,0 +1,111 @@ +import { mkdtempSync, copyFileSync, existsSync, rmSync, readdirSync, statSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +export interface ConduitSnapshot { + /** Absolute path to the copied DB file inside the temp directory. */ + dbPath: string; + /** Removes the temp directory and all snapshot files. */ + cleanup: () => void; +} + +/** + * Find the active RemoteGraph SQLite database under a conduit-storage tree. + * + * Evernote v10 nests the per-account databases one level deep under a + * URL-encoded host directory (e.g. + * `conduit-storage/https%3A%2F%2Fwww.evernote.com/UDB-User1234+RemoteGraph.sql`). + * Older builds wrote them directly inside `conduit-storage/`. We accept both. + * The matching rule: filename ends with `+RemoteGraph.sql` (the trailing + * literal excludes `+BackupRemoteGraph.sql` and `+LocalStorage.sql`). When + * multiple candidates exist (e.g. several accounts) we return the most + * recently modified one. + */ +export function findRemoteGraphDb(conduitStorageDir: string): string | null { + if (!existsSync(conduitStorageDir)) return null; + const candidates: { path: string; mtime: number }[] = []; + walk(conduitStorageDir, 2, (path) => { + const name = path.split("/").pop() ?? ""; + if (!name.endsWith("+RemoteGraph.sql")) return; + try { + candidates.push({ path, mtime: statSync(path).mtimeMs }); + } catch { + // skip unreadable + } + }); + if (candidates.length === 0) return null; + candidates.sort((a, b) => b.mtime - a.mtime); + return candidates[0].path; +} + +function walk(dir: string, maxDepth: number, visit: (path: string) => void): void { + if (maxDepth < 0) return; + let entries: import("fs").Dirent[] = []; + try { + entries = readdirSync(dir, { withFileTypes: true }) as import("fs").Dirent[]; + } catch { + return; + } + for (const e of entries) { + const full = join(dir, String(e.name)); + if (e.isDirectory()) walk(full, maxDepth - 1, visit); + else if (e.isFile()) visit(full); + } +} + +/** + * Returns true when the DB has uncheckpointed WAL pages — i.e. Evernote + * (or some other writer) currently has the database open with pending + * writes. When this returns false the DB file on disk is internally + * consistent and can be opened read-only without the snapshot dance. + */ +export function isWalDirty(dbPath: string): boolean { + const wal = `${dbPath}-wal`; + if (!existsSync(wal)) return false; + try { + return statSync(wal).size > 0; + } catch { + // If we can't stat the WAL, assume the worst and snapshot. + return true; + } +} + +/** + * Copy the conduit-storage SQLite DB (plus its `-wal` and `-shm` siblings, + * if present) into a fresh temp directory so that we can read a stable + * snapshot even while Evernote is running. Returns the new DB path and a + * cleanup function that removes the temp directory. + */ +export function copyConduitStorage(srcDir: string): ConduitSnapshot { + const dbSrc = findRemoteGraphDb(srcDir); + if (!dbSrc) { + throw new Error( + `No Evernote RemoteGraph DB found in ${srcDir}. Is Evernote v10 installed and signed in?`, + ); + } + + const tmp = mkdtempSync(join(tmpdir(), "frozenink-evernote-")); + const baseName = dbSrc.split("/").pop()!; + const dbDst = join(tmp, baseName); + + // Copy DB + sidecars together so the snapshot is internally consistent + // (WAL pages may be needed to reconstruct the latest state). + copyFileSync(dbSrc, dbDst); + for (const suffix of ["-wal", "-shm"]) { + const side = `${dbSrc}${suffix}`; + if (existsSync(side)) { + copyFileSync(side, `${dbDst}${suffix}`); + } + } + + return { + dbPath: dbDst, + cleanup: () => { + try { + rmSync(tmp, { recursive: true, force: true }); + } catch { + // best-effort cleanup + } + }, + }; +} diff --git a/packages/crawlers/src/evernote/theme.ts b/packages/crawlers/src/evernote/theme.ts new file mode 100644 index 0000000..c56baef --- /dev/null +++ b/packages/crawlers/src/evernote/theme.ts @@ -0,0 +1,168 @@ +import type { Theme, ThemeRenderContext } from "@frozenink/core/theme"; +import { frontmatter } from "@frozenink/core/theme"; +import { encodePathForMarkdown } from "./enml"; + +/** Tiny slug helper used to build per-notebook folders. */ +function slugify(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 80) || "note"; +} + + +export class EvernoteTheme implements Theme { + crawlerType = "evernote"; + + render(context: ThemeRenderContext): string { + const data = context.entity.data as Record<string, unknown>; + const markdown = (data.markdown as string | undefined) ?? ""; + const created = data.created as number | undefined; + const updated = data.updated as number | undefined; + const notebookName = data.notebookName as string | undefined; + const tags = (context.entity.tags ?? []) as string[]; + const allAttachments = (data.attachments as Array<{ + filename: string; + mimeType: string; + storagePath: string; + inline?: boolean; + }> | undefined) ?? []; + // Inline attachments are already substituted into the body by the + // crawler — show them only once. The trailing list is reserved for + // orphans that the body didn't reference. + const orphanAttachments = allAttachments.filter((a) => !a.inline); + + const fm = frontmatter({ + title: context.entity.title, + ...(notebookName ? { notebook: notebookName } : {}), + ...(created ? { created: new Date(created).toISOString() } : {}), + ...(updated ? { updated: new Date(updated).toISOString() } : {}), + ...(tags.length > 0 ? { tags } : {}), + ...(context.entity.url ? { source: context.entity.url } : {}), + }); + + const heading = `# ${context.entity.title}\n`; + + // Notebook + tags as a structured metadata line above the body. Notebook + // is shown by its display name (the `label` from Nodes_Notebook), + // separated from the tag chips so the user sees which notebook the note + // came from without it polluting the tag set. + const metaParts: string[] = []; + if (notebookName) metaParts.push(`📓 **Notebook:** ${notebookName}`); + if (tags.length > 0) { + metaParts.push(`🏷 ${tags.map((t) => `#${t.replace(/\s+/g, "_")}`).join(" ")}`); + } + const metaLine = metaParts.length > 0 ? `\n${metaParts.join("  ·  ")}\n` : ""; + + // Inline attachments. Images render via standard markdown image syntax; + // PDFs use the same `![label](path)` shape so the UI's `<img>` override + // can detect the .pdf extension and swap in an `<embed>` for inline + // viewing. Other file types degrade to a plain link. URLs are + // percent-encoded segment-by-segment because Evernote filenames often + // contain spaces — markdown parsers stop at the first whitespace inside + // `(...)`, breaking the link entirely. + const attachmentBlock = orphanAttachments.length > 0 + ? `\n## Attachments\n\n${orphanAttachments + .map((a) => { + const url = encodePathForMarkdown(a.storagePath); + const isImage = a.mimeType.startsWith("image/"); + const isPdf = a.mimeType === "application/pdf" || /\.pdf$/i.test(a.filename); + if (isImage || isPdf) return `![${a.filename}](${url})`; + return `[${a.filename}](${url})`; + }) + .join("\n\n")}\n` + : ""; + + // Defensive cleanup of stale rendered markdown: older sync runs emitted + // PDF references as plain markdown links `[file.pdf](url)` instead of + // image-syntax `![file.pdf](url)`, so the UI's `<img>` override never + // fired and PDFs showed as a clickable filename instead of an inline + // embed. Rewrite link-shaped PDF refs at render time so the fix takes + // effect without needing a `--full` re-sync of every entity. Also + // re-encode any unescaped spaces in the URL since pre-fix runs left + // those raw too. + const cleanedBody = repairStaleAttachmentLinks(markdown); + const body = cleanedBody.trim() ? cleanedBody : "_(no body — Evernote v10 stores note bodies in a binary format that this crawler cannot decode yet)_"; + return `${fm}\n${heading}${metaLine}\n${body}\n${attachmentBlock}`; + } + + labelFilesWithTitle() { + return true; + } + + folderConfigs() { + return { + // `notes/` sorts its notebook subfolders DESC by name. Each notebook + // subfolder inherits `sort: DESC` via subdirConfig so files inside + // sort newest-first by their `sortKey` (set by the crawler to the + // note's last-updated timestamp). + notes: { + sort: "DESC" as const, + subdirConfig: { sort: "DESC" as const }, + }, + }; + } + + agentsMarkdown(options: { title: string; description?: string }): string { + const { title, description } = options; + return [ + `# ${title}`, + "", + description ?? "Notes imported from a local Evernote v10 database.", + "", + "## Content", + "", + "- `notes/` — one markdown file per note, organised by notebook.", + "- Attachments live alongside notes and are searchable via OCR text indexed in the FTS `attachment_text` column.", + "", + ].join("\n"); + } + + getFilePath(context: ThemeRenderContext): string { + const data = context.entity.data as Record<string, unknown>; + const notebookName = (data.notebookName as string | undefined) ?? "default"; + const slug = slugify(context.entity.title); + const id = context.entity.externalId.slice(0, 8); + return `notes/${slugify(notebookName)}/${slug}-${id}.md`; + } +} + +/** + * Heal markdown produced by older sync runs that: + * 1. Emitted PDF references as `[file.pdf](url)` (link syntax) instead of + * `![file.pdf](url)` (image syntax). Without the leading `!`, the UI's + * `<img>` override doesn't fire and the PDF shows as a clickable + * filename rather than an inline `<object>` embed. + * 2. Left unescaped spaces in the URL — markdown link parsing stops at + * the first whitespace inside `(...)`, breaking the entire link. + * + * Both of those are emitted correctly by the current crawler, but stored + * markdown from previous sync runs persists in `data.markdown` until the + * note is re-rendered. Repairing at theme-render time makes the fix take + * effect immediately for every existing note. + */ +function repairStaleAttachmentLinks(md: string): string { + if (!md) return md; + // Match plain-link refs into the attachments tree pointing at PDFs (or + // images) where the URL might have unescaped whitespace or markdown- + // special chars. We only touch links that target `attachments/...` so + // user-authored external links are untouched. + return md.replace( + /(!?)\[([^\]]+)\]\(((?:\.\.\/)*attachments\/[^)]+?\.(?:pdf|png|jpe?g|gif|webp|bmp|svg))\)/gi, + (_match, _bang: string, alt: string, url: string) => { + // Re-encode segment-by-segment so spaces and `*`/`(`/`)` (which break + // markdown link parsing) get percent-escaped properly. + const segments = url.split("/").map((seg) => { + try { + return decodeURIComponent(seg); + } catch { + return seg; + } + }); + const cleanUrl = encodePathForMarkdown(segments.join("/")); + // Force image syntax so the UI's `<img>` override picks the file up. + return `![${alt}](${cleanUrl})`; + }, + ); +} diff --git a/packages/crawlers/src/evernote/types.ts b/packages/crawlers/src/evernote/types.ts new file mode 100644 index 0000000..987e043 --- /dev/null +++ b/packages/crawlers/src/evernote/types.ts @@ -0,0 +1,71 @@ +import type { SyncCursor } from "@frozenink/core"; + +export interface EvernoteConfig { + /** + * Optional override for the conduit-storage directory. Default is the + * auto-detected macOS path under + * `~/Library/Containers/com.evernote.Evernote/Data/Library/Application Support/Evernote/conduit-storage/`. + */ + conduitStoragePath?: string; + /** Optional notebook allowlist (names or GUIDs). Empty/undefined = all. */ + notebooks?: string[]; + /** + * Default true. Snapshot DB+WAL+SHM into a temp dir before reading so + * Evernote can stay running during sync. + */ + snapshot?: boolean; +} + +export interface EvernoteCredentials { + /** Local crawler — no secrets. Kept for symmetry with the Crawler interface. */ + conduitStoragePath?: string; +} + +export interface EvernoteSyncCursor extends SyncCursor { + /** Per syncContext (notebook scope), the highest USN already imported. */ + highWaterByContext?: Record<string, number>; + /** + * Snapshot of (nodeId → version) pairs from the previous sync — used to + * detect deletions: any id present before but missing now is a tombstone. + */ + knownNodes?: Record<string, number>; +} + +/** Internal: a parsed Evernote note row (after JSON-decoding `NodeFields`). */ +export interface EvernoteNoteRow { + guid: string; + title: string; + notebookId?: string; + contentHash?: string; + active: boolean; + created?: number; + updated?: number; + updateSequenceNum: number; + /** Raw `evernoteTags` field — list of tag GUIDs that resolve via tagMap. */ + tagGuids?: string[]; + /** XML/HTML 'recognition' blob from per-resource fields, if any. */ + recognitionByResourceHash?: Record<string, string>; + /** ENML body if Evernote stored it directly inside the note's NodeFields. */ + enml?: string; +} + +export interface EvernoteNotebookRow { + guid: string; + name: string; + updateSequenceNum: number; +} + +export interface EvernoteTagRow { + guid: string; + name: string; +} + +export interface EvernoteResourceRow { + guid: string; + noteGuid: string; + hash: string; + mime?: string; + filename?: string; + size?: number; + recognition?: string; +} diff --git a/packages/crawlers/src/index.ts b/packages/crawlers/src/index.ts index 744973b..32b3677 100644 --- a/packages/crawlers/src/index.ts +++ b/packages/crawlers/src/index.ts @@ -9,6 +9,8 @@ import { MantisHubCrawler } from "./mantishub/crawler"; import { MantisHubTheme } from "./mantishub/theme"; import { RssCrawler } from "./rss/crawler"; import { RssTheme } from "./rss/theme"; +import { EvernoteCrawler } from "./evernote/crawler"; +import { EvernoteTheme } from "./evernote/theme"; export { GitHubCrawler } from "./github/crawler"; export { GitHubTheme } from "./github/theme"; @@ -30,6 +32,11 @@ export { RssCrawler } from "./rss/crawler"; export { RssTheme } from "./rss/theme"; export type { RssConfig, RssCredentials } from "./rss/types"; +export { EvernoteCrawler, listEvernoteNotebooks } from "./evernote/crawler"; +export type { EvernoteNotebookSummary } from "./evernote/crawler"; +export { EvernoteTheme } from "./evernote/theme"; +export type { EvernoteConfig, EvernoteCredentials } from "./evernote/types"; + export function createDefaultRegistry(): CrawlerRegistry { const registry = new CrawlerRegistry(); registry.register("github", () => new GitHubCrawler()); @@ -37,6 +44,7 @@ export function createDefaultRegistry(): CrawlerRegistry { registry.register("git", () => new GitCrawler()); registry.register("mantishub", () => new MantisHubCrawler()); registry.register("rss", () => new RssCrawler()); + registry.register("evernote", () => new EvernoteCrawler()); return registry; } @@ -45,3 +53,4 @@ export const obsidianTheme = new ObsidianTheme(); export const gitTheme = new GitTheme(); export const mantisHubTheme = new MantisHubTheme(); export const rssTheme = new RssTheme(); +export const evernoteTheme = new EvernoteTheme(); diff --git a/packages/crawlers/src/themes.ts b/packages/crawlers/src/themes.ts index 3dccc13..32790dd 100644 --- a/packages/crawlers/src/themes.ts +++ b/packages/crawlers/src/themes.ts @@ -4,3 +4,4 @@ export { ObsidianTheme } from "./obsidian/theme"; export { GitTheme } from "./git/theme"; export { MantisHubTheme } from "./mantishub/theme"; export { RssTheme } from "./rss/theme"; +export { EvernoteTheme } from "./evernote/theme"; diff --git a/packages/desktop/src/main/index.ts b/packages/desktop/src/main/index.ts index 9c61a5b..0c2d0f1 100644 --- a/packages/desktop/src/main/index.ts +++ b/packages/desktop/src/main/index.ts @@ -64,6 +64,11 @@ function createMainWindow(): BrowserWindow { preload: existsSync(preloadPath) ? preloadPath : undefined, contextIsolation: true, nodeIntegration: false, + // Enable Chromium's built-in PDF plugin so `<object type="application/pdf">` + // renders inline. Without this flag the embed silently falls back to the + // `<a>` link content and PDF attachments show as a clickable filename + // with no viewer. + plugins: true, }, titleBarStyle: "default", }); diff --git a/packages/ui/src/App.css b/packages/ui/src/App.css index 1a3d031..831ce1c 100644 --- a/packages/ui/src/App.css +++ b/packages/ui/src/App.css @@ -822,6 +822,51 @@ body { font-family: var(--font-body); } +/* + * When the markdown body contains an embedded PDF, drop the 800px reading + * column so the viewer can use the full available width. `:has()` is + * supported in all current browsers Polyscope targets. + */ +.markdown-view:has(.mt-md-pdf) { + max-width: none; +} + +.mt-md-pdf-wrap { + display: flex; + flex-direction: column; + width: 100%; + margin: 16px 0; +} + +.mt-md-pdf { + display: block; + width: 100%; + /* Grow with viewport height; reserve room for the app's top chrome. */ + height: calc(100vh - 200px); + min-height: 600px; + border: 1px solid var(--border, rgba(255, 255, 255, 0.08)); + background: var(--bg, #1e1e1e); +} + +/* + * Lightweight "open in new tab" link sits below the inline viewer so the + * user always has a working escape hatch when the embedded viewer fails + * (some webviews don't bundle a PDF plugin, file integrity errors, etc.). + */ +.mt-md-pdf-fallback { + display: inline-block; + margin-top: 8px; + padding: 4px 8px; + font-size: 12px; + color: var(--text-muted); + text-decoration: none; + border-radius: 4px; +} +.mt-md-pdf-fallback:hover { + color: var(--text); + background: var(--hover-bg, rgba(255, 255, 255, 0.05)); +} + .markdown-view h1 { font-size: 28px; font-weight: 600; @@ -1296,6 +1341,26 @@ body { color: var(--text-muted); } +/* + * Subtle non-blocking indicator shown while a new file or collection is + * loading on top of an existing view. Pinned to the top-right corner so + * it never covers the previous content the user is browsing. + */ +.loading-inline { + position: fixed; + top: 12px; + right: 16px; + padding: 4px 10px; + font-size: 12px; + color: var(--text-muted); + background: var(--bg-secondary, rgba(0, 0, 0, 0.4)); + border: 1px solid var(--border, rgba(255, 255, 255, 0.08)); + border-radius: 4px; + pointer-events: none; + z-index: 50; + opacity: 0.85; +} + /* ===== Search Overlay ===== */ .search-overlay { position: fixed; diff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsx index 7277997..203a0a0 100644 --- a/packages/ui/src/App.tsx +++ b/packages/ui/src/App.tsx @@ -406,7 +406,11 @@ export default function App() { } const collection = selectedCollection; // capture for async callbacks saveLastCollection(collection); - setFileTree([]); + // Don't blank the tree — keep showing the previous one while the new + // collection's tree loads. The FileTree shows an inline loading + // indicator via the `loading` prop. Sync runs that touch the active + // collection re-fire this effect via `refreshKey`; we want browsing + // to stay possible the whole time. setTreeLoading(true); // Close tabs from other collections so a stale tab doesn't linger setTabs((prev) => { @@ -1140,11 +1144,25 @@ export default function App() { </div> <div className="main-body"> <div className="main-inner"> - {(loading || (viewMode === "html" && htmlLoading)) && <div className="loading">Loading...</div>} - {!loading && !fileNotFound && selectedFile && viewMode === "html" && htmlContent !== null && ( + {/* + Stay responsive while a new file (or collection) loads — keep + rendering the previous content and surface a small inline + indicator instead of replacing the pane with a "Loading…" block + that blanks the view. Only show the blocking placeholder when + there's literally nothing else to show. + */} + {(loading || (viewMode === "html" && htmlLoading)) + && !fileContent && !htmlContent && ( + <div className="loading">Loading...</div> + )} + {(loading || (viewMode === "html" && htmlLoading)) + && (fileContent || htmlContent) && ( + <div className="loading-inline" aria-live="polite">Loading…</div> + )} + {!fileNotFound && selectedFile && viewMode === "html" && htmlContent !== null && ( <HtmlView html={htmlContent} onWikilinkClick={handleWikilinkNavigate} /> )} - {!loading && !fileNotFound && selectedFile && viewMode === "markdown" && fileContent !== null && ( + {!fileNotFound && selectedFile && viewMode === "markdown" && fileContent !== null && ( <MarkdownView content={fileContent} collection={selectedCollection || ""} diff --git a/packages/ui/src/components/FileTree.tsx b/packages/ui/src/components/FileTree.tsx index 25d024a..83a60c9 100644 --- a/packages/ui/src/components/FileTree.tsx +++ b/packages/ui/src/components/FileTree.tsx @@ -155,7 +155,11 @@ export default function FileTree({ tree, loading, selectedFile, onSelect }: File } } - if (loading) { + // Only show the blocking "Loading files…" placeholder when there's + // nothing to render yet. If we already have a tree (e.g. a sync just + // finished and we're reloading), keep showing it so the user can keep + // browsing while the new data arrives. + if (loading && tree.length === 0) { return ( <div className="file-tree-empty"> <p>Loading files…</p> diff --git a/packages/ui/src/components/MarkdownView.tsx b/packages/ui/src/components/MarkdownView.tsx index 1610a0f..8ebd7a7 100644 --- a/packages/ui/src/components/MarkdownView.tsx +++ b/packages/ui/src/components/MarkdownView.tsx @@ -44,14 +44,23 @@ function preprocessMarkdown(raw: string, collection: string, filePath?: string): `![${path}](/api/attachments/${encodeURIComponent(collection)}/${path})`, ); - // Rewrite relative attachment paths (../attachments/... or ../../attachments/...) to API URLs - // so they resolve in the web viewer where files aren't on the local filesystem. - // Root-level vault notes produce ../attachments/...; nested notes produce ../../attachments/... + // Rewrite attachment-relative paths to API URLs so they resolve in the + // web viewer where files aren't on the local filesystem. Accepts both + // root-relative (`attachments/...`, used by the Evernote crawler) and + // ancestor-relative (`../attachments/...`, `../../attachments/...`) forms. + // Both image refs (`![]`) and plain links (`[]`) are rewritten so PDF + // download links keep working too. + const attachmentPathRe = /(?:\.\.\/)*attachments\/([^)]+)/; content = content.replace( - /!\[([^\]]*)\]\((?:\.\.\/)+attachments\/([^)]+)\)/g, - (_match, alt: string, path: string) => + new RegExp(`!\\[([^\\]]*)\\]\\((${attachmentPathRe.source})\\)`, "g"), + (_match, alt: string, _full: string, path: string) => `![${alt}](/api/attachments/${encodeURIComponent(collection)}/${path})`, ); + content = content.replace( + new RegExp(`(?<!!)\\[([^\\]]*)\\]\\((${attachmentPathRe.source})\\)`, "g"), + (_match, label: string, _full: string, path: string) => + `[${label}](/api/attachments/${encodeURIComponent(collection)}/${path})`, + ); // Rewrite MantisBT-style relative asset paths: ![alt](assets/filename) // The file is stored at content/{dir}/assets/{filename} relative to the collection root. @@ -203,6 +212,40 @@ export default function MarkdownView({ }, img({ src, alt }) { const url = typeof src === "string" ? src : ""; + // Inline PDFs: an `<img>` ref to a `.pdf` is treated as a request to + // embed the document inline. URL fragment params strip the + // thumbnail sidebar/bookmark pane and fit-to-width — most browser + // PDF viewers (Chrome/Edge built-in, pdf.js, Safari) honour these. + // + // Use `<iframe>` rather than `<object>` because `<object>` doesn't + // reliably re-load when React reuses the same DOM node and only + // swaps the `data` attribute (browsing between two PDF notes was + // intermittently showing the previous note's PDF — or none at all). + // `<iframe src=…>` triggers a full navigation; pairing it with a + // `key` of the URL forces React to unmount + remount on every + // change, killing any stale viewer state. + if (/\.pdf(\?|#|$)/i.test(url)) { + const sep = url.includes("#") ? "&" : "#"; + const viewerUrl = `${url}${sep}navpanes=0&pagemode=none&toolbar=1&zoom=page-width`; + return ( + <div className="mt-md-pdf-wrap"> + <iframe + key={viewerUrl} + src={viewerUrl} + title={alt || "PDF"} + className="mt-md-pdf" + /> + <a + className="mt-md-pdf-fallback" + href={url} + target="_blank" + rel="noopener noreferrer" + > + Open {alt || "PDF"} in a new tab + </a> + </div> + ); + } const video = url ? videoEmbedFromUrl(url) : null; if (video) { if (video.provider === "komodo") { diff --git a/packages/ui/src/components/manage/CollectionForm.tsx b/packages/ui/src/components/manage/CollectionForm.tsx index d0a3eff..6e5041d 100644 --- a/packages/ui/src/components/manage/CollectionForm.tsx +++ b/packages/ui/src/components/manage/CollectionForm.tsx @@ -27,9 +27,10 @@ const CRAWLER_TYPES = [ { id: "git", label: "Git", description: "Commits, branches, and tags from a local repository" }, { id: "mantishub", label: "MantisHub", description: "Issues from a MantisHub instance" }, { id: "rss", label: "RSS/Atom", description: "Posts from RSS or Atom feeds with optional sitemap backfill" }, + { id: "evernote", label: "Evernote", description: "Notes from a local Evernote v10 install (macOS)" }, ]; -const NO_CREDENTIALS_CRAWLERS = ["obsidian", "git", "remote", "rss"]; +const NO_CREDENTIALS_CRAWLERS = ["obsidian", "git", "remote", "rss", "evernote"]; function configToStrings(obj: Record<string, unknown>): Record<string, string> { const result: Record<string, string> = {}; @@ -483,6 +484,8 @@ function renderConfigFields( ); case "remote": return field("sourceUrl", "Source URL", "https://example.workers.dev"); + case "evernote": + return <EvernoteConfigFields config={config} setConfig={setConfig} />; default: return <p className="text-muted">No configuration needed.</p>; } @@ -514,6 +517,7 @@ function renderCredentialFields( return field("token", "API Token", "your-api-token"); case "obsidian": case "git": + case "evernote": return <p className="text-muted">No credentials needed for local sources.</p>; case "remote": return <p className="text-muted">Credentials are inherited from the source.</p>; @@ -521,3 +525,169 @@ function renderCredentialFields( return null; } } + +interface EvernoteNotebookSummary { + guid: string; + name: string; + noteCount: number; + totalBytes: number; +} + +function formatBytes(bytes: number): string { + if (!bytes) return "0 B"; + const units = ["B", "KB", "MB", "GB"]; + let i = 0; + let n = bytes; + while (n >= 1024 && i < units.length - 1) { n /= 1024; i++; } + return `${n < 10 && i > 0 ? n.toFixed(1) : Math.round(n)} ${units[i]}`; +} + +function EvernoteConfigFields({ + config, + setConfig, +}: { + config: Record<string, string>; + setConfig: (c: Record<string, string>) => void; +}) { + const set = (key: string, val: string) => setConfig({ ...config, [key]: val }); + + // Selection mode: "all" syncs every notebook; "subset" enables the multi-select. + // We persist the comma-separated selection in the same `notebooks` config key + // the CLI uses, so loaded collections round-trip cleanly. + // The serialized form supports both a JSON array (preferred — round-trips + // cleanly through configToStrings + JSON.parse on submit) and the legacy + // comma-separated form the CLI accepts. + const initialList = (() => { + const raw = config.notebooks ?? ""; + if (!raw) return [] as string[]; + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) return parsed.filter((x): x is string => typeof x === "string"); + } catch { + // fall through to comma split + } + return raw.split(",").map((s) => s.trim()).filter(Boolean); + })(); + const [mode, setMode] = useState<"all" | "subset">(initialList.length > 0 ? "subset" : "all"); + const [selected, setSelected] = useState<Set<string>>(new Set(initialList)); + const [notebooks, setNotebooks] = useState<EvernoteNotebookSummary[] | null>(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState<string | null>(null); + + const fetchNotebooks = async () => { + setLoading(true); + setError(null); + try { + const params = new URLSearchParams(); + if (config.conduitStoragePath) params.set("path", config.conduitStoragePath); + const res = await fetch(`/api/crawlers/evernote/notebooks?${params.toString()}`); + const body = await res.json(); + if (body.error) { + setError(body.error); + setNotebooks([]); + } else { + setNotebooks(body.notebooks as EvernoteNotebookSummary[]); + } + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setLoading(false); + } + }; + + // Auto-load on first mount when no path is set (covers the auto-detect case). + useEffect(() => { + if (!config.conduitStoragePath) fetchNotebooks(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + const toggle = (name: string) => { + setSelected((prev) => { + const next = new Set(prev); + if (next.has(name)) next.delete(name); + else next.add(name); + // Store as a JSON array so handleSubmit's JSON.parse turns it into + // a real string[] for the crawler config. + set("notebooks", next.size > 0 ? JSON.stringify([...next]) : ""); + return next; + }); + }; + + return ( + <> + <PathField + label="Conduit Storage Path" + value={config.conduitStoragePath ?? ""} + onChange={(v) => set("conduitStoragePath", v)} + placeholder="leave blank to auto-detect ~/Library/Containers/com.evernote.Evernote/..." + /> + <div> + <label>Notebooks</label> + <div> + <label> + <input + type="radio" + checked={mode === "all"} + onChange={() => { + setMode("all"); + set("notebooks", ""); + setSelected(new Set()); + }} + /> + {" "}All notebooks + </label> + <label style={{ marginLeft: "1em" }}> + <input + type="radio" + checked={mode === "subset"} + onChange={() => setMode("subset")} + /> + {" "}Pick notebooks + </label> + <button + type="button" + className="btn btn-sm" + onClick={fetchNotebooks} + style={{ marginLeft: "1em" }} + > + {loading ? "Loading..." : "Refresh list"} + </button> + </div> + {error && <p className="text-muted">{error}</p>} + {mode === "subset" && notebooks && notebooks.length > 0 && ( + <div className="evernote-notebook-list" style={{ maxHeight: 220, overflowY: "auto", marginTop: 8 }}> + {notebooks.map((nb) => ( + <div key={nb.guid}> + <label> + <input + type="checkbox" + checked={selected.has(nb.name)} + onChange={() => toggle(nb.name)} + /> + {" "}{nb.name} + <span className="form-label-hint"> + {" "}— {nb.noteCount} note{nb.noteCount === 1 ? "" : "s"} + {nb.totalBytes > 0 ? `, ${formatBytes(nb.totalBytes)}` : ""} + </span> + </label> + </div> + ))} + </div> + )} + {mode === "subset" && notebooks && notebooks.length === 0 && !loading && ( + <p className="text-muted">No notebooks found at that path.</p> + )} + </div> + <div> + <label> + <input + type="checkbox" + checked={config.snapshot !== "false"} + onChange={(e) => set("snapshot", String(e.target.checked))} + /> + {" "}Snapshot DB before reading (only used when Evernote has uncheckpointed writes) + </label> + </div> + </> + ); +} diff --git a/packages/worker/src/db/search.ts b/packages/worker/src/db/search.ts index 01c2024..ef78359 100644 --- a/packages/worker/src/db/search.ts +++ b/packages/worker/src/db/search.ts @@ -19,16 +19,20 @@ export async function searchEntities( const ftsQuery = buildFtsQuery(query); if (!ftsQuery) return []; - // FTS5 query — weight title column higher than content/tags so title - // matches rank ahead of body mentions. bm25() weights are positional for - // every declared column including UNINDEXED ones. Published worker schema: - // entity_id, external_id, entity_type, title, content, tags. - // Lower bm25 = better match. Snippet uses column 4 (content) so the UI - // shows where in the body the query matched, not a re-render of the title. + // FTS5 query — keep weights in lockstep with the local SearchIndexer + // (`packages/core/src/search/indexer.ts`). bm25() weights are positional + // for every declared column including UNINDEXED ones. Published worker + // schema (after the attachment_text migration in publish.ts): + // entity_id, external_id, entity_type, title, content, tags, attachment_text. + // Weights: title 10, tags 5, body 1, attachment 0.25 (and 1.0 for the + // three UNINDEXED housekeeping columns, which never match). attachment + // text is weighted low so a body or tag hit decisively outranks an + // OCR-only match. Lower bm25 score = better match. Snippet still uses + // column 4 (content) so the UI shows the body context for the match. let sql = ` SELECT entity_id, external_id, entity_type, title, - bm25(entities_fts, 1.0, 1.0, 1.0, 10.0, 1.0, 2.0) AS rank, + bm25(entities_fts, 1.0, 1.0, 1.0, 10.0, 1.0, 5.0, 0.25) AS rank, snippet(entities_fts, 4, '<mark>', '</mark>', '…', 48) AS snippet FROM entities_fts WHERE entities_fts MATCH ? diff --git a/packages/worker/src/handlers/api.ts b/packages/worker/src/handlers/api.ts index 6717cec..7c01535 100644 --- a/packages/worker/src/handlers/api.ts +++ b/packages/worker/src/handlers/api.ts @@ -112,14 +112,15 @@ api.get("/api/collections/:name/html/*", async (c) => { api.get("/api/collections/:name/tree", async (c) => { const name = c.req.param("name"); const col = await getCollectionConfig(c.env.BUCKET, name); - // D1 .all() returns max 5000 rows — paginate to fetch all entities - const allResults: Array<{ folder: string; slug: string; title: string }> = []; + // D1 .all() returns max 5000 rows — paginate to fetch all entities. Pull + // `data` too so we can extract per-entity sortKey for the tree sort. + const allResults: Array<{ folder: string; slug: string; title: string; data: string }> = []; const PAGE_SIZE = 5000; let offset = 0; for (;;) { const { results } = await c.env.DB.prepare( - "SELECT folder, slug, title FROM entities WHERE folder IS NOT NULL AND slug IS NOT NULL LIMIT ? OFFSET ?", - ).bind(PAGE_SIZE, offset).all<{ folder: string; slug: string; title: string }>(); + "SELECT folder, slug, title, data FROM entities WHERE folder IS NOT NULL AND slug IS NOT NULL LIMIT ? OFFSET ?", + ).bind(PAGE_SIZE, offset).all<{ folder: string; slug: string; title: string; data: string }>(); if (!results || results.length === 0) break; allResults.push(...results); if (results.length < PAGE_SIZE) break; @@ -128,10 +129,16 @@ api.get("/api/collections/:name/tree", async (c) => { const labelWithTitle = col?.crawler ? themeEngine.labelFilesWithTitle(col.crawler) : true; const titleByPath = new Map<string, string>(); + const sortKeyByPath = new Map<string, string>(); const mdPaths = allResults .map((row) => { const rel = row.folder ? `${row.folder}/${row.slug}.md` : `${row.slug}.md`; if (labelWithTitle && row.title) titleByPath.set(rel, row.title); + try { + const data = row.data ? JSON.parse(row.data) : null; + const sortKey = data?.sortKey; + if (typeof sortKey === "string" && sortKey) sortKeyByPath.set(rel, sortKey); + } catch { /* malformed data — skip sortKey */ } return rel; }) .filter((p) => p.endsWith(".md")); @@ -196,7 +203,7 @@ api.get("/api/collections/:name/tree", async (c) => { return true; }); - const tree = buildTreeFromPaths(filteredPaths, titleByPath, folderConfigs); + const tree = buildTreeFromPaths(filteredPaths, titleByPath, folderConfigs, sortKeyByPath); return c.json(tree); }); @@ -618,6 +625,7 @@ function buildTreeFromPaths( paths: string[], titleByPath: Map<string, string> = new Map(), folderConfigs: Map<string, { visible?: boolean; sort?: "ASC" | "DESC"; hide?: string[]; showCount?: boolean; expandFirstN?: number; expanded?: boolean; created_at_prefix?: boolean }> = new Map(), + sortKeyByPath: Map<string, string> = new Map(), ): object[] { interface TreeNode { name: string; @@ -704,11 +712,12 @@ function buildTreeFromPaths( const files = nodes .filter((n) => n.type === "file") .filter((n) => folderHideCompiled.length === 0 || !matchesHidePattern(folderHideCompiled, n.name)); - if (sortOrder === "DESC") { - files.sort((a, b) => b.name.localeCompare(a.name)); - } else { - files.sort((a, b) => a.name.localeCompare(b.name)); - } + const fileEffectiveKey = (f: TreeNode): string => sortKeyByPath.get(f.path) ?? f.name; + files.sort((a, b) => { + const ka = fileEffectiveKey(a); + const kb = fileEffectiveKey(b); + return sortOrder === "DESC" ? kb.localeCompare(ka) : ka.localeCompare(kb); + }); if (config?.created_at_prefix) { for (const file of files) { const datePrefix = file.name.slice(0, 8);