diff --git a/SCHEMA.md b/SCHEMA.md index 8bdbcf7..b3d03f4 100644 --- a/SCHEMA.md +++ b/SCHEMA.md @@ -48,6 +48,13 @@ Two concurrent changes that minimise the gratuitous local↔worker divergence: - **Drop `collection_name` from `entities_fts`.** Each local collection has its own SQLite file, so collection scoping is implicit — the column was written but never queried. FTS5 has no DROP COLUMN, so detect via `PRAGMA table_info` then drop+recreate. Final 7-column shape: `entity_id UNINDEXED, external_id UNINDEXED, entity_type UNINDEXED, title, content, tags, attachment_text` — same as the worker. SyncEngine re-INSERTs FTS rows on the next sync. +### v5 — Repair inconsistent FTS migration state + +Some databases were stamped at v4 while retaining the pre-v3 FTS shape. The +repair migration verifies the actual columns rather than trusting only +`schema.version`, then rebuilds the canonical 7-column table and preserves +existing indexed rows where possible. + - **Drop `AUTOINCREMENT` from `entities.id`.** The historical local table used `INTEGER PRIMARY KEY AUTOINCREMENT` while the worker uses plain `INTEGER PRIMARY KEY`; we never depended on the no-id-reuse guarantee that `AUTOINCREMENT` provides (FTS rows are deleted alongside their entity row before any new INSERT, so collisions can't occur). The migration detects the historical shape via `sqlite_master.sql`, builds a clone table with the new shape, copies all rows preserving their existing ids, drops the original, renames the clone, and rebuilds the indexes. Wrapped in `BEGIN/COMMIT` — partial failure rolls back. Final FTS bm25 weights used by `SearchIndexer.search()`: `title=10, tags=5, body=1, attachment=0.25` (UNINDEXED columns get weight `1.0` but never match). `snippet()` highlights column index 4 (content). Worker `bm25()` and `snippet()` are kept identical per the parity rule. diff --git a/package.json b/package.json index 9700814..d193b0f 100644 --- a/package.json +++ b/package.json @@ -25,5 +25,5 @@ "@types/js-yaml": "^4.0.9", "js-yaml": "^4.1.1" }, - "version": "0.10.9" + "version": "0.10.15" } diff --git a/packages/cli/package.json b/packages/cli/package.json index 6d0f965..3a47bc3 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@frozenink/cli", - "version": "0.10.9", + "version": "0.10.15", "private": true, "type": "module", "main": "src/index.ts", diff --git a/packages/cli/src/commands/__tests__/wrangler-api.test.ts b/packages/cli/src/commands/__tests__/wrangler-api.test.ts new file mode 100644 index 0000000..4e00d3f --- /dev/null +++ b/packages/cli/src/commands/__tests__/wrangler-api.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "bun:test"; +import { encodeR2ObjectKey, shouldRetryR2UploadWithWrangler } from "../wrangler-api"; + +describe("encodeR2ObjectKey", () => { + test("preserves path separators and encodes reserved characters in each segment", () => { + expect(encodeR2ObjectKey("mantisbt/content/issues/assets/2709-mantis bugnote..JPG")) + .toBe("mantisbt/content/issues/assets/2709-mantis%20bugnote..JPG"); + }); +}); + +describe("shouldRetryR2UploadWithWrangler", () => { + test("recognizes an edge-generated HTML 403", () => { + expect(shouldRetryR2UploadWithWrangler(403, "Forbidden")) + .toBe(true); + }); + + test("leaves normal API errors unchanged", () => { + expect(shouldRetryR2UploadWithWrangler(403, '{"errors":[{"message":"forbidden"}]}')) + .toBe(false); + expect(shouldRetryR2UploadWithWrangler(500, "Server error")) + .toBe(false); + }); +}); diff --git a/packages/cli/src/commands/publish.ts b/packages/cli/src/commands/publish.ts index 31d9bc3..2055005 100644 --- a/packages/cli/src/commands/publish.ts +++ b/packages/cli/src/commands/publish.ts @@ -20,6 +20,7 @@ import { saveNamedCredentials, removeNamedCredentials, MetadataStore, + toR2SafeKey, } from "@frozenink/core"; import { getPublishCredentialKey } from "./publish-credentials"; import type { EntityData } from "@frozenink/core"; @@ -466,7 +467,7 @@ export async function publishCollections( const uiUploads: Array<{ r2Key: string; fullPath: string }> = []; if (existsSync(uiDistDir)) { for (const file of collectFiles(uiDistDir)) { - uiUploads.push({ r2Key: `_ui/${file.relativePath}`, fullPath: file.fullPath }); + uiUploads.push({ r2Key: toR2SafeKey(`_ui/${file.relativePath}`), fullPath: file.fullPath }); } } if (uiUploads.length > 0) { @@ -496,7 +497,7 @@ export async function publishCollections( const collectionDir = join(home, "collections", colName); for (const file of collectFiles(join(collectionDir, "content"))) { if (file.relativePath.endsWith(".md") || file.relativePath.endsWith(".yml")) continue; - uploads.push({ r2Key: `${colName}/content/${file.relativePath}`, fullPath: file.fullPath }); + uploads.push({ r2Key: toR2SafeKey(`${colName}/content/${file.relativePath}`), fullPath: file.fullPath }); } // Upload the sibling attachments/ directory — the worker serves these at // /api/attachments/{name}/* and the UI rewrites markdown image paths to @@ -504,13 +505,13 @@ export async function publishCollections( const attachmentsDir = join(collectionDir, "attachments"); if (existsSync(attachmentsDir)) { for (const file of collectFiles(attachmentsDir)) { - uploads.push({ r2Key: `${colName}/attachments/${file.relativePath}`, fullPath: file.fullPath }); + uploads.push({ r2Key: toR2SafeKey(`${colName}/attachments/${file.relativePath}`), fullPath: file.fullPath }); } } } if (existsSync(uiDistDir)) { for (const file of collectFiles(uiDistDir)) { - uploads.push({ r2Key: `_ui/${file.relativePath}`, fullPath: file.fullPath }); + uploads.push({ r2Key: toR2SafeKey(`_ui/${file.relativePath}`), fullPath: file.fullPath }); } } diff --git a/packages/cli/src/commands/wrangler-api.ts b/packages/cli/src/commands/wrangler-api.ts index 9e7b76e..7fcbfc9 100644 --- a/packages/cli/src/commands/wrangler-api.ts +++ b/packages/cli/src/commands/wrangler-api.ts @@ -1,4 +1,4 @@ -import { join } from "path"; +import { delimiter, dirname, join } from "path"; import { existsSync, readFileSync, writeFileSync, unlinkSync } from "fs"; import { homedir } from "os"; import { randomBytes } from "crypto"; @@ -55,9 +55,18 @@ async function runWrangler( opts?: { cwd?: string; allowFailure?: boolean }, ): Promise<{ stdout: string; stderr: string; exitCode: number }> { const bin = await resolveWranglerBin(); + // A Dock/Spotlight-launched macOS app has a minimal PATH. Wrangler's npm + // executable uses `#!/usr/bin/env node`, so include its installation + // directory to make the matching Node runtime available to that shebang. + const path = [dirname(bin), process.env.PATH] + .filter(Boolean) + .join(delimiter); const { stdout, stderr, exitCode } = await spawnProcess([bin, ...args], { - cwd: opts?.cwd, - env: { ...process.env }, + // GUI apps launched from Spotlight often inherit `/` as their CWD. Wrangler + // creates `.wrangler/cache` relative to CWD, so keep it inside the active + // Frozen Ink workspace unless a caller explicitly chooses another folder. + cwd: opts?.cwd ?? getFrozenInkHome(), + env: { ...process.env, PATH: path }, }); if (exitCode !== 0 && !opts?.allowFailure) { @@ -314,6 +323,26 @@ async function fetchCloudflareApi( // --- R2 operations (bucket via wrangler CLI, objects via HTTP for speed) --- +/** Encode each key segment while preserving `/` as the R2 path delimiter. */ +export function encodeR2ObjectKey(key: string): string { + return key.split("/").map(encodeURIComponent).join("/"); +} + +export function shouldRetryR2UploadWithWrangler(status: number, responseBody: string): boolean { + return status === 403 && /]/i.test(responseBody); +} + +async function putR2ObjectWithWrangler( + bucket: string, + key: string, + filePath: string, + contentType?: string, +): Promise { + const args = ["r2", "object", "put", `${bucket}/${key}`, "--file", filePath, "--remote"]; + if (contentType) args.push("--content-type", contentType); + await runWrangler(args); +} + export async function createR2Bucket(name: string): Promise { try { await runWrangler(["r2", "bucket", "create", name]); @@ -333,7 +362,7 @@ export async function putR2Object( ): Promise { const body = readFileSync(filePath); const res = await fetchCloudflareApi(({ apiToken, accountId }) => ({ - url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeURIComponent(key)}`, + url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeR2ObjectKey(key)}`, init: { method: "PUT", headers: { @@ -345,6 +374,13 @@ export async function putR2Object( })); if (!res.ok) { const text = await res.text(); + // Cloudflare's edge can reject an otherwise valid direct REST upload with + // an HTML 403 for particular object names or payloads. Wrangler uses the + // supported R2 command path and handles those cases correctly. + if (shouldRetryR2UploadWithWrangler(res.status, text)) { + await putR2ObjectWithWrangler(bucket, key, filePath, contentType); + return; + } const hint = res.status === 401 ? " — the Cloudflare OAuth token looks expired. Run `wrangler login` again, or restart the desktop app to refresh the cached credentials." : ""; @@ -359,7 +395,7 @@ export async function putR2String( contentType: string = "text/plain", ): Promise { const res = await fetchCloudflareApi(({ apiToken, accountId }) => ({ - url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeURIComponent(key)}`, + url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeR2ObjectKey(key)}`, init: { method: "PUT", headers: { Authorization: `Bearer ${apiToken}`, "Content-Type": contentType }, @@ -374,7 +410,7 @@ export async function putR2String( export async function getR2String(bucket: string, key: string): Promise { const res = await fetchCloudflareApi(({ apiToken, accountId }) => ({ - url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeURIComponent(key)}`, + url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeR2ObjectKey(key)}`, init: { method: "GET", headers: { Authorization: `Bearer ${apiToken}` }, @@ -387,7 +423,7 @@ export async function getR2String(bucket: string, key: string): Promise { const res = await fetchCloudflareApi(({ apiToken, accountId }) => ({ - url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeURIComponent(key)}`, + url: `https://api.cloudflare.com/client/v4/accounts/${accountId}/r2/buckets/${bucket}/objects/${encodeR2ObjectKey(key)}`, init: { method: "DELETE", headers: { Authorization: `Bearer ${apiToken}` }, diff --git a/packages/cli/src/tui/sync-jobs.ts b/packages/cli/src/tui/sync-jobs.ts index 02f65e5..b1c91e3 100644 --- a/packages/cli/src/tui/sync-jobs.ts +++ b/packages/cli/src/tui/sync-jobs.ts @@ -14,6 +14,7 @@ import { SyncEngine, ThemeEngine, LocalStorageBackend, + resolveCredentials, } from "@frozenink/core"; import { createDefaultRegistry, @@ -194,7 +195,7 @@ async function runJob(job: TuiSyncJob): Promise { try { await crawler.initialize( col.config as Record, - col.credentials as Record, + resolveCredentials(col.credentials), ); const dbPath = getCollectionDbPath(name); diff --git a/packages/core/package.json b/packages/core/package.json index a4e9af4..bb2d4cd 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -9,6 +9,7 @@ ".": "./src/index.ts", "./theme": "./src/theme/index.ts", "./export": "./src/export/index.ts", + "./storage/r2-key": "./src/storage/r2-key.ts", "./search/fts-query": "./src/search/fts-query.ts" }, "scripts": { diff --git a/packages/core/src/compat/__tests__/resources.test.ts b/packages/core/src/compat/__tests__/resources.test.ts new file mode 100644 index 0000000..2f6a9b6 --- /dev/null +++ b/packages/core/src/compat/__tests__/resources.test.ts @@ -0,0 +1,36 @@ +import { afterEach, describe, expect, it } from "bun:test"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { resolveWrangler } from "../resources"; + +const originalHome = process.env.HOME; +const originalNvmDir = process.env.NVM_DIR; +let testDir: string; + +afterEach(() => { + if (originalHome === undefined) delete process.env.HOME; + else process.env.HOME = originalHome; + if (originalNvmDir === undefined) delete process.env.NVM_DIR; + else process.env.NVM_DIR = originalNvmDir; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +describe("resolveWrangler", () => { + it("finds a Wrangler install under NVM when the app PATH omits it", async () => { + testDir = mkdtempSync(join(tmpdir(), "frozenink-wrangler-")); + process.env.HOME = testDir; + delete process.env.NVM_DIR; + + const binDir = join(testDir, ".nvm", "versions", "node", "v20.19.5", "bin"); + mkdirSync(binDir, { recursive: true }); + const nodePath = join(binDir, "node"); + const wranglerPath = join(binDir, "wrangler"); + writeFileSync(nodePath, "#!/bin/sh\nexit 0\n"); + writeFileSync(wranglerPath, "#!/bin/sh\nexit 0\n"); + chmodSync(nodePath, 0o755); + chmodSync(wranglerPath, 0o755); + + await expect(resolveWrangler(join(testDir, "packaged-app"))).resolves.toBe(wranglerPath); + }); +}); diff --git a/packages/core/src/compat/resources.ts b/packages/core/src/compat/resources.ts index 265364a..02e9dbc 100644 --- a/packages/core/src/compat/resources.ts +++ b/packages/core/src/compat/resources.ts @@ -1,4 +1,4 @@ -import { existsSync, readFileSync } from "fs"; +import { existsSync, readFileSync, readdirSync } from "fs"; import { join } from "path"; import { getModuleDir } from "./paths"; import { spawnProcess } from "./subprocess"; @@ -75,6 +75,19 @@ export function resolveUiDist(moduleDir: string): string | null { const WRANGLER_INSTALL_URL = "https://developers.cloudflare.com/workers/wrangler/install-and-update/"; +function getNvmNodeBinDirs(home: string): string[] { + const nvmDir = process.env.NVM_DIR || join(home, ".nvm"); + const versionsDir = join(nvmDir, "versions", "node"); + try { + return readdirSync(versionsDir) + .sort((a, b) => b.localeCompare(a, undefined, { numeric: true })) + .map((version) => join(versionsDir, version, "bin")) + .filter((binDir) => existsSync(join(binDir, "node"))); + } catch { + return []; + } +} + /** * Resolve the wrangler binary path. * @@ -107,18 +120,23 @@ export async function resolveWrangler(moduleDir: string, configuredPath?: string // Dock-launched processes inherit a minimal PATH. const home = process.env.HOME ?? ""; const candidates: string[] = []; + const nvmBinDirs = getNvmNodeBinDirs(home); if (process.platform === "darwin") { candidates.push( "/opt/homebrew/bin/wrangler", // Homebrew on Apple Silicon "/usr/local/bin/wrangler", // Homebrew on Intel / classic npm global join(home, ".volta/bin/wrangler"), // Volta join(home, ".bun/bin/wrangler"), // Bun global + join(home, ".npm-global/bin/wrangler"), // Custom npm global prefix + ...nvmBinDirs.map((binDir) => join(binDir, "wrangler")), ); } else if (process.platform === "linux") { candidates.push( "/usr/local/bin/wrangler", join(home, ".volta/bin/wrangler"), join(home, ".bun/bin/wrangler"), + join(home, ".npm-global/bin/wrangler"), + ...nvmBinDirs.map((binDir) => join(binDir, "wrangler")), ); } for (const p of candidates) { diff --git a/packages/core/src/db/__tests__/migrations.test.ts b/packages/core/src/db/__tests__/migrations.test.ts index 355483b..e499825 100644 --- a/packages/core/src/db/__tests__/migrations.test.ts +++ b/packages/core/src/db/__tests__/migrations.test.ts @@ -122,4 +122,42 @@ describe("runSyncMigrations", () => { expect(idxNames).toContain("idx_entities_external"); expect(idxNames).toContain("idx_entities_folder"); }); + + it("v5 repairs a falsely-versioned legacy FTS table and preserves rows", () => { + const db = open(); + runSyncMigrations(db, LOCAL_MIGRATIONS); + db.exec("DROP TABLE entities_fts"); + db.exec(` + CREATE VIRTUAL TABLE entities_fts USING fts5( + collection_name UNINDEXED, + entity_id UNINDEXED, + external_id UNINDEXED, + entity_type UNINDEXED, + title, + content, + tags + ); + `); + db.exec("INSERT INTO entities_fts (collection_name, entity_id, external_id, entity_type, title, content, tags) VALUES ('legacy', 1, 'issue:1', 'issue', 'Title', 'Body', 'tag')"); + db.prepare("UPDATE metadata SET value = '4' WHERE key = ?").run(SCHEMA_VERSION_KEY); + + runSyncMigrations(db, LOCAL_MIGRATIONS); + + const cols = db + .prepare("PRAGMA table_info(entities_fts)") + .all() as Array<{ name: string }>; + expect(cols.map((c) => c.name)).toEqual([ + "entity_id", + "external_id", + "entity_type", + "title", + "content", + "tags", + "attachment_text", + ]); + const row = db + .prepare("SELECT external_id, title, content FROM entities_fts") + .get() as { external_id: string; title: string; content: string }; + expect(row).toEqual({ external_id: "issue:1", title: "Title", content: "Body" }); + }); }); diff --git a/packages/core/src/db/migrations/local.ts b/packages/core/src/db/migrations/local.ts index c4f816f..d770b5d 100644 --- a/packages/core/src/db/migrations/local.ts +++ b/packages/core/src/db/migrations/local.ts @@ -243,4 +243,75 @@ export const LOCAL_MIGRATIONS: SyncMigration[] = [ } }, }, + { + id: 5, + description: + "Repair a falsely-versioned or partially-migrated entities_fts table by restoring the canonical 7-column shape while preserving indexed rows.", + up: (db) => { + const expectedColumns = [ + "entity_id", + "external_id", + "entity_type", + "title", + "content", + "tags", + "attachment_text", + ]; + + let columns: string[] = []; + try { + columns = (db.prepare("PRAGMA table_info(entities_fts)").all() as Array<{ name: string }>) + .map((c) => c.name); + } catch { + // The CREATE below handles a missing table. + } + + const canonical = columns.length === expectedColumns.length + && expectedColumns.every((name) => columns.includes(name)) + && !columns.includes("collection_name"); + + if (canonical) return; + + const backup = "entities_fts_repair_backup"; + db.exec(`DROP TABLE IF EXISTS ${backup}`); + + if (columns.length > 0) { + // Preserve existing searchable content. Older tables may have + // collection_name and/or lack attachment_text; normalize both cases + // while copying into the new FTS table. + const attachmentText = columns.includes("attachment_text") + ? "attachment_text" + : "NULL AS attachment_text"; + db.exec(` + CREATE TABLE ${backup} AS + SELECT entity_id, external_id, entity_type, title, content, tags, + ${attachmentText} + FROM entities_fts; + `); + db.exec("DROP TABLE IF EXISTS entities_fts"); + } + + db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5( + entity_id UNINDEXED, + external_id UNINDEXED, + entity_type UNINDEXED, + title, + content, + tags, + attachment_text + ); + `); + + if (columns.length > 0) { + db.exec(` + INSERT INTO entities_fts + (entity_id, external_id, entity_type, title, content, tags, attachment_text) + SELECT entity_id, external_id, entity_type, title, content, tags, attachment_text + FROM ${backup}; + `); + db.exec(`DROP TABLE IF EXISTS ${backup}`); + } + }, + }, ]; diff --git a/packages/core/src/storage/__tests__/r2-key.test.ts b/packages/core/src/storage/__tests__/r2-key.test.ts new file mode 100644 index 0000000..2069721 --- /dev/null +++ b/packages/core/src/storage/__tests__/r2-key.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from "bun:test"; +import { toR2SafeKey } from "../r2-key"; + +describe("toR2SafeKey", () => { + test("rewrites traversal-like dot pairs inside a filename", () => { + expect(toR2SafeKey("mantisbt/content/issues/assets/2709-mantis bugnote..JPG")) + .toBe("mantisbt/content/issues/assets/2709-mantis bugnote~dd~JPG"); + }); + + test("escapes existing tildes to avoid collisions", () => { + expect(toR2SafeKey("files/name~dd~JPG")).toBe("files/name~~dd~~JPG"); + }); + + test("leaves ordinary keys unchanged", () => { + expect(toR2SafeKey("mantisbt/content/issues/assets/image.jpg")) + .toBe("mantisbt/content/issues/assets/image.jpg"); + }); +}); diff --git a/packages/core/src/storage/index.ts b/packages/core/src/storage/index.ts index 17b06c2..8654c76 100644 --- a/packages/core/src/storage/index.ts +++ b/packages/core/src/storage/index.ts @@ -1,2 +1,3 @@ export type { StorageBackend } from "./interface"; export { LocalStorageBackend } from "./local"; +export { toR2SafeKey } from "./r2-key"; diff --git a/packages/core/src/storage/r2-key.ts b/packages/core/src/storage/r2-key.ts new file mode 100644 index 0000000..979c83c --- /dev/null +++ b/packages/core/src/storage/r2-key.ts @@ -0,0 +1,12 @@ +/** + * Map local paths to R2 keys that do not contain traversal-like `..` + * sequences. Cloudflare's edge WAF blocks those sequences before requests + * reach the authenticated R2 API. Tildes are escaped first so the mapping is + * deterministic and collision-free for transformed segments. + */ +export function toR2SafeKey(key: string): string { + return key + .split("/") + .map((segment) => segment.replace(/~/g, "~~").replace(/\.\./g, "~dd~")) + .join("/"); +} diff --git a/packages/crawlers/src/mantishub/crawler.ts b/packages/crawlers/src/mantishub/crawler.ts index 7366ec0..868dd6f 100644 --- a/packages/crawlers/src/mantishub/crawler.ts +++ b/packages/crawlers/src/mantishub/crawler.ts @@ -391,18 +391,12 @@ export class MantisHubCrawler implements Crawler { if (this.projectId) { url += `&project_id=${this.projectId}`; } - // Pin the dataset to the snapshot ceiling so concurrent updates after - // sync started don't shift later pages. MantisBT supports `filter_updated_before`. - if (c.snapshotCeiling) { - url += `&filter_updated_before=${encodeURIComponent(c.snapshotCeiling)}`; - } - const response = await this.apiFetch(url); const data = (await response.json()) as { issues: Array<{ id: number; updated_at: string }> }; let issues = data.issues ?? []; - // Belt-and-suspenders: if the server doesn't honor filter_updated_before, - // drop items above the ceiling client-side. With DESC ordering, items + // MantisBT does not expose an updated-before filter in its REST API, so + // enforce the snapshot ceiling client-side. With DESC ordering, items // newer than the ceiling appear at the front of the page. if (c.snapshotCeiling) { const ceilingTs = parseTimestamp(c.snapshotCeiling); diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 93d8dff..4d1a752 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -1,6 +1,6 @@ { "name": "@frozenink/desktop", - "version": "0.10.9", + "version": "0.10.15", "productName": "Frozen Ink", "private": true, "main": "dist/main/index.mjs", diff --git a/packages/desktop/src/main/index.ts b/packages/desktop/src/main/index.ts index 0c2d0f1..fb6bf26 100644 --- a/packages/desktop/src/main/index.ts +++ b/packages/desktop/src/main/index.ts @@ -97,7 +97,7 @@ app.whenReady().then(async () => { const aboutIconPath = join(__desktopDirname, "../../build/icon.png"); app.setAboutPanelOptions({ applicationName: "Frozen Ink", - applicationVersion: "0.1.0", + applicationVersion: app.getVersion(), iconPath: aboutIconPath, }); Menu.setApplicationMenu(Menu.buildFromTemplate([ diff --git a/packages/ui/src/App.tsx b/packages/ui/src/App.tsx index 203a0a0..5782e05 100644 --- a/packages/ui/src/App.tsx +++ b/packages/ui/src/App.tsx @@ -353,6 +353,14 @@ export default function App() { }, []); const handleUIModeChange = useCallback((mode: UIMode) => { + if (mode === "manage" && uiMode !== "manage") { + // Open the collection the user is currently browsing directly in its + // management detail view, next to the Manage navigation sidebar. + setManageSection("collections"); + setViewingCollection(selectedCollection); + setEditingCollection(null); + setAddingCollection(false); + } // When leaving manage for browse, carry the collection the user was // looking at (or editing) over to the browse view so they don't have // to re-pick it from the dropdown. The enabled-only effect will fall @@ -363,7 +371,7 @@ export default function App() { } setUIMode(mode); saveUIMode(mode); - }, [viewingCollection, editingCollection]); + }, [editingCollection, selectedCollection, uiMode, viewingCollection]); const handleThemeChange = useCallback((newTheme: ThemeId) => { setTheme(newTheme); diff --git a/packages/ui/src/__tests__/SyncProgress.test.tsx b/packages/ui/src/__tests__/SyncProgress.test.tsx new file mode 100644 index 0000000..0cbf09a --- /dev/null +++ b/packages/ui/src/__tests__/SyncProgress.test.tsx @@ -0,0 +1,49 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import SyncProgress from "../components/manage/SyncProgress"; + +describe("SyncProgress", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("passes auto-republish failures to the parent when the job completes", async () => { + const activeJob = { + collectionName: "mantisbt", + active: true, + status: "republishing: uploading data", + created: 0, + updated: 1, + deleted: 0, + error: null, + startedAt: 1, + completedAt: null, + full: false, + }; + const completedJob = { + ...activeJob, + active: false, + status: "completed", + completedAt: 2, + republishError: "Cloudflare authentication expired", + }; + const fetchMock = vi.fn() + .mockResolvedValueOnce(new Response(JSON.stringify([activeJob]))) + .mockResolvedValueOnce(new Response(JSON.stringify([completedJob]))); + vi.stubGlobal("fetch", fetchMock); + const onComplete = vi.fn(); + + render(); + expect(await screen.findByText("republishing: uploading data")).toBeInTheDocument(); + + await waitFor(() => { + expect(onComplete).toHaveBeenCalledWith({ + created: 0, + updated: 1, + deleted: 0, + error: null, + republishError: "Cloudflare authentication expired", + }); + }); + }); +}); diff --git a/packages/ui/src/components/manage/ActiveSyncs.tsx b/packages/ui/src/components/manage/ActiveSyncs.tsx index 7f0347b..ea43ba9 100644 --- a/packages/ui/src/components/manage/ActiveSyncs.tsx +++ b/packages/ui/src/components/manage/ActiveSyncs.tsx @@ -88,6 +88,11 @@ export default function ActiveSyncs({ onJobComplete }: ActiveSyncsProps) { )} {job.error &&
{job.error}
} + {job.republishError && ( +
+ Sync completed, but publishing the updated collection failed: {job.republishError} +
+ )} ); })} diff --git a/packages/ui/src/components/manage/CollectionDetail.tsx b/packages/ui/src/components/manage/CollectionDetail.tsx index a4c6943..ea22deb 100644 --- a/packages/ui/src/components/manage/CollectionDetail.tsx +++ b/packages/ui/src/components/manage/CollectionDetail.tsx @@ -174,8 +174,9 @@ export default function CollectionDetail({ name, onBack, onEdit, onCollectionsCh setSyncing(false); setLastSyncResult(result); loadStatus(); + loadCollection(); onCollectionsChanged?.(); - }, [loadStatus, onCollectionsChanged]); + }, [loadCollection, loadStatus, onCollectionsChanged]); // --- Enable/Disable --- const handleToggle = (enabled: boolean) => { @@ -434,6 +435,11 @@ export default function CollectionDetail({ name, onBack, onEdit, onCollectionsCh {status?.lastSyncRun && Last sync: {formatTimestamp(status.lastSyncRun.startedAt)}} )} + {lastSyncResult.republishError && ( +
+ Sync completed, but publishing the updated collection failed: {lastSyncResult.republishError} +
+ )} )} diff --git a/packages/ui/src/components/manage/CollectionList.tsx b/packages/ui/src/components/manage/CollectionList.tsx index 58d33c7..ff87324 100644 --- a/packages/ui/src/components/manage/CollectionList.tsx +++ b/packages/ui/src/components/manage/CollectionList.tsx @@ -1,5 +1,5 @@ import { useState, useEffect } from "react"; -import SyncProgress from "./SyncProgress"; +import SyncProgress, { type SyncResult } from "./SyncProgress"; import { formatTimestamp, type Collection } from "../../types"; function formatSize(bytes: number): string { @@ -80,6 +80,7 @@ interface CollectionListProps { export default function CollectionList({ onSelect, onAdd, onSyncComplete, onCollectionsChanged }: CollectionListProps) { const [collections, setCollections] = useState([]); const [syncing, setSyncing] = useState(false); + const [lastSyncResult, setLastSyncResult] = useState(null); const load = () => { fetch("/api/collections") @@ -92,6 +93,7 @@ export default function CollectionList({ onSelect, onAdd, onSyncComplete, onColl const syncAll = (full: boolean) => { setSyncing(true); + setLastSyncResult(null); fetch("/api/sync", { method: "POST", headers: { "Content-Type": "application/json" }, @@ -99,8 +101,9 @@ export default function CollectionList({ onSelect, onAdd, onSyncComplete, onColl }).catch(console.error); }; - const handleSyncComplete = () => { + const handleSyncComplete = (result: SyncResult) => { setSyncing(false); + setLastSyncResult(result); load(); onSyncComplete?.(); onCollectionsChanged?.(); @@ -121,6 +124,14 @@ export default function CollectionList({ onSelect, onAdd, onSyncComplete, onColl {syncing && } + {!syncing && lastSyncResult?.error && ( +
{lastSyncResult.error}
+ )} + {!syncing && lastSyncResult?.republishError && ( +
+ Sync completed, but publishing one or more updated collections failed: {lastSyncResult.republishError} +
+ )} {(() => { const byName = (a: Collection, b: Collection) => diff --git a/packages/ui/src/components/manage/PublishPanel.tsx b/packages/ui/src/components/manage/PublishPanel.tsx index 1e04ab1..3fb7e3a 100644 --- a/packages/ui/src/components/manage/PublishPanel.tsx +++ b/packages/ui/src/components/manage/PublishPanel.tsx @@ -39,6 +39,7 @@ export default function PublishPanel() { }, []); const publishedCollections = collections.filter((c) => c.publish); + const initialPublishCollections = collections.filter((c) => c.enabled && !c.publish); const checkAuth = () => { setAuthError(null); @@ -223,7 +224,7 @@ export default function PublishPanel() { )} -
+ {initialPublishCollections.length > 0 &&

Publish a Collection

@@ -233,9 +234,9 @@ export default function PublishPanel() { onChange={(e) => setSelectedCollection(e.target.value)} > - {collections.filter((c) => c.enabled).map((col) => ( + {initialPublishCollections.map((col) => ( ))} @@ -271,7 +272,7 @@ export default function PublishPanel() { > {publishing ? "Publishing..." : "Publish"} -
+
}
); } diff --git a/packages/ui/src/components/manage/SyncProgress.tsx b/packages/ui/src/components/manage/SyncProgress.tsx index 222a41b..3453bad 100644 --- a/packages/ui/src/components/manage/SyncProgress.tsx +++ b/packages/ui/src/components/manage/SyncProgress.tsx @@ -6,6 +6,7 @@ export interface SyncResult { updated: number; deleted: number; error: string | null; + republishError: string | null; } interface SyncProgressProps { @@ -114,6 +115,7 @@ export default function SyncProgress({ collectionName, onComplete }: SyncProgres updated: next.updated, deleted: next.deleted, error: next.error, + republishError: next.republishError, }); } }) diff --git a/packages/worker/src/handlers/api.ts b/packages/worker/src/handlers/api.ts index 7c01535..411d0f7 100644 --- a/packages/worker/src/handlers/api.ts +++ b/packages/worker/src/handlers/api.ts @@ -15,7 +15,7 @@ import { import type { Entity } from "../db/client"; import { getCollections, getCollectionConfig } from "../config"; import { searchEntities } from "../db/search"; -import { getR2Object, getMimeType } from "../storage/r2"; +import { getR2Asset, getMimeType } from "../storage/r2"; import { ThemeEngine } from "@frozenink/core/theme"; import type { ThemeRenderContext, FolderConfig } from "@frozenink/core/theme"; import { GitHubTheme, ObsidianTheme, GitTheme, MantisHubTheme, RssTheme } from "@frozenink/crawlers/themes"; @@ -273,6 +273,24 @@ api.get("/api/collections/:name/markdown/*", async (c) => { }); }); +// GET /api/collections/:name/file/*storagePath +// Serve crawler assets stored below the collection's content directory. +api.get("/api/collections/:name/file/*", async (c) => { + const name = c.req.param("name"); + const filePath = c.req.path.replace(`/api/collections/${encodeURIComponent(name)}/file/`, ""); + const decoded = decodeURIComponent(filePath); + if (!decoded.startsWith("content/") || decoded.split("/").includes("..")) { + return c.text("Forbidden", 403); + } + + const obj = await getR2Asset(c.env.BUCKET, `${name}/${decoded}`); + if (!obj) return c.text("File not found", 404); + + return new Response(obj.body, { + headers: { "Content-Type": getMimeType(decoded) }, + }); +}); + // GET /api/collections/:name/entities api.get("/api/collections/:name/entities", async (c) => { const name = c.req.param("name"); @@ -543,7 +561,7 @@ api.get("/api/attachments/:collection/*", async (c) => { const decoded = decodeURIComponent(filePath); const r2Key = `${collection}/attachments/${decoded}`; - const obj = await getR2Object(c.env.BUCKET, r2Key); + const obj = await getR2Asset(c.env.BUCKET, r2Key); if (!obj) return c.text("Attachment not found", 404); return new Response(obj.body, { diff --git a/packages/worker/src/handlers/mcp.ts b/packages/worker/src/handlers/mcp.ts index 637082d..1cbeb76 100644 --- a/packages/worker/src/handlers/mcp.ts +++ b/packages/worker/src/handlers/mcp.ts @@ -7,7 +7,7 @@ import { } from "../db/client"; import { getCollections } from "../config"; import { searchEntities } from "../db/search"; -import { getR2Object } from "../storage/r2"; +import { getR2Asset } from "../storage/r2"; import { ThemeEngine } from "@frozenink/core/theme"; import { GitHubTheme, ObsidianTheme, GitTheme, MantisHubTheme } from "@frozenink/crawlers/themes"; import { buildRenderContext } from "./api"; @@ -214,7 +214,7 @@ async function callTool( } if (name === "entity_get_attachment") { - const obj = await getR2Object(env.BUCKET, `${args.collection}/attachments/${args.path}`); + const obj = await getR2Asset(env.BUCKET, `${args.collection}/attachments/${args.path}`); if (!obj) return [{ type: "text", text: JSON.stringify({ error: "Attachment not found" }) }]; const buf = await new Response(obj.body).arrayBuffer(); diff --git a/packages/worker/src/storage/__tests__/r2.test.ts b/packages/worker/src/storage/__tests__/r2.test.ts new file mode 100644 index 0000000..67bdb5d --- /dev/null +++ b/packages/worker/src/storage/__tests__/r2.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, test } from "bun:test"; +import { getR2Asset } from "../r2"; + +describe("R2 asset upgrade compatibility", () => { + test("reads legacy keys after worker-only upgrades", async () => { + const keys: string[] = []; + const legacy = { body: new ReadableStream() }; + const bucket = { get: async (key: string) => { + keys.push(key); + return key === "collection/attachments/image~1.png" ? legacy : null; + } }; + const result = await getR2Asset(bucket as unknown as R2Bucket, "collection/attachments/image~1.png"); + expect(result?.body).toBe(legacy.body); + expect(keys).toEqual(["collection/attachments/image~~1.png", "collection/attachments/image~1.png"]); + }); + test("prefers newly published encoded objects", async () => { + const keys: string[] = []; + const object = { body: new ReadableStream() }; + const bucket = { get: async (key: string) => { keys.push(key); return object; } }; + expect((await getR2Asset(bucket as unknown as R2Bucket, "collection/attachments/image..png"))?.body).toBe(object.body); + expect(keys).toEqual(["collection/attachments/image~dd~png"]); + }); +}); diff --git a/packages/worker/src/storage/r2.ts b/packages/worker/src/storage/r2.ts index 6f565aa..b812937 100644 --- a/packages/worker/src/storage/r2.ts +++ b/packages/worker/src/storage/r2.ts @@ -1,3 +1,5 @@ +import { toR2SafeKey } from "@frozenink/core/storage/r2-key"; + export async function getR2Object( bucket: R2Bucket, key: string, @@ -31,3 +33,11 @@ export function getMimeType(path: string): string { }; return map[ext] || "application/octet-stream"; } + +/** Read new encoded keys while retaining compatibility with existing deployments. */ +export async function getR2Asset(bucket: R2Bucket, originalKey: string) { + const safeKey = toR2SafeKey(originalKey); + const object = await getR2Object(bucket, safeKey); + if (object || safeKey === originalKey) return object; + return getR2Object(bucket, originalKey); +}