diff --git a/docs/architecture/native-bridge.md b/docs/architecture/native-bridge.md new file mode 100644 index 00000000..ef320f77 --- /dev/null +++ b/docs/architecture/native-bridge.md @@ -0,0 +1,39 @@ +# Native Bridge Architecture + +## Goal + +Provide a single, resilient source of truth for platform-native capabilities while keeping Electron transport thin and renderer APIs unified. + +## Layers + +1. Native adapters +Platform-specific providers implement stable domain interfaces such as cursor telemetry or system asset discovery. + +2. Main-process services +Services orchestrate adapters, own runtime state, and expose domain-level operations. + +3. Unified IPC transport +Renderer code talks to a single `native-bridge:invoke` channel using versioned contracts. + +4. Renderer client +React code should consume `src/native/client.ts` rather than binding directly to ad hoc Electron APIs. + +## Principles + +- Single source of truth: runtime-native state lives in the Electron main process. +- Capability-first: renderer can query support before attempting native behavior. +- Versioned contracts: requests and responses are explicit and evolve predictably. +- Resilience: every response uses a consistent result envelope with stable error codes. + +## Current rollout + +This repository now contains the initial scaffold: + +- shared contracts in `src/native/contracts.ts` +- renderer SDK in `src/native/client.ts` +- main-process state store in `electron/native-bridge/store.ts` +- cursor telemetry adapter in `electron/native-bridge/cursor/telemetryCursorAdapter.ts` +- domain services in `electron/native-bridge/services/*` +- unified handler registration in `electron/ipc/nativeBridge.ts` + +The legacy `window.electronAPI` surface still exists for backward compatibility. New native-facing features should prefer the unified bridge client. \ No newline at end of file diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index b771e469..d4d73fbb 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -24,6 +24,9 @@ declare namespace NodeJS { // Used in Renderer process, expose in `preload.ts` interface Window { electronAPI: { + invokeNativeBridge: ( + request: import("../src/native/contracts").NativeBridgeRequest, + ) => Promise>; getSources: (opts: Electron.SourcesOptions) => Promise; switchToEditor: () => Promise; openSourceSelector: () => Promise; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 70d3ae40..0043b870 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -2,7 +2,17 @@ import fs from "node:fs/promises"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, screen, shell } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, + ProjectFileResult, + ProjectPathResult, +} from "../../src/native/contracts"; import { RECORDINGS_DIR } from "../main"; +import { createCursorRecordingSession } from "../native-bridge/cursor/recording/factory"; +import type { CursorRecordingSession } from "../native-bridge/cursor/recording/session"; +import { registerNativeBridgeHandlers } from "./nativeBridge"; const PROJECT_FILE_EXTENSION = "openscreen"; const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json"); @@ -14,6 +24,7 @@ type SelectedSource = { let selectedSource: SelectedSource | null = null; let currentProjectPath: string | null = null; +let currentVideoPath: string | null = null; function normalizePath(filePath: string) { return path.resolve(filePath); @@ -47,55 +58,162 @@ function isTrustedProjectPath(filePath?: string | null) { return normalizePath(filePath) === normalizePath(currentProjectPath); } -const CURSOR_TELEMETRY_VERSION = 1; +const CURSOR_TELEMETRY_VERSION = 2; const CURSOR_SAMPLE_INTERVAL_MS = 100; const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz -interface CursorTelemetryPoint { - timeMs: number; - cx: number; - cy: number; -} - -let cursorCaptureInterval: NodeJS.Timeout | null = null; -let cursorCaptureStartTimeMs = 0; -let activeCursorSamples: CursorTelemetryPoint[] = []; -let pendingCursorSamples: CursorTelemetryPoint[] = []; +let cursorRecordingSession: CursorRecordingSession | null = null; +let pendingCursorRecordingData: CursorRecordingData | null = null; function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } -function stopCursorCapture() { - if (cursorCaptureInterval) { - clearInterval(cursorCaptureInterval); - cursorCaptureInterval = null; +function normalizeCursorSample(sample: unknown): CursorRecordingSample | null { + if (!sample || typeof sample !== "object") { + return null; + } + + const point = sample as Partial; + return { + timeMs: + typeof point.timeMs === "number" && Number.isFinite(point.timeMs) + ? Math.max(0, point.timeMs) + : 0, + cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5, + cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5, + assetId: typeof point.assetId === "string" ? point.assetId : null, + visible: typeof point.visible === "boolean" ? point.visible : true, + }; +} + +function normalizeCursorAsset(asset: unknown): NativeCursorAsset | null { + if (!asset || typeof asset !== "object") { + return null; + } + + const candidate = asset as Partial; + if (typeof candidate.id !== "string" || typeof candidate.imageDataUrl !== "string") { + return null; + } + + return { + id: candidate.id, + platform: + candidate.platform === "win32" ? "win32" : process.platform === "darwin" ? "darwin" : "linux", + imageDataUrl: candidate.imageDataUrl, + width: + typeof candidate.width === "number" && Number.isFinite(candidate.width) + ? Math.max(1, Math.round(candidate.width)) + : 1, + height: + typeof candidate.height === "number" && Number.isFinite(candidate.height) + ? Math.max(1, Math.round(candidate.height)) + : 1, + hotspotX: + typeof candidate.hotspotX === "number" && Number.isFinite(candidate.hotspotX) + ? Math.max(0, Math.round(candidate.hotspotX)) + : 0, + hotspotY: + typeof candidate.hotspotY === "number" && Number.isFinite(candidate.hotspotY) + ? Math.max(0, Math.round(candidate.hotspotY)) + : 0, + scaleFactor: + typeof candidate.scaleFactor === "number" && Number.isFinite(candidate.scaleFactor) + ? Math.max(0.1, candidate.scaleFactor) + : undefined, + }; +} + +async function readCursorRecordingFile(targetVideoPath: string): Promise { + const telemetryPath = `${targetVideoPath}.cursor.json`; + try { + const content = await fs.readFile(telemetryPath, "utf-8"); + const parsed = JSON.parse(content); + const rawSamples = Array.isArray(parsed) + ? parsed + : Array.isArray(parsed?.samples) + ? parsed.samples + : []; + const rawAssets = Array.isArray(parsed?.assets) ? parsed.assets : []; + + const samples = rawSamples + .map((sample: unknown) => normalizeCursorSample(sample)) + .filter((sample: CursorRecordingSample | null): sample is CursorRecordingSample => + Boolean(sample), + ) + .sort((a: CursorRecordingSample, b: CursorRecordingSample) => a.timeMs - b.timeMs); + + const assets = rawAssets + .map((asset: unknown) => normalizeCursorAsset(asset)) + .filter((asset: NativeCursorAsset | null): asset is NativeCursorAsset => Boolean(asset)); + + return { + version: + typeof parsed?.version === "number" && Number.isFinite(parsed.version) ? parsed.version : 1, + provider: parsed?.provider === "native" ? "native" : "none", + samples, + assets, + }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { + version: CURSOR_TELEMETRY_VERSION, + provider: "none", + samples: [], + assets: [], + }; + } + + console.error("Failed to load cursor telemetry:", error); + throw error; + } +} + +async function readCursorTelemetryFile(targetVideoPath: string) { + try { + const recordingData = await readCursorRecordingFile(targetVideoPath); + return { + success: true, + samples: recordingData.samples.map((sample) => ({ + timeMs: sample.timeMs, + cx: sample.cx, + cy: sample.cy, + })), + }; + } catch (error) { + console.error("Failed to load cursor telemetry:", error); + return { + success: false, + message: "Failed to load cursor telemetry", + error: String(error), + samples: [], + }; } } -function sampleCursorPoint() { +function resolveAssetBasePath() { + try { + if (app.isPackaged) { + const assetPath = path.join(process.resourcesPath, "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } + const assetPath = path.join(app.getAppPath(), "public", "assets"); + return pathToFileURL(`${assetPath}${path.sep}`).toString(); + } catch (err) { + console.error("Failed to resolve asset base path:", err); + return null; + } +} + +function getSelectedSourceBounds() { const cursor = screen.getCursorScreenPoint(); const sourceDisplayId = Number(selectedSource?.display_id); const sourceDisplay = Number.isFinite(sourceDisplayId) ? (screen.getAllDisplays().find((display) => display.id === sourceDisplayId) ?? null) : null; - const display = sourceDisplay ?? screen.getDisplayNearestPoint(cursor); - const bounds = display.bounds; - const width = Math.max(1, bounds.width); - const height = Math.max(1, bounds.height); - - const cx = clamp((cursor.x - bounds.x) / width, 0, 1); - const cy = clamp((cursor.y - bounds.y) / height, 0, 1); - - activeCursorSamples.push({ - timeMs: Math.max(0, Date.now() - cursorCaptureStartTimeMs), - cx, - cy, - }); - - if (activeCursorSamples.length > MAX_CURSOR_SAMPLES) { - activeCursorSamples.shift(); - } + return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds; } export function registerIpcHandlers( @@ -153,18 +271,14 @@ export function registerIpcHandlers( currentProjectPath = null; const telemetryPath = `${videoPath}.cursor.json`; - if (pendingCursorSamples.length > 0) { + if (pendingCursorRecordingData && pendingCursorRecordingData.samples.length > 0) { await fs.writeFile( telemetryPath, - JSON.stringify( - { version: CURSOR_TELEMETRY_VERSION, samples: pendingCursorSamples }, - null, - 2, - ), + JSON.stringify(pendingCursorRecordingData, null, 2), "utf-8", ); } - pendingCursorSamples = []; + pendingCursorRecordingData = null; return { success: true, @@ -200,18 +314,38 @@ export function registerIpcHandlers( } }); - ipcMain.handle("set-recording-state", (_, recording: boolean) => { + ipcMain.handle("set-recording-state", async (_, recording: boolean) => { if (recording) { - stopCursorCapture(); - activeCursorSamples = []; - pendingCursorSamples = []; - cursorCaptureStartTimeMs = Date.now(); - sampleCursorPoint(); - cursorCaptureInterval = setInterval(sampleCursorPoint, CURSOR_SAMPLE_INTERVAL_MS); + if (cursorRecordingSession) { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + cursorRecordingSession = null; + } + + pendingCursorRecordingData = null; + cursorRecordingSession = createCursorRecordingSession({ + getDisplayBounds: getSelectedSourceBounds, + maxSamples: MAX_CURSOR_SAMPLES, + platform: process.platform, + sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS, + }); + + try { + await cursorRecordingSession.start(); + } catch (error) { + console.error("Failed to start cursor recording session:", error); + cursorRecordingSession = null; + } } else { - stopCursorCapture(); - pendingCursorSamples = [...activeCursorSamples]; - activeCursorSamples = []; + if (cursorRecordingSession) { + try { + pendingCursorRecordingData = await cursorRecordingSession.stop(); + } catch (error) { + console.error("Failed to stop cursor recording session:", error); + pendingCursorRecordingData = null; + } finally { + cursorRecordingSession = null; + } + } } const source = selectedSource || { name: "Screen" }; @@ -226,51 +360,7 @@ export function registerIpcHandlers( return { success: true, samples: [] }; } - const telemetryPath = `${targetVideoPath}.cursor.json`; - try { - const content = await fs.readFile(telemetryPath, "utf-8"); - const parsed = JSON.parse(content); - const rawSamples = Array.isArray(parsed) - ? parsed - : Array.isArray(parsed?.samples) - ? parsed.samples - : []; - - const samples: CursorTelemetryPoint[] = rawSamples - .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) - .map((sample: unknown) => { - const point = sample as Partial; - return { - timeMs: - typeof point.timeMs === "number" && Number.isFinite(point.timeMs) - ? Math.max(0, point.timeMs) - : 0, - cx: - typeof point.cx === "number" && Number.isFinite(point.cx) - ? clamp(point.cx, 0, 1) - : 0.5, - cy: - typeof point.cy === "number" && Number.isFinite(point.cy) - ? clamp(point.cy, 0, 1) - : 0.5, - }; - }) - .sort((a: CursorTelemetryPoint, b: CursorTelemetryPoint) => a.timeMs - b.timeMs); - - return { success: true, samples }; - } catch (error) { - const nodeError = error as NodeJS.ErrnoException; - if (nodeError.code === "ENOENT") { - return { success: true, samples: [] }; - } - console.error("Failed to load cursor telemetry:", error); - return { - success: false, - message: "Failed to load cursor telemetry", - error: String(error), - samples: [], - }; - } + return readCursorTelemetryFile(targetVideoPath); }); ipcMain.handle("open-external-url", async (_, url: string) => { @@ -285,17 +375,7 @@ export function registerIpcHandlers( // Return base path for assets so renderer can resolve file:// paths in production ipcMain.handle("get-asset-base-path", () => { - try { - if (app.isPackaged) { - const assetPath = path.join(process.resourcesPath, "assets"); - return pathToFileURL(`${assetPath}${path.sep}`).toString(); - } - const assetPath = path.join(app.getAppPath(), "public", "assets"); - return pathToFileURL(`${assetPath}${path.sep}`).toString(); - } catch (err) { - console.error("Failed to resolve asset base path:", err); - return null; - } + return resolveAssetBasePath(); }); ipcMain.handle("save-exported-video", async (_, videoData: ArrayBuffer, fileName: string) => { @@ -393,72 +473,83 @@ export function registerIpcHandlers( } }); - let currentVideoPath: string | null = null; ipcMain.handle( "save-project-file", async (_, projectData: unknown, suggestedName?: string, existingProjectPath?: string) => { - try { - const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) - ? existingProjectPath - : null; - - if (trustedExistingProjectPath) { - await fs.writeFile( - trustedExistingProjectPath, - JSON.stringify(projectData, null, 2), - "utf-8", - ); - currentProjectPath = trustedExistingProjectPath; - return { - success: true, - path: trustedExistingProjectPath, - message: "Project saved successfully", - }; - } - - const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); - const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) - ? safeName - : `${safeName}.${PROJECT_FILE_EXTENSION}`; - - const result = await dialog.showSaveDialog({ - title: "Save OpenScreen Project", - defaultPath: path.join(RECORDINGS_DIR, defaultName), - filters: [ - { name: "OpenScreen Project", extensions: [PROJECT_FILE_EXTENSION] }, - { name: "JSON", extensions: ["json"] }, - ], - properties: ["createDirectory", "showOverwriteConfirmation"], - }); - - if (result.canceled || !result.filePath) { - return { - success: false, - canceled: true, - message: "Save project canceled", - }; - } + return saveProjectFile(projectData, suggestedName, existingProjectPath); + }, + ); - await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); - currentProjectPath = result.filePath; + async function saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ): Promise { + try { + const trustedExistingProjectPath = isTrustedProjectPath(existingProjectPath) + ? existingProjectPath + : null; + if (trustedExistingProjectPath) { + await fs.writeFile( + trustedExistingProjectPath, + JSON.stringify(projectData, null, 2), + "utf-8", + ); + currentProjectPath = trustedExistingProjectPath; return { success: true, - path: result.filePath, + path: trustedExistingProjectPath, message: "Project saved successfully", }; - } catch (error) { - console.error("Failed to save project file:", error); + } + + const safeName = (suggestedName || `project-${Date.now()}`).replace(/[^a-zA-Z0-9-_]/g, "_"); + const defaultName = safeName.endsWith(`.${PROJECT_FILE_EXTENSION}`) + ? safeName + : `${safeName}.${PROJECT_FILE_EXTENSION}`; + + const result = await dialog.showSaveDialog({ + title: "Save OpenScreen Project", + defaultPath: path.join(RECORDINGS_DIR, defaultName), + filters: [ + { name: "OpenScreen Project", extensions: [PROJECT_FILE_EXTENSION] }, + { name: "JSON", extensions: ["json"] }, + ], + properties: ["createDirectory", "showOverwriteConfirmation"], + }); + + if (result.canceled || !result.filePath) { return { success: false, - message: "Failed to save project file", - error: String(error), + canceled: true, + message: "Save project canceled", }; } - }, - ); + + await fs.writeFile(result.filePath, JSON.stringify(projectData, null, 2), "utf-8"); + currentProjectPath = result.filePath; + + return { + success: true, + path: result.filePath, + message: "Project saved successfully", + }; + } catch (error) { + console.error("Failed to save project file:", error); + return { + success: false, + message: "Failed to save project file", + error: String(error), + }; + } + } ipcMain.handle("load-project-file", async () => { + return loadProjectFile(); + }); + + async function loadProjectFile(): Promise { try { const result = await dialog.showOpenDialog({ title: "Open OpenScreen Project", @@ -496,9 +587,13 @@ export function registerIpcHandlers( error: String(error), }; } - }); + } ipcMain.handle("load-current-project-file", async () => { + return loadCurrentProjectFile(); + }); + + async function loadCurrentProjectFile(): Promise { try { if (!currentProjectPath) { return { success: false, message: "No active project" }; @@ -522,21 +617,34 @@ export function registerIpcHandlers( error: String(error), }; } - }); + } + ipcMain.handle("set-current-video-path", (_, path: string) => { + return setCurrentVideoPath(path); + }); + + function setCurrentVideoPath(path: string): ProjectPathResult { currentVideoPath = normalizeVideoSourcePath(path) ?? path; currentProjectPath = null; return { success: true }; - }); + } ipcMain.handle("get-current-video-path", () => { - return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; + return getCurrentVideoPathResult(); }); + function getCurrentVideoPathResult(): ProjectPathResult { + return currentVideoPath ? { success: true, path: currentVideoPath } : { success: false }; + } + ipcMain.handle("clear-current-video-path", () => { + return clearCurrentVideoPath(); + }); + + function clearCurrentVideoPath(): ProjectPathResult { currentVideoPath = null; return { success: true }; - }); + } ipcMain.handle("get-platform", () => { return process.platform; @@ -560,4 +668,21 @@ export function registerIpcHandlers( return { success: false, error: String(error) }; } }); + + registerNativeBridgeHandlers({ + getPlatform: () => process.platform, + getCurrentProjectPath: () => currentProjectPath, + getCurrentVideoPath: () => currentVideoPath, + saveProjectFile, + loadProjectFile, + loadCurrentProjectFile, + setCurrentVideoPath, + getCurrentVideoPathResult, + clearCurrentVideoPath, + resolveAssetBasePath, + resolveVideoPath: (videoPath?: string | null) => + normalizeVideoSourcePath(videoPath ?? currentVideoPath), + loadCursorRecordingData: readCursorRecordingFile, + loadCursorTelemetry: readCursorTelemetryFile, + }); } diff --git a/electron/ipc/nativeBridge.ts b/electron/ipc/nativeBridge.ts new file mode 100644 index 00000000..ba6258a3 --- /dev/null +++ b/electron/ipc/nativeBridge.ts @@ -0,0 +1,229 @@ +import { ipcMain } from "electron"; +import { + NATIVE_BRIDGE_CHANNEL, + NATIVE_BRIDGE_VERSION, + type NativeBridgeErrorCode, + type NativeBridgeRequest, + type NativeBridgeResponse, + type NativePlatform, + type ProjectFileResult, + type ProjectPathResult, +} from "../../src/native/contracts"; +import type { CursorTelemetryLoadResult } from "../native-bridge/cursor/adapter"; +import { TelemetryCursorAdapter } from "../native-bridge/cursor/telemetryCursorAdapter"; +import { CursorService } from "../native-bridge/services/cursorService"; +import { ProjectService } from "../native-bridge/services/projectService"; +import { SystemService } from "../native-bridge/services/systemService"; +import { NativeBridgeStateStore } from "../native-bridge/store"; + +export interface NativeBridgeContext { + getPlatform: () => NodeJS.Platform; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; + resolveAssetBasePath: () => string | null; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadCursorRecordingData: ( + videoPath: string, + ) => Promise; + loadCursorTelemetry: (videoPath: string) => Promise; +} + +function normalizePlatform(platform: NodeJS.Platform): NativePlatform { + if (platform === "darwin" || platform === "win32") { + return platform; + } + + return "linux"; +} + +function createMeta(requestId?: string) { + return { + version: NATIVE_BRIDGE_VERSION, + requestId: requestId || `native-${Date.now()}`, + timestampMs: Date.now(), + } as const; +} + +function createSuccessResponse(requestId: string | undefined, data: TData) { + return { + ok: true, + data, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function createErrorResponse( + requestId: string | undefined, + code: NativeBridgeErrorCode, + message: string, + retryable = false, +) { + return { + ok: false, + error: { + code, + message, + retryable, + }, + meta: createMeta(requestId), + } satisfies NativeBridgeResponse; +} + +function isBridgeRequest(value: unknown): value is NativeBridgeRequest { + if (!value || typeof value !== "object") { + return false; + } + + const candidate = value as Partial; + return typeof candidate.domain === "string" && typeof candidate.action === "string"; +} + +export function registerNativeBridgeHandlers(context: NativeBridgeContext) { + ipcMain.removeHandler(NATIVE_BRIDGE_CHANNEL); + + const platform = normalizePlatform(context.getPlatform()); + const store = new NativeBridgeStateStore(platform); + const projectService = new ProjectService({ + store, + getCurrentProjectPath: context.getCurrentProjectPath, + getCurrentVideoPath: context.getCurrentVideoPath, + saveProjectFile: context.saveProjectFile, + loadProjectFile: context.loadProjectFile, + loadCurrentProjectFile: context.loadCurrentProjectFile, + setCurrentVideoPath: context.setCurrentVideoPath, + getCurrentVideoPathResult: context.getCurrentVideoPathResult, + clearCurrentVideoPath: context.clearCurrentVideoPath, + }); + const cursorService = new CursorService({ + store, + adapter: new TelemetryCursorAdapter({ + loadRecordingData: context.loadCursorRecordingData, + resolveVideoPath: context.resolveVideoPath, + loadTelemetry: context.loadCursorTelemetry, + }), + }); + const systemService = new SystemService({ + store, + getPlatform: () => platform, + getAssetBasePath: context.resolveAssetBasePath, + getCursorCapabilities: () => cursorService.getCapabilities(), + }); + + ipcMain.handle(NATIVE_BRIDGE_CHANNEL, async (_, request: unknown) => { + if (!isBridgeRequest(request)) { + return createErrorResponse(undefined, "INVALID_REQUEST", "Invalid native bridge request."); + } + + const requestId = request.requestId; + const domain = request.domain as string; + + try { + switch (request.domain) { + case "system": { + const action = request.action as string; + switch (request.action) { + case "getPlatform": + return createSuccessResponse(requestId, systemService.getPlatform()); + case "getAssetBasePath": + return createSuccessResponse(requestId, systemService.getAssetBasePath()); + case "getCapabilities": + return createSuccessResponse(requestId, await systemService.getCapabilities()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported system action: ${action}`, + ); + } + } + + case "project": { + const action = request.action as string; + switch (request.action) { + case "getCurrentContext": + return createSuccessResponse(requestId, projectService.getCurrentContext()); + case "saveProjectFile": + return createSuccessResponse( + requestId, + await projectService.saveProjectFile( + request.payload.projectData, + request.payload.suggestedName, + request.payload.existingProjectPath, + ), + ); + case "loadProjectFile": + return createSuccessResponse(requestId, await projectService.loadProjectFile()); + case "loadCurrentProjectFile": + return createSuccessResponse( + requestId, + await projectService.loadCurrentProjectFile(), + ); + case "setCurrentVideoPath": + return createSuccessResponse( + requestId, + projectService.setCurrentVideoPath(request.payload.path), + ); + case "getCurrentVideoPath": + return createSuccessResponse(requestId, projectService.getCurrentVideoPath()); + case "clearCurrentVideoPath": + return createSuccessResponse(requestId, projectService.clearCurrentVideoPath()); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported project action: ${action}`, + ); + } + } + + case "cursor": { + const action = request.action as string; + switch (request.action) { + case "getCapabilities": + return createSuccessResponse(requestId, await cursorService.getCapabilities()); + case "getTelemetry": + return createSuccessResponse( + requestId, + await cursorService.getTelemetry(request.payload?.videoPath), + ); + case "getRecordingData": + return createSuccessResponse( + requestId, + await cursorService.getRecordingData(request.payload?.videoPath), + ); + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported cursor action: ${action}`, + ); + } + } + + default: + return createErrorResponse( + requestId, + "UNSUPPORTED_ACTION", + `Unsupported bridge domain: ${domain}`, + ); + } + } catch (error) { + return createErrorResponse( + requestId, + "INTERNAL_ERROR", + error instanceof Error ? error.message : "Unknown native bridge error.", + true, + ); + } + }); +} diff --git a/electron/native-bridge/cursor/adapter.ts b/electron/native-bridge/cursor/adapter.ts new file mode 100644 index 00000000..cdb88e24 --- /dev/null +++ b/electron/native-bridge/cursor/adapter.ts @@ -0,0 +1,20 @@ +import type { + CursorCapabilities, + CursorProviderKind, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; + +export interface CursorTelemetryLoadResult { + success: boolean; + samples: CursorTelemetryPoint[]; + message?: string; + error?: string; +} + +export interface CursorNativeAdapter { + readonly kind: CursorProviderKind; + getCapabilities(): Promise; + getRecordingData(videoPath?: string | null): Promise; + getTelemetry(videoPath?: string | null): Promise; +} diff --git a/electron/native-bridge/cursor/recording/factory.ts b/electron/native-bridge/cursor/recording/factory.ts new file mode 100644 index 00000000..fe92991f --- /dev/null +++ b/electron/native-bridge/cursor/recording/factory.ts @@ -0,0 +1,29 @@ +import type { Rectangle } from "electron"; +import type { CursorRecordingSession } from "./session"; +import { TelemetryRecordingSession } from "./telemetryRecordingSession"; +import { WindowsNativeRecordingSession } from "./windowsNativeRecordingSession"; + +interface CreateCursorRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + platform: NodeJS.Platform; + sampleIntervalMs: number; +} + +export function createCursorRecordingSession( + options: CreateCursorRecordingSessionOptions, +): CursorRecordingSession { + if (options.platform === "win32") { + return new WindowsNativeRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + }); + } + + return new TelemetryRecordingSession({ + getDisplayBounds: options.getDisplayBounds, + maxSamples: options.maxSamples, + sampleIntervalMs: options.sampleIntervalMs, + }); +} diff --git a/electron/native-bridge/cursor/recording/session.ts b/electron/native-bridge/cursor/recording/session.ts new file mode 100644 index 00000000..9cebe9f4 --- /dev/null +++ b/electron/native-bridge/cursor/recording/session.ts @@ -0,0 +1,6 @@ +import type { CursorRecordingData } from "../../../../src/native/contracts"; + +export interface CursorRecordingSession { + start(): Promise; + stop(): Promise; +} diff --git a/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts new file mode 100644 index 00000000..dd42871b --- /dev/null +++ b/electron/native-bridge/cursor/recording/telemetryRecordingSession.ts @@ -0,0 +1,62 @@ +import { type Rectangle, screen } from "electron"; +import type { CursorRecordingData, CursorRecordingSample } from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface TelemetryRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export class TelemetryRecordingSession implements CursorRecordingSession { + private samples: CursorRecordingSample[] = []; + private interval: NodeJS.Timeout | null = null; + private startTimeMs = 0; + + constructor(private readonly options: TelemetryRecordingSessionOptions) {} + + async start(): Promise { + this.samples = []; + this.startTimeMs = Date.now(); + this.captureSample(); + this.interval = setInterval(() => { + this.captureSample(); + }, this.options.sampleIntervalMs); + } + + async stop(): Promise { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + + return { + version: 2, + provider: "none", + samples: this.samples, + assets: [], + }; + } + + private captureSample() { + const cursor = screen.getCursorScreenPoint(); + const display = this.options.getDisplayBounds() ?? screen.getDisplayNearestPoint(cursor).bounds; + const width = Math.max(1, display.width); + const height = Math.max(1, display.height); + + this.samples.push({ + timeMs: Math.max(0, Date.now() - this.startTimeMs), + cx: clamp((cursor.x - display.x) / width, 0, 1), + cy: clamp((cursor.y - display.y) / height, 0, 1), + visible: true, + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } +} diff --git a/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts new file mode 100644 index 00000000..a0540ed8 --- /dev/null +++ b/electron/native-bridge/cursor/recording/windowsNativeRecordingSession.ts @@ -0,0 +1,326 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import type { Readable } from "node:stream"; +import { type Rectangle, screen } from "electron"; +import type { + CursorRecordingData, + CursorRecordingSample, + NativeCursorAsset, +} from "../../../../src/native/contracts"; +import type { CursorRecordingSession } from "./session"; + +interface WindowsCursorSampleEvent { + type: "sample"; + timestampMs: number; + x: number; + y: number; + visible: boolean; + handle: string | null; + asset?: WindowsCursorAssetPayload; +} + +interface WindowsCursorReadyEvent { + type: "ready"; + timestampMs: number; +} + +interface WindowsCursorErrorEvent { + type: "error"; + timestampMs: number; + message: string; +} + +interface WindowsCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; +} + +type WindowsCursorEvent = + | WindowsCursorSampleEvent + | WindowsCursorReadyEvent + | WindowsCursorErrorEvent; + +interface WindowsNativeRecordingSessionOptions { + getDisplayBounds: () => Rectangle | null; + maxSamples: number; + sampleIntervalMs: number; +} + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function buildPowerShellCommand(sampleIntervalMs: number) { + const script = String.raw` +$ErrorActionPreference = 'Stop' +Add-Type -AssemblyName System.Drawing + +$source = @" +using System; +using System.Runtime.InteropServices; + +public static class OpenScreenCursorInterop { + [StructLayout(LayoutKind.Sequential)] + public struct POINT { + public int X; + public int Y; + } + + [StructLayout(LayoutKind.Sequential)] + public struct CURSORINFO { + public int cbSize; + public int flags; + public IntPtr hCursor; + public POINT ptScreenPos; + } + + [StructLayout(LayoutKind.Sequential)] + public struct ICONINFO { + [MarshalAs(UnmanagedType.Bool)] + public bool fIcon; + public int xHotspot; + public int yHotspot; + public IntPtr hbmMask; + public IntPtr hbmColor; + } + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetCursorInfo(ref CURSORINFO pci); + + [DllImport("user32.dll", SetLastError = true)] + public static extern IntPtr CopyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DestroyIcon(IntPtr hIcon); + + [DllImport("user32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo); + + [DllImport("gdi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + public static extern bool DeleteObject(IntPtr hObject); +} +"@ + +Add-Type -TypeDefinition $source + +function Write-JsonLine($payload) { + [Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6)) +} + +function Get-CursorAsset($cursorHandle, $cursorId) { + $copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle) + if ($copiedHandle -eq [IntPtr]::Zero) { + return $null + } + + $iconInfo = New-Object OpenScreenCursorInterop+ICONINFO + $hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo) + + try { + $icon = [System.Drawing.Icon]::FromHandle($copiedHandle) + $bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $memoryStream = New-Object System.IO.MemoryStream + + try { + $graphics.Clear([System.Drawing.Color]::Transparent) + $graphics.DrawIcon($icon, 0, 0) + $bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png) + $base64 = [System.Convert]::ToBase64String($memoryStream.ToArray()) + + return @{ + id = $cursorId + imageDataUrl = "data:image/png;base64,$base64" + width = $bitmap.Width + height = $bitmap.Height + hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 } + hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 } + } + } + finally { + $memoryStream.Dispose() + $graphics.Dispose() + $bitmap.Dispose() + $icon.Dispose() + } + } + finally { + if ($hasIconInfo) { + if ($iconInfo.hbmMask -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null + } + if ($iconInfo.hbmColor -ne [IntPtr]::Zero) { + [OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null + } + } + [OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null + } +} + +Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() } + +$lastCursorId = $null +while ($true) { + $cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO + $cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO]) + + if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) { + Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' } + Start-Sleep -Milliseconds ${sampleIntervalMs} + continue + } + + $visible = ($cursorInfo.flags -band 1) -ne 0 + $cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) } + $asset = $null + + if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) { + $asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId + $lastCursorId = $cursorId + } + + Write-JsonLine @{ + type = 'sample' + timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + x = $cursorInfo.ptScreenPos.X + y = $cursorInfo.ptScreenPos.Y + visible = $visible + handle = $cursorId + asset = $asset + } + + Start-Sleep -Milliseconds ${sampleIntervalMs} +} +`; + + return Buffer.from(script, "utf16le").toString("base64"); +} + +export class WindowsNativeRecordingSession implements CursorRecordingSession { + private assets = new Map(); + private samples: CursorRecordingSample[] = []; + private process: ChildProcessByStdio | null = null; + private lineBuffer = ""; + private startTimeMs = 0; + + constructor(private readonly options: WindowsNativeRecordingSessionOptions) {} + + async start(): Promise { + this.assets.clear(); + this.samples = []; + this.lineBuffer = ""; + this.startTimeMs = Date.now(); + + const encodedCommand = buildPowerShellCommand(this.options.sampleIntervalMs); + const child = spawn( + "powershell.exe", + [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-EncodedCommand", + encodedCommand, + ], + { + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }, + ); + + this.process = child; + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + this.handleStdoutChunk(chunk); + }); + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + console.error("[cursor-native]", chunk.trim()); + }); + } + + async stop(): Promise { + const child = this.process; + this.process = null; + + if (child && !child.killed) { + child.kill(); + } + + return { + version: 2, + provider: this.assets.size > 0 ? "native" : "none", + samples: this.samples, + assets: [...this.assets.values()], + }; + } + + private handleStdoutChunk(chunk: string) { + this.lineBuffer += chunk; + const lines = this.lineBuffer.split(/\r?\n/); + this.lineBuffer = lines.pop() ?? ""; + + for (const line of lines) { + const trimmedLine = line.trim(); + if (!trimmedLine) { + continue; + } + + try { + const payload = JSON.parse(trimmedLine) as WindowsCursorEvent; + this.handleEvent(payload); + } catch (error) { + console.error("Failed to parse Windows cursor helper output:", error, trimmedLine); + } + } + } + + private handleEvent(payload: WindowsCursorEvent) { + if (payload.type === "error") { + console.error("Windows cursor helper error:", payload.message); + return; + } + + if (payload.type === "ready") { + return; + } + + if (payload.asset?.id && !this.assets.has(payload.asset.id)) { + const assetDisplay = screen.getDisplayNearestPoint({ x: payload.x, y: payload.y }); + this.assets.set(payload.asset.id, { + id: payload.asset.id, + platform: "win32", + imageDataUrl: payload.asset.imageDataUrl, + width: payload.asset.width, + height: payload.asset.height, + hotspotX: payload.asset.hotspotX, + hotspotY: payload.asset.hotspotY, + scaleFactor: assetDisplay.scaleFactor, + }); + } + + const bounds = this.options.getDisplayBounds() ?? screen.getPrimaryDisplay().bounds; + const width = Math.max(1, bounds.width); + const height = Math.max(1, bounds.height); + + this.samples.push({ + timeMs: Math.max(0, payload.timestampMs - this.startTimeMs), + cx: clamp((payload.x - bounds.x) / width, 0, 1), + cy: clamp((payload.y - bounds.y) / height, 0, 1), + assetId: payload.handle, + visible: payload.visible, + }); + + if (this.samples.length > this.options.maxSamples) { + this.samples.shift(); + } + } +} diff --git a/electron/native-bridge/cursor/telemetryCursorAdapter.ts b/electron/native-bridge/cursor/telemetryCursorAdapter.ts new file mode 100644 index 00000000..d0839952 --- /dev/null +++ b/electron/native-bridge/cursor/telemetryCursorAdapter.ts @@ -0,0 +1,48 @@ +import type { CursorCapabilities, CursorRecordingData } from "../../../src/native/contracts"; +import type { CursorNativeAdapter, CursorTelemetryLoadResult } from "./adapter"; + +interface TelemetryCursorAdapterOptions { + loadRecordingData: (videoPath: string) => Promise; + resolveVideoPath: (videoPath?: string | null) => string | null; + loadTelemetry: (videoPath: string) => Promise; +} + +export class TelemetryCursorAdapter implements CursorNativeAdapter { + readonly kind = "none" as const; + + constructor(private readonly options: TelemetryCursorAdapterOptions) {} + + async getCapabilities(): Promise { + return { + telemetry: true, + systemAssets: false, + provider: this.kind, + }; + } + + async getRecordingData(videoPath?: string | null): Promise { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + version: 2, + provider: this.kind, + samples: [], + assets: [], + }; + } + + return this.options.loadRecordingData(resolvedVideoPath); + } + + async getTelemetry(videoPath?: string | null) { + const resolvedVideoPath = this.options.resolveVideoPath(videoPath); + if (!resolvedVideoPath) { + return { + success: true, + samples: [], + } satisfies CursorTelemetryLoadResult; + } + + return this.options.loadTelemetry(resolvedVideoPath); + } +} diff --git a/electron/native-bridge/services/cursorService.ts b/electron/native-bridge/services/cursorService.ts new file mode 100644 index 00000000..e3e9a255 --- /dev/null +++ b/electron/native-bridge/services/cursorService.ts @@ -0,0 +1,46 @@ +import type { + CursorCapabilities, + CursorRecordingData, + CursorTelemetryPoint, +} from "../../../src/native/contracts"; +import type { CursorNativeAdapter } from "../cursor/adapter"; +import type { NativeBridgeStateStore } from "../store"; + +interface CursorServiceOptions { + store: NativeBridgeStateStore; + adapter: CursorNativeAdapter; +} + +export class CursorService { + constructor(private readonly options: CursorServiceOptions) {} + + async getCapabilities(): Promise { + const capabilities = await this.options.adapter.getCapabilities(); + this.options.store.setCursorCapabilities(capabilities); + return capabilities; + } + + async getTelemetry(videoPath?: string | null): Promise { + const result = await this.options.adapter.getTelemetry(videoPath); + if (!result.success) { + throw new Error(result.message || result.error || "Failed to load cursor telemetry"); + } + + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, result.samples.length); + } + + return result.samples; + } + + async getRecordingData(videoPath?: string | null): Promise { + const data = await this.options.adapter.getRecordingData(videoPath); + const resolvedVideoPath = videoPath ?? this.options.store.getState().project.currentVideoPath; + if (resolvedVideoPath) { + this.options.store.markCursorTelemetryLoaded(resolvedVideoPath, data.samples.length); + } + + return data; + } +} diff --git a/electron/native-bridge/services/projectService.ts b/electron/native-bridge/services/projectService.ts new file mode 100644 index 00000000..e8d1cd5c --- /dev/null +++ b/electron/native-bridge/services/projectService.ts @@ -0,0 +1,80 @@ +import type { + ProjectContext, + ProjectFileResult, + ProjectPathResult, +} from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface ProjectServiceOptions { + store: NativeBridgeStateStore; + getCurrentProjectPath: () => string | null; + getCurrentVideoPath: () => string | null; + saveProjectFile: ( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) => Promise; + loadProjectFile: () => Promise; + loadCurrentProjectFile: () => Promise; + setCurrentVideoPath: (path: string) => ProjectPathResult; + getCurrentVideoPathResult: () => ProjectPathResult; + clearCurrentVideoPath: () => ProjectPathResult; +} + +export class ProjectService { + constructor(private readonly options: ProjectServiceOptions) {} + + getCurrentContext(): ProjectContext { + const context = { + currentProjectPath: this.options.getCurrentProjectPath(), + currentVideoPath: this.options.getCurrentVideoPath(), + }; + + this.options.store.setProjectContext(context); + return context; + } + + async saveProjectFile( + projectData: unknown, + suggestedName?: string, + existingProjectPath?: string, + ) { + const result = await this.options.saveProjectFile( + projectData, + suggestedName, + existingProjectPath, + ); + this.getCurrentContext(); + return result; + } + + async loadProjectFile() { + const result = await this.options.loadProjectFile(); + this.getCurrentContext(); + return result; + } + + async loadCurrentProjectFile() { + const result = await this.options.loadCurrentProjectFile(); + this.getCurrentContext(); + return result; + } + + setCurrentVideoPath(path: string) { + const result = this.options.setCurrentVideoPath(path); + this.getCurrentContext(); + return result; + } + + getCurrentVideoPath() { + const result = this.options.getCurrentVideoPathResult(); + this.getCurrentContext(); + return result; + } + + clearCurrentVideoPath() { + const result = this.options.clearCurrentVideoPath(); + this.getCurrentContext(); + return result; + } +} diff --git a/electron/native-bridge/services/systemService.ts b/electron/native-bridge/services/systemService.ts new file mode 100644 index 00000000..50eff283 --- /dev/null +++ b/electron/native-bridge/services/systemService.ts @@ -0,0 +1,43 @@ +import type { + CursorCapabilities, + NativePlatform, + SystemCapabilities, +} from "../../../src/native/contracts"; +import { NATIVE_BRIDGE_VERSION } from "../../../src/native/contracts"; +import type { NativeBridgeStateStore } from "../store"; + +interface SystemServiceOptions { + store: NativeBridgeStateStore; + getPlatform: () => NativePlatform; + getAssetBasePath: () => string | null; + getCursorCapabilities: () => Promise; +} + +export class SystemService { + constructor(private readonly options: SystemServiceOptions) {} + + getPlatform() { + return this.options.getPlatform(); + } + + getAssetBasePath() { + return this.options.getAssetBasePath(); + } + + async getCapabilities(): Promise { + const platform = this.getPlatform(); + const cursorCapabilities = await this.options.getCursorCapabilities(); + + const capabilities: SystemCapabilities = { + bridgeVersion: NATIVE_BRIDGE_VERSION, + platform, + cursor: cursorCapabilities, + project: { + currentContext: true, + }, + }; + + this.options.store.setSystemCapabilities(capabilities); + return capabilities; + } +} diff --git a/electron/native-bridge/store.ts b/electron/native-bridge/store.ts new file mode 100644 index 00000000..dcdbed15 --- /dev/null +++ b/electron/native-bridge/store.ts @@ -0,0 +1,88 @@ +import type { + CursorCapabilities, + NativePlatform, + ProjectContext, + SystemCapabilities, +} from "../../src/native/contracts"; + +export interface NativeBridgeState { + system: { + platform: NativePlatform; + capabilities: SystemCapabilities | null; + }; + project: ProjectContext; + cursor: { + capabilities: CursorCapabilities | null; + lastTelemetryLoad: { + videoPath: string; + sampleCount: number; + loadedAt: number; + } | null; + }; +} + +export class NativeBridgeStateStore { + private state: NativeBridgeState; + + constructor(platform: NativePlatform) { + this.state = { + system: { + platform, + capabilities: null, + }, + project: { + currentProjectPath: null, + currentVideoPath: null, + }, + cursor: { + capabilities: null, + lastTelemetryLoad: null, + }, + }; + } + + getState() { + return this.state; + } + + setProjectContext(project: ProjectContext) { + this.state = { + ...this.state, + project, + }; + } + + setSystemCapabilities(capabilities: SystemCapabilities) { + this.state = { + ...this.state, + system: { + ...this.state.system, + capabilities, + }, + }; + } + + setCursorCapabilities(capabilities: CursorCapabilities) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + capabilities, + }, + }; + } + + markCursorTelemetryLoaded(videoPath: string, sampleCount: number) { + this.state = { + ...this.state, + cursor: { + ...this.state.cursor, + lastTelemetryLoad: { + videoPath, + sampleCount, + loadedAt: Date.now(), + }, + }, + }; + } +} diff --git a/electron/preload.ts b/electron/preload.ts index 9eeb5b16..4e035f14 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,6 +1,10 @@ import { contextBridge, ipcRenderer } from "electron"; +import { NATIVE_BRIDGE_CHANNEL, type NativeBridgeRequest } from "../src/native/contracts"; contextBridge.exposeInMainWorld("electronAPI", { + invokeNativeBridge: (request: NativeBridgeRequest) => { + return ipcRenderer.invoke(NATIVE_BRIDGE_CHANNEL, request) as Promise; + }, hudOverlayHide: () => { ipcRenderer.send("hud-overlay-hide"); }, diff --git a/src/assets/cursors/Cursor=Beachball.svg b/src/assets/cursors/Cursor=Beachball.svg new file mode 100644 index 00000000..30bdbe50 --- /dev/null +++ b/src/assets/cursors/Cursor=Beachball.svg @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Cross.svg b/src/assets/cursors/Cursor=Cross.svg new file mode 100644 index 00000000..b404553d --- /dev/null +++ b/src/assets/cursors/Cursor=Cross.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Default.svg b/src/assets/cursors/Cursor=Default.svg new file mode 100644 index 00000000..f76f31fd --- /dev/null +++ b/src/assets/cursors/Cursor=Default.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Grabbing).svg b/src/assets/cursors/Cursor=Hand-(Grabbing).svg new file mode 100644 index 00000000..08278675 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Grabbing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Open).svg b/src/assets/cursors/Cursor=Hand-(Open).svg new file mode 100644 index 00000000..4ceafb0f --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Open).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Hand-(Pointing).svg b/src/assets/cursors/Cursor=Hand-(Pointing).svg new file mode 100644 index 00000000..19a70a67 --- /dev/null +++ b/src/assets/cursors/Cursor=Hand-(Pointing).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Menu.svg b/src/assets/cursors/Cursor=Menu.svg new file mode 100644 index 00000000..3489257b --- /dev/null +++ b/src/assets/cursors/Cursor=Menu.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/assets/cursors/Cursor=Move.svg b/src/assets/cursors/Cursor=Move.svg new file mode 100644 index 00000000..50e56b76 --- /dev/null +++ b/src/assets/cursors/Cursor=Move.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Down).svg b/src/assets/cursors/Cursor=Resize-(Down).svg new file mode 100644 index 00000000..fba36729 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left).svg b/src/assets/cursors/Cursor=Resize-(Left).svg new file mode 100644 index 00000000..6e21fb77 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Left-Right).svg b/src/assets/cursors/Cursor=Resize-(Left-Right).svg new file mode 100644 index 00000000..7021d229 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Left-Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Right).svg b/src/assets/cursors/Cursor=Resize-(Right).svg new file mode 100644 index 00000000..1ce801ce --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Right).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up).svg b/src/assets/cursors/Cursor=Resize-(Up).svg new file mode 100644 index 00000000..9c4ac0f0 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-(Up-Down).svg b/src/assets/cursors/Cursor=Resize-(Up-Down).svg new file mode 100644 index 00000000..b01a40e3 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-(Up-Down).svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-East-South-West.svg b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg new file mode 100644 index 00000000..1185c1ff --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-East-South-West.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-South.svg b/src/assets/cursors/Cursor=Resize-North-South.svg new file mode 100644 index 00000000..57eaa056 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-South.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-North-West-South-East.svg b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg new file mode 100644 index 00000000..f00fc879 --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-North-West-South-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Resize-West-East.svg b/src/assets/cursors/Cursor=Resize-West-East.svg new file mode 100644 index 00000000..ef1929fb --- /dev/null +++ b/src/assets/cursors/Cursor=Resize-West-East.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Text-Cursor.svg b/src/assets/cursors/Cursor=Text-Cursor.svg new file mode 100644 index 00000000..1bfd0809 --- /dev/null +++ b/src/assets/cursors/Cursor=Text-Cursor.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-In.svg b/src/assets/cursors/Cursor=Zoom-In.svg new file mode 100644 index 00000000..8ec9b3ce --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-In.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/assets/cursors/Cursor=Zoom-Out.svg b/src/assets/cursors/Cursor=Zoom-Out.svg new file mode 100644 index 00000000..810878ba --- /dev/null +++ b/src/assets/cursors/Cursor=Zoom-Out.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/components/launch/LaunchWindow.tsx b/src/components/launch/LaunchWindow.tsx index b456dd64..b260adae 100644 --- a/src/components/launch/LaunchWindow.tsx +++ b/src/components/launch/LaunchWindow.tsx @@ -6,6 +6,7 @@ import { FaFolderOpen } from "react-icons/fa6"; import { FiMinus, FiX } from "react-icons/fi"; import { MdMic, MdMicOff, MdMonitor, MdVideoFile, MdVolumeOff, MdVolumeUp } from "react-icons/md"; import { RxDragHandleDots2 } from "react-icons/rx"; +import { nativeBridgeClient } from "@/native"; import { useAudioLevelMeter } from "../../hooks/useAudioLevelMeter"; import { useMicrophoneDevices } from "../../hooks/useMicrophoneDevices"; import { useScreenRecorder } from "../../hooks/useScreenRecorder"; @@ -131,13 +132,13 @@ export function LaunchWindow() { } if (result.success && result.path) { - await window.electronAPI.setCurrentVideoPath(result.path); + await nativeBridgeClient.project.setCurrentVideoPath(result.path); await window.electronAPI.switchToEditor(); } }; const openProjectFile = async () => { - const result = await window.electronAPI.loadProjectFile(); + const result = await nativeBridgeClient.project.loadProjectFile(); if (result.canceled || !result.success) return; await window.electronAPI.switchToEditor(); }; diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 70c38c4c..1c10a645 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -130,6 +130,18 @@ interface SettingsPanelProps { selectedSpeedValue?: PlaybackSpeed | null; onSpeedChange?: (speed: PlaybackSpeed) => void; onSpeedDelete?: (id: string) => void; + // Cursor settings + showCursor?: boolean; + onShowCursorChange?: (show: boolean) => void; + cursorSize?: number; + onCursorSizeChange?: (size: number) => void; + cursorSmoothing?: number; + onCursorSmoothingChange?: (smoothing: number) => void; + cursorMotionBlur?: number; + onCursorMotionBlurChange?: (blur: number) => void; + cursorClickBounce?: number; + onCursorClickBounceChange?: (bounce: number) => void; + hasCursorData?: boolean; } export default SettingsPanel; @@ -194,6 +206,17 @@ export function SettingsPanel({ selectedSpeedValue, onSpeedChange, onSpeedDelete, + showCursor = true, + onShowCursorChange, + cursorSize = 3.0, + onCursorSizeChange, + cursorSmoothing = 0.67, + onCursorSmoothingChange, + cursorMotionBlur = 0.35, + onCursorMotionBlurChange, + cursorClickBounce = 2.5, + onCursorClickBounceChange, + hasCursorData = false, }: SettingsPanelProps) { const [wallpaperPaths, setWallpaperPaths] = useState([]); const [customImages, setCustomImages] = useState([]); @@ -234,8 +257,6 @@ export function SettingsPanel({ const [selectedColor, setSelectedColor] = useState("#ADADAD"); const [gradient, setGradient] = useState(GRADIENTS[0]); - const [showCropModal, setShowCropModal] = useState(false); - const cropSnapshotRef = useRef(null); const [cropAspectLocked, setCropAspectLocked] = useState(false); const [cropAspectRatio, setCropAspectRatio] = useState(""); @@ -335,6 +356,7 @@ export function SettingsPanel({ }, [cropRegion, videoWidth, videoHeight], ); + const [showCropDropdown, setShowCropDropdown] = useState(false); const zoomEnabled = Boolean(selectedZoomDepth); const trimEnabled = Boolean(selectedTrimId); @@ -398,20 +420,6 @@ export function SettingsPanel({ } }; - const handleCropToggle = () => { - if (!showCropModal && cropRegion) { - cropSnapshotRef.current = { ...cropRegion }; - } - setShowCropModal(!showCropModal); - }; - - const handleCropCancel = () => { - if (cropSnapshotRef.current && onCropChange) { - onCropChange(cropSnapshotRef.current); - } - setShowCropModal(false); - }; - // Find selected annotation const selectedAnnotation = selectedAnnotationId ? annotationRegions.find((a) => a.id === selectedAnnotationId) @@ -643,7 +651,7 @@ export function SettingsPanel({