From 4f2828fd436cfb3b784c1195586eebba42a025ac Mon Sep 17 00:00:00 2001 From: Jakub Balinski <31112335+balins@users.noreply.github.com> Date: Thu, 6 Feb 2025 13:58:22 +0100 Subject: [PATCH 1/6] support bidirectional keyboard on iOS, fix cmd+v on Android --- .../vscode-extension/src/common/Project.ts | 1 + .../src/devices/AndroidEmulatorDevice.ts | 6 ++- .../src/devices/DeviceBase.ts | 40 +++++++++++++++-- .../src/devices/IosSimulatorDevice.ts | 44 +++++-------------- .../vscode-extension/src/devices/preview.ts | 6 ++- .../src/project/deviceSession.ts | 8 +++- .../vscode-extension/src/project/project.ts | 11 ++++- .../src/webview/components/Preview.tsx | 11 +++-- 8 files changed, 80 insertions(+), 47 deletions(-) diff --git a/packages/vscode-extension/src/common/Project.ts b/packages/vscode-extension/src/common/Project.ts index fe55c44a7..972d2435d 100644 --- a/packages/vscode-extension/src/common/Project.ts +++ b/packages/vscode-extension/src/common/Project.ts @@ -176,6 +176,7 @@ export interface ProjectInterface { dispatchTouches(touches: Array, type: "Up" | "Move" | "Down"): Promise; dispatchKeyPress(keyCode: number, direction: "Up" | "Down"): Promise; dispatchPaste(text: string): Promise; + dispatchCopy(): Promise; inspectElementAt( xRatio: number, yRatio: number, diff --git a/packages/vscode-extension/src/devices/AndroidEmulatorDevice.ts b/packages/vscode-extension/src/devices/AndroidEmulatorDevice.ts index 7a65ebc31..b2745298b 100644 --- a/packages/vscode-extension/src/devices/AndroidEmulatorDevice.ts +++ b/packages/vscode-extension/src/devices/AndroidEmulatorDevice.ts @@ -565,7 +565,11 @@ export class AndroidEmulatorDevice extends DeviceBase { } async sendBiometricAuthorization(isMatch: boolean) { - // TO DO: implement android biometric authorization + // TODO: implement android biometric authorization + } + + async getClipboard() { + // No need to copy clipboard, Android Emulator syncs it for us whenever a user clicks on 'Copy' } } diff --git a/packages/vscode-extension/src/devices/DeviceBase.ts b/packages/vscode-extension/src/devices/DeviceBase.ts index f12284acf..d0f760490 100644 --- a/packages/vscode-extension/src/devices/DeviceBase.ts +++ b/packages/vscode-extension/src/devices/DeviceBase.ts @@ -6,16 +6,24 @@ import { AppPermissionType, DeviceSettings, TouchPoint } from "../common/Project import { DeviceInfo, DevicePlatform } from "../common/DeviceManager"; import { tryAcquiringLock } from "../utilities/common"; +const LEFT_META_HID_CODE = 0xe3; +const RIGHT_META_HID_CODE = 0xe7; +const V_KEY_HID_CODE = 0x19; +const C_KEY_HID_CODE = 0x06; + export abstract class DeviceBase implements Disposable { protected preview: Preview | undefined; private previewStartPromise: Promise | undefined; private acquired = false; + private pressingLeftMetaKey = false; + private pressingRightMetaKey = false; abstract get lockFilePath(): string; abstract bootDevice(deviceSettings: DeviceSettings): Promise; abstract changeSettings(settings: DeviceSettings): Promise; abstract sendBiometricAuthorization(isMatch: boolean): Promise; + abstract getClipboard(): Promise; abstract installApp(build: BuildResult, forceReinstall: boolean): Promise; abstract launchApp(build: BuildResult, metroPort: number, devtoolsPort: number): Promise; abstract makePreview(): Preview; @@ -96,11 +104,37 @@ export abstract class DeviceBase implements Disposable { } public sendKey(keyCode: number, direction: "Up" | "Down") { - this.preview?.sendKey(keyCode, direction); + // iOS simulator has a buggy behavior when sending cmd+V key combination. + // It sometimes triggers paste action but with a very low success rate. + // Other times it kicks in before the pasteboard is filled with the content + // therefore pasting the previously copied content instead. + // As a temporary workaround, we disable sending cmd+V as key combination + // entirely to prevent this buggy behavior. Users can still paste content + // using the context menu method as they'd do on an iOS device. + // This is not an ideal workaround as people may still trigger cmd+v by + // pressing V first and then cmd, but it is good enough to filter out + // the majority of the noisy behavior since typically you press cmd first. + // Similarly, when pasting into Android Emulator, cmd+V has results in a + // side effect of typing the letter "v" into the text field (the same + // applies to cmd+C). + if (keyCode === LEFT_META_HID_CODE) { + this.pressingLeftMetaKey = direction === "Down"; + } else if (keyCode === RIGHT_META_HID_CODE) { + this.pressingRightMetaKey = direction === "Down"; + } + + if ( + (this.pressingLeftMetaKey || this.pressingRightMetaKey) && + (keyCode === C_KEY_HID_CODE || keyCode === V_KEY_HID_CODE) + ) { + // ignore sending C and V when meta key is pressed + } else { + this.preview?.sendKey(keyCode, direction); + } } - public async sendPaste(text: string) { - return this.preview?.sendPaste(text); + public async sendClipboard(text: string) { + return this.preview?.sendClipboard(text); } async startPreview() { diff --git a/packages/vscode-extension/src/devices/IosSimulatorDevice.ts b/packages/vscode-extension/src/devices/IosSimulatorDevice.ts index f6906d42c..9807002f5 100644 --- a/packages/vscode-extension/src/devices/IosSimulatorDevice.ts +++ b/packages/vscode-extension/src/devices/IosSimulatorDevice.ts @@ -14,10 +14,6 @@ import { AppPermissionType, DeviceSettings, Locale } from "../common/Project"; import { EXPO_GO_BUNDLE_ID, fetchExpoLaunchDeeplink } from "../builders/expoGo"; import { IOSBuildResult } from "../builders/buildIOS"; -const LEFT_META_HID_CODE = 0xe3; -const RIGHT_META_HID_CODE = 0xe7; -const V_KEY_HID_CODE = 0x19; - interface SimulatorInfo { availability?: string; state?: string; @@ -218,44 +214,24 @@ export class IosSimulatorDevice extends DeviceBase { ]); } - private pressingLeftMetaKey = false; - private pressingRightMetaKey = false; - - public sendKey(keyCode: number, direction: "Up" | "Down"): void { - // iOS simulator has a buggy behavior when sending cmd+V key combination. - // It sometimes triggers paste action but with a very low success rate. - // Other times it kicks in before the pasteboard is filled with the content - // therefore pasting the previously compied content instead. - // As a temporary workaround, we disable sending cmd+V as key combination - // entirely to prevent this buggy behavior. Users can still paste content - // using the context menu method as they'd do on an iOS device. - // This is not an ideal workaround as people may still trigger cmd+v by - // pressing V first and then cmd, but it is good enough to filter out - // the majority of the noisy behavior since typically you press cmd first. - if (keyCode === LEFT_META_HID_CODE) { - this.pressingLeftMetaKey = direction === "Down"; - } else if (keyCode === RIGHT_META_HID_CODE) { - this.pressingRightMetaKey = direction === "Down"; - } - if ((this.pressingLeftMetaKey || this.pressingRightMetaKey) && keyCode === V_KEY_HID_CODE) { - // ignore sending V when meta key is pressed - } else { - this.preview?.sendKey(keyCode, direction); - } + public async sendClipboard(text: string) { + const deviceSetLocation = getOrCreateDeviceSet(this.deviceUDID); + await exec("xcrun", ["simctl", "--set", deviceSetLocation, "pbcopy", this.deviceUDID], { + input: text, + }); } - public async sendPaste(text: string) { + public async getClipboard() { const deviceSetLocation = getOrCreateDeviceSet(this.deviceUDID); - const subprocess = exec("xcrun", [ + const { stdout } = await exec("xcrun", [ "simctl", "--set", deviceSetLocation, - "pbcopy", + "pbpaste", this.deviceUDID, ]); - subprocess.stdin?.write(text); - subprocess.stdin?.end(); - await subprocess; + + return stdout; } private async changeLocale(newLocale: Locale): Promise { diff --git a/packages/vscode-extension/src/devices/preview.ts b/packages/vscode-extension/src/devices/preview.ts index 02688c6ac..b61a3d86c 100644 --- a/packages/vscode-extension/src/devices/preview.ts +++ b/packages/vscode-extension/src/devices/preview.ts @@ -191,7 +191,9 @@ export class Preview implements Disposable { this.subprocess?.stdin?.write(`key ${direction} ${keyCode}\n`); } - public async sendPaste(text: string) { - this.subprocess?.stdin?.write(`paste ${text}\n`); + public sendClipboard(text: string) { + // This is bad, but prevents simulator server going crazy with multiline pastes + // If we want to support multiline pastes we need to change the communication protocol + this.subprocess?.stdin?.write(`paste ${text.replace(/(?:\r\n|\r|\n)/g, " ")}\n`); } } diff --git a/packages/vscode-extension/src/project/deviceSession.ts b/packages/vscode-extension/src/project/deviceSession.ts index 14dad1b96..347f6b959 100644 --- a/packages/vscode-extension/src/project/deviceSession.ts +++ b/packages/vscode-extension/src/project/deviceSession.ts @@ -310,8 +310,12 @@ export class DeviceSession implements Disposable { this.device.sendKey(keyCode, direction); } - public sendPaste(text: string) { - return this.device.sendPaste(text); + public sendClipboard(text: string) { + return this.device.sendClipboard(text); + } + + public async getClipboard() { + return this.device.getClipboard(); } public inspectElementAt( diff --git a/packages/vscode-extension/src/project/project.ts b/packages/vscode-extension/src/project/project.ts index def5fc7a1..70facafd4 100644 --- a/packages/vscode-extension/src/project/project.ts +++ b/packages/vscode-extension/src/project/project.ts @@ -1,6 +1,6 @@ import { EventEmitter } from "stream"; import os from "os"; -import { Disposable, commands, workspace, window, DebugSessionCustomEvent } from "vscode"; +import { env, Disposable, commands, workspace, window, DebugSessionCustomEvent } from "vscode"; import _ from "lodash"; import stripAnsi from "strip-ansi"; import { minimatch } from "minimatch"; @@ -253,7 +253,14 @@ export class Project //#endregion async dispatchPaste(text: string) { - await this.deviceSession?.sendPaste(text); + await this.deviceSession?.sendClipboard(text); + } + + async dispatchCopy() { + const text = await this.deviceSession?.getClipboard(); + if (text) { + env.clipboard.writeText(text); + } } onBundleError(): void { diff --git a/packages/vscode-extension/src/webview/components/Preview.tsx b/packages/vscode-extension/src/webview/components/Preview.tsx index 3e71e54e1..01df876dc 100644 --- a/packages/vscode-extension/src/webview/components/Preview.tsx +++ b/packages/vscode-extension/src/webview/components/Preview.tsx @@ -347,19 +347,24 @@ function Preview({ }, []); useEffect(() => { - function dispatchPaste(e: ClipboardEvent) { + function synchronizeClipboard(e: ClipboardEvent) { if (document.activeElement === wrapperDivRef.current) { e.preventDefault(); const text = e.clipboardData?.getData("text"); if (text) { project.dispatchPaste(text); + } else { + project.dispatchCopy(); } } } - addEventListener("paste", dispatchPaste); + + addEventListener("paste", synchronizeClipboard); + addEventListener("copy", synchronizeClipboard); return () => { - removeEventListener("paste", dispatchPaste); + removeEventListener("paste", synchronizeClipboard); + removeEventListener("copy", synchronizeClipboard); }; }, [project]); From 39b759212a35810563a97ee3dfe65179eb77efe4 Mon Sep 17 00:00:00 2001 From: Jakub Balinski <31112335+balins@users.noreply.github.com> Date: Mon, 10 Feb 2025 15:34:04 +0100 Subject: [PATCH 2/6] handle multiline pastes --- packages/vscode-extension/src/devices/preview.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/vscode-extension/src/devices/preview.ts b/packages/vscode-extension/src/devices/preview.ts index b61a3d86c..7c58a4fba 100644 --- a/packages/vscode-extension/src/devices/preview.ts +++ b/packages/vscode-extension/src/devices/preview.ts @@ -192,8 +192,7 @@ export class Preview implements Disposable { } public sendClipboard(text: string) { - // This is bad, but prevents simulator server going crazy with multiline pastes - // If we want to support multiline pastes we need to change the communication protocol - this.subprocess?.stdin?.write(`paste ${text.replace(/(?:\r\n|\r|\n)/g, " ")}\n`); + // We use markers for start and end of the paste to handle multi-line pastes + this.subprocess?.stdin?.write(`paste START-SIMSERVER-PASTE>>>${text}<< Date: Tue, 11 Feb 2025 11:15:21 +0100 Subject: [PATCH 3/6] show notification on copy/paste --- .../vscode-extension/src/project/project.ts | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/vscode-extension/src/project/project.ts b/packages/vscode-extension/src/project/project.ts index 70facafd4..f23f2fc28 100644 --- a/packages/vscode-extension/src/project/project.ts +++ b/packages/vscode-extension/src/project/project.ts @@ -1,6 +1,14 @@ import { EventEmitter } from "stream"; import os from "os"; -import { env, Disposable, commands, workspace, window, DebugSessionCustomEvent } from "vscode"; +import { + env, + Disposable, + commands, + workspace, + window, + DebugSessionCustomEvent, + ProgressLocation, +} from "vscode"; import _ from "lodash"; import stripAnsi from "strip-ansi"; import { minimatch } from "minimatch"; @@ -252,14 +260,30 @@ export class Project //#endregion + async showToast(message: string, timeout: number) { + // VSCode doesn't support auto hiding notifications, so we use a workaround with progress + await window.withProgress( + { + location: ProgressLocation.Notification, + cancellable: false, + }, + async (progress) => { + progress.report({ message, increment: 100 }); + await new Promise((resolve) => setTimeout(resolve, timeout)); + } + ); + } + async dispatchPaste(text: string) { await this.deviceSession?.sendClipboard(text); + await this.showToast("Pasted to device clipboard", 2000); } async dispatchCopy() { const text = await this.deviceSession?.getClipboard(); if (text) { env.clipboard.writeText(text); + await this.showToast("Copied from device clipboard", 2000); } } From 9d2b9c6b2e241b5c74d5c21cdeaeaa9c491c9ecd Mon Sep 17 00:00:00 2001 From: Jakub Balinski <31112335+balins@users.noreply.github.com> Date: Tue, 11 Feb 2025 12:18:55 +0100 Subject: [PATCH 4/6] update simulator server to handle multiline paste --- packages/simulator-server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/simulator-server b/packages/simulator-server index ee1d3bae9..3a303934b 160000 --- a/packages/simulator-server +++ b/packages/simulator-server @@ -1 +1 @@ -Subproject commit ee1d3bae94006e9efcb688667f06ebe531b5e61d +Subproject commit 3a303934ba0ed91194cb846448de398c692f6ffc From 16873fa433998e0be7d947bb2ff438fb73c3898f Mon Sep 17 00:00:00 2001 From: Jakub Balinski <31112335+balins@users.noreply.github.com> Date: Wed, 12 Feb 2025 10:24:46 +0100 Subject: [PATCH 5/6] move `showToast` to `utils` --- packages/vscode-extension/src/common/utils.ts | 2 ++ .../vscode-extension/src/project/project.ts | 28 ++----------------- .../vscode-extension/src/utilities/utils.ts | 16 ++++++++++- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/packages/vscode-extension/src/common/utils.ts b/packages/vscode-extension/src/common/utils.ts index a0251fc1f..f14271184 100644 --- a/packages/vscode-extension/src/common/utils.ts +++ b/packages/vscode-extension/src/common/utils.ts @@ -39,4 +39,6 @@ export interface UtilsInterface { eventType: K, listener: UtilsEventListener ): Promise; + + showToast(message: string, timeout: number): Promise; } diff --git a/packages/vscode-extension/src/project/project.ts b/packages/vscode-extension/src/project/project.ts index f23f2fc28..6b7d8ddda 100644 --- a/packages/vscode-extension/src/project/project.ts +++ b/packages/vscode-extension/src/project/project.ts @@ -1,14 +1,6 @@ import { EventEmitter } from "stream"; import os from "os"; -import { - env, - Disposable, - commands, - workspace, - window, - DebugSessionCustomEvent, - ProgressLocation, -} from "vscode"; +import { env, Disposable, commands, workspace, window, DebugSessionCustomEvent } from "vscode"; import _ from "lodash"; import stripAnsi from "strip-ansi"; import { minimatch } from "minimatch"; @@ -260,30 +252,16 @@ export class Project //#endregion - async showToast(message: string, timeout: number) { - // VSCode doesn't support auto hiding notifications, so we use a workaround with progress - await window.withProgress( - { - location: ProgressLocation.Notification, - cancellable: false, - }, - async (progress) => { - progress.report({ message, increment: 100 }); - await new Promise((resolve) => setTimeout(resolve, timeout)); - } - ); - } - async dispatchPaste(text: string) { await this.deviceSession?.sendClipboard(text); - await this.showToast("Pasted to device clipboard", 2000); + await this.utils.showToast("Pasted to device clipboard", 2000); } async dispatchCopy() { const text = await this.deviceSession?.getClipboard(); if (text) { env.clipboard.writeText(text); - await this.showToast("Copied from device clipboard", 2000); + await this.utils.showToast("Copied from device clipboard", 2000); } } diff --git a/packages/vscode-extension/src/utilities/utils.ts b/packages/vscode-extension/src/utilities/utils.ts index 67255264e..677ac5912 100644 --- a/packages/vscode-extension/src/utilities/utils.ts +++ b/packages/vscode-extension/src/utilities/utils.ts @@ -2,7 +2,7 @@ import { homedir } from "node:os"; import { EventEmitter } from "stream"; import fs from "fs"; import path from "path"; -import { commands, env, Uri, window } from "vscode"; +import { commands, env, ProgressLocation, Uri, window } from "vscode"; import JSON5 from "json5"; import vscode from "vscode"; import { TelemetryEventProperties } from "@vscode/extension-telemetry"; @@ -174,4 +174,18 @@ export class Utils implements UtilsInterface { ) { this.eventEmitter.removeListener(eventType, listener); } + + async showToast(message: string, timeout: number) { + // VSCode doesn't support auto hiding notifications, so we use a workaround with progress + await window.withProgress( + { + location: ProgressLocation.Notification, + cancellable: false, + }, + async (progress) => { + progress.report({ message, increment: 100 }); + await new Promise((resolve) => setTimeout(resolve, timeout)); + } + ); + } } From 12c02d7b1339f2c78c84592b9abb4600003c8c54 Mon Sep 17 00:00:00 2001 From: Jakub Balinski <31112335+balins@users.noreply.github.com> Date: Wed, 12 Feb 2025 10:25:49 +0100 Subject: [PATCH 6/6] show "copied" toast despite Android clipboard is automatically synced --- packages/vscode-extension/src/project/project.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/vscode-extension/src/project/project.ts b/packages/vscode-extension/src/project/project.ts index 6b7d8ddda..d83c34a0a 100644 --- a/packages/vscode-extension/src/project/project.ts +++ b/packages/vscode-extension/src/project/project.ts @@ -261,8 +261,9 @@ export class Project const text = await this.deviceSession?.getClipboard(); if (text) { env.clipboard.writeText(text); - await this.utils.showToast("Copied from device clipboard", 2000); } + // For consistency between iOS and Android, we always display toast message + await this.utils.showToast("Copied from device clipboard", 2000); } onBundleError(): void {