diff --git a/.changeset/calm-codecs-stream.md b/.changeset/calm-codecs-stream.md new file mode 100644 index 00000000..6bc93bdb --- /dev/null +++ b/.changeset/calm-codecs-stream.md @@ -0,0 +1,5 @@ +--- +'expo-device-hub': minor +--- + +Add selectable H.264, VP8, and VP9 host encoding for Android emulator gRPC WebSocket streams. diff --git a/README.md b/README.md index dff05ee2..8db7429b 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,8 @@ registers the plugin against itself, so the standalone app still gets the real [`@expo/hub-client`](packages/@expo/hub-client) is the **device-client layer**. The two backends speak very different wire protocols — serve-sim streams MJPEG/H.264 and -takes binary touch packets, while serve-emu streams H.264 (WebCodecs) and takes JSON -gestures — so this package hides that behind one shared contract: +takes binary touch packets, while serve-emu streams H.264, VP8, or VP9 through +WebCodecs and takes JSON gestures — so this package hides that behind one shared contract: - a hook (`useIosDeviceClient` / `useAndroidDeviceClient` and general `useActiveDeviceClient`) that owns the WebSocket connection and exposes the live diff --git a/packages/@expo/hub-client/src/__tests__/android-stream-source.test.ts b/packages/@expo/hub-client/src/__tests__/android-stream-source.test.ts index ec4ee357..604acff7 100644 --- a/packages/@expo/hub-client/src/__tests__/android-stream-source.test.ts +++ b/packages/@expo/hub-client/src/__tests__/android-stream-source.test.ts @@ -1,6 +1,9 @@ import { describe, expect, test } from 'bun:test'; -import { parseAndroidStreamSource } from '../android-stream-source'; +import { + androidStreamSourceSupportsWebRtc, + parseAndroidStreamSource, +} from '../android-stream-source'; describe('parseAndroidStreamSource', () => { test('parses the authoritative gRPC image mode', () => { @@ -11,6 +14,7 @@ describe('parseAndroidStreamSource', () => { grpcImageMode: 'mmap', inputSource: 'scrcpy', availableInputSources: ['scrcpy', 'grpc'], + grpcVideoCodec: 'vp9', availableModes: ['scrcpy', 'grpc-screenshot'], sessionGeneration: 4, }), @@ -19,11 +23,74 @@ describe('parseAndroidStreamSource', () => { grpcImageMode: 'mmap', inputSource: 'scrcpy', availableInputSources: ['scrcpy', 'grpc'], + grpcVideoCodec: 'vp9', availableModes: ['scrcpy', 'grpc-screenshot'], sessionGeneration: 4, }); }); + test('uses WebRTC only for sources backed by H.264', () => { + expect(androidStreamSourceSupportsWebRtc(null)).toBeTrue(); + expect( + androidStreamSourceSupportsWebRtc({ + mode: 'scrcpy', + grpcImageMode: 'png', + inputSource: 'scrcpy', + availableInputSources: ['scrcpy'], + grpcVideoCodec: 'vp9', + availableModes: ['scrcpy', 'grpc-screenshot'], + sessionGeneration: 1, + }), + ).toBeTrue(); + expect( + androidStreamSourceSupportsWebRtc({ + mode: 'grpc-screenshot', + grpcImageMode: 'mmap', + inputSource: 'scrcpy', + availableInputSources: ['scrcpy', 'grpc'], + grpcVideoCodec: 'h264', + availableModes: ['scrcpy', 'grpc-screenshot'], + sessionGeneration: 2, + }), + ).toBeTrue(); + expect( + androidStreamSourceSupportsWebRtc({ + mode: 'grpc-screenshot', + grpcImageMode: 'mmap', + inputSource: 'scrcpy', + availableInputSources: ['scrcpy', 'grpc'], + grpcVideoCodec: 'vp8', + availableModes: ['scrcpy', 'grpc-screenshot'], + sessionGeneration: 3, + }), + ).toBeFalse(); + expect( + androidStreamSourceSupportsWebRtc({ + mode: 'grpc-screenshot', + grpcImageMode: 'mmap', + inputSource: 'grpc', + availableInputSources: ['scrcpy', 'grpc'], + grpcVideoCodec: 'vp9', + availableModes: ['scrcpy', 'grpc-screenshot'], + sessionGeneration: 4, + }), + ).toBeFalse(); + }); + + test('defaults legacy responses without a video codec to H.264', () => { + expect( + parseAndroidStreamSource({ + ok: true, + mode: 'grpc-screenshot', + grpcImageMode: 'png', + inputSource: 'scrcpy', + availableInputSources: ['scrcpy', 'grpc'], + availableModes: ['scrcpy', 'grpc-screenshot'], + sessionGeneration: 1, + }), + ).toMatchObject({ grpcVideoCodec: 'h264' }); + }); + test('rejects missing or unsupported image modes', () => { const response = { ok: true, @@ -35,6 +102,13 @@ describe('parseAndroidStreamSource', () => { }; expect(parseAndroidStreamSource(response)).toBeNull(); expect(parseAndroidStreamSource({ ...response, grpcImageMode: 'rgb' })).toBeNull(); + expect( + parseAndroidStreamSource({ + ...response, + grpcImageMode: 'png', + grpcVideoCodec: 'av1', + }), + ).toBeNull(); }); test('rejects unavailable or unsupported input sources', () => { diff --git a/packages/@expo/hub-client/src/__tests__/android-stream.test.ts b/packages/@expo/hub-client/src/__tests__/android-stream.test.ts index 507f70ea..8a4cffe7 100644 --- a/packages/@expo/hub-client/src/__tests__/android-stream.test.ts +++ b/packages/@expo/hub-client/src/__tests__/android-stream.test.ts @@ -8,7 +8,21 @@ import { androidStreamSourceErrorMessage, parseAndroidStreamSource, } from '../android-stream-source'; -import { androidWsUrlFor, parseServeEmuStreamSettings } from '../useAndroidDevice'; +import { + fixedWebCodecsCodec, + isRawVideoKeyFrame, + isWebCodecsUnsupportedError, + mseFallbackCodecError, + parseAndroidVideoSession, + resolveVideoKeyFrame, + webCodecsCodec, +} from '../android-video-codec'; +import { parseFramePacket } from '../h264'; +import { + androidWsUrlFor, + parseServeEmuStreamSettings, + parseServeEmuViewerTransports, +} from '../useAndroidDevice'; describe('serve-emu stream contract', () => { test('uses a metadata video socket for H.264 and an input-only socket for WebRTC', () => { @@ -20,6 +34,78 @@ describe('serve-emu stream contract', () => { ); }); + test('parses authoritative codec generation boundaries for WebSocket video', () => { + expect( + parseAndroidVideoSession({ + type: 'video-session', + size: { width: 1080, height: 2400 }, + codec: 'vp9', + }), + ).toEqual({ size: { width: 1080, height: 2400 }, codec: 'vp9' }); + expect( + parseAndroidVideoSession({ + type: 'video-session', + size: { width: 1080, height: 2400 }, + }), + ).toEqual({ size: { width: 1080, height: 2400 }, codec: 'h264' }); + expect( + parseAndroidVideoSession({ + type: 'video-session', + size: { width: 0, height: 2400 }, + codec: 'vp8', + }), + ).toBeNull(); + expect(fixedWebCodecsCodec('h264')).toBeNull(); + expect(fixedWebCodecsCodec('vp8')).toBe('vp8'); + expect(fixedWebCodecsCodec('vp9')).toBe('vp09.00.10.08'); + expect(webCodecsCodec('h264', Uint8Array.of(0x67, 0x64, 0, 0x29))).toBe('avc1.640029'); + expect(webCodecsCodec('h264', Uint8Array.of(0x67, 0x64))).toBeNull(); + expect(mseFallbackCodecError('h264')).toBeNull(); + expect(mseFallbackCodecError('h264', false)).toBe( + 'This browser cannot decode H.264 (WebCodecs unavailable).', + ); + expect(mseFallbackCodecError('vp8')).toBe( + 'VP8 WebSocket video requires WebCodecs.', + ); + expect(isWebCodecsUnsupportedError({ name: 'NotSupportedError' })).toBe(true); + expect(isWebCodecsUnsupportedError(new Error('decode failed'))).toBe(false); + }); + + test('recovers VPx keyframes from raw payloads only when SEMU metadata is absent', () => { + const vp8Key = Uint8Array.of(0xf0, 0x02, 0, 0x9d, 0x01, 0x2a, 0x10, 0, 0x10, 0); + const vp9Key = Uint8Array.of(0x82, 0x49, 0x83, 0x42); + + expect(isRawVideoKeyFrame('vp8', vp8Key)).toBe(true); + expect(isRawVideoKeyFrame('vp8', Uint8Array.of(0xb1, 1, 0, 5))).toBe(false); + expect(isRawVideoKeyFrame('vp9', vp9Key)).toBe(true); + expect(isRawVideoKeyFrame('vp9', Uint8Array.of(0x82, 0, 0, 0))).toBe(false); + expect(resolveVideoKeyFrame('vp8', false, vp8Key)).toBe(false); + expect(resolveVideoKeyFrame('vp9', true, Uint8Array.of(1, 2, 3))).toBe(true); + expect(resolveVideoKeyFrame('vp8', null, vp8Key)).toBe(true); + }); + + test('reads keyframe and PTS metadata from SEMU v1 and v2 packets', () => { + const packet = (version: 1 | 2, payload: number[]) => { + const headerBytes = version === 1 ? 16 : 24; + const bytes = new Uint8Array(headerBytes + payload.length); + const view = new DataView(bytes.buffer); + view.setUint32(0, 0x53454d55, false); + view.setUint8(4, version); + view.setUint8(5, 1); + view.setBigUint64(8, 123_456n, false); + if (version === 2) view.setBigUint64(16, 987_654_321n, false); + bytes.set(payload, headerBytes); + return bytes; + }; + + for (const version of [1, 2] as const) { + const parsed = parseFramePacket(packet(version, [1, 2, 3])); + expect(parsed.isKey).toBe(true); + expect(parsed.timestamp).toBe(123_456); + expect([...parsed.data]).toEqual([1, 2, 3]); + } + }); + test('accepts the host-owned H.264 WebRTC configuration', () => { expect( parseServeEmuStreamSettings({ @@ -50,6 +136,29 @@ describe('serve-emu stream contract', () => { }); }); + test('disables WebRTC when the active source codec only supports WebSocket viewing', () => { + expect( + parseServeEmuViewerTransports({ + default: 'websocket', + available: ['websocket'], + webrtc: null, + }), + ).toEqual({ transport: 'websocket' }); + expect( + parseServeEmuViewerTransports({ + default: 'webrtc', + available: ['websocket', 'webrtc'], + webrtc: { + transport: 'webrtc', + codec: 'h264', + iceServers: [{ urls: ['stun:stun.example.test:3478'] }], + iceTransportPolicy: 'all', + }, + }), + ).toMatchObject({ transport: 'webrtc', codec: 'h264' }); + expect(parseServeEmuViewerTransports({ available: ['webtransport'] })).toBeNull(); + }); + test('rejects unsupported codecs and malformed ICE settings', () => { expect(parseServeEmuStreamSettings({ transport: 'websocket' })).toEqual({ transport: 'websocket', @@ -130,6 +239,7 @@ describe('serve-emu capture source contract', () => { grpcImageMode: 'mmap', inputSource: 'scrcpy', availableInputSources: ['scrcpy', 'grpc'], + grpcVideoCodec: 'vp8', availableModes: ['scrcpy', 'grpc-screenshot'], sessionGeneration: 2, }), @@ -138,6 +248,7 @@ describe('serve-emu capture source contract', () => { grpcImageMode: 'mmap', inputSource: 'scrcpy', availableInputSources: ['scrcpy', 'grpc'], + grpcVideoCodec: 'vp8', availableModes: ['scrcpy', 'grpc-screenshot'], sessionGeneration: 2, }); @@ -157,6 +268,7 @@ describe('serve-emu capture source contract', () => { grpcImageMode: 'png', inputSource: 'scrcpy', availableInputSources: ['scrcpy'], + grpcVideoCodec: 'h264', availableModes: ['scrcpy'], sessionGeneration: 0, }); diff --git a/packages/@expo/hub-client/src/android-stream-source.ts b/packages/@expo/hub-client/src/android-stream-source.ts index 1d1912a6..f29e0943 100644 --- a/packages/@expo/hub-client/src/android-stream-source.ts +++ b/packages/@expo/hub-client/src/android-stream-source.ts @@ -1,5 +1,6 @@ import { type DeviceGrpcImageMode, + type DeviceGrpcVideoCodec, type DeviceInputSource, type DeviceStreamSource, type DeviceStreamSourceStatus, @@ -39,6 +40,17 @@ function isInputSource(value: unknown): value is DeviceInputSource { return value === 'scrcpy' || value === 'grpc'; } +function isGrpcVideoCodec(value: unknown): value is DeviceGrpcVideoCodec { + return value === 'h264' || value === 'vp8' || value === 'vp9'; +} + +/** Whether the active source can feed serve-emu's H.264-only WebRTC publisher. */ +export function androidStreamSourceSupportsWebRtc( + source: DeviceStreamSourceStatus | null, +): boolean { + return source?.mode !== 'grpc-screenshot' || source.grpcVideoCodec === 'h264'; +} + /** Parse serve-emu's authoritative device-scoped `/api/stream-mode` response. */ export function parseAndroidStreamSource(value: unknown): DeviceStreamSourceStatus | null { if (!value || typeof value !== 'object' || Array.isArray(value)) return null; @@ -47,7 +59,8 @@ export function parseAndroidStreamSource(value: unknown): DeviceStreamSourceStat candidate.ok !== true || !isAndroidStreamSource(candidate.mode) || !isGrpcImageMode(candidate.grpcImageMode) || - !isInputSource(candidate.inputSource) + !isInputSource(candidate.inputSource) || + (candidate.grpcVideoCodec !== undefined && !isGrpcVideoCodec(candidate.grpcVideoCodec)) ) { return null; } @@ -81,6 +94,8 @@ export function parseAndroidStreamSource(value: unknown): DeviceStreamSourceStat grpcImageMode: candidate.grpcImageMode, inputSource: candidate.inputSource, availableInputSources: candidate.availableInputSources, + // Older serve-emu versions always emitted H.264 and omit this field. + grpcVideoCodec: candidate.grpcVideoCodec ?? 'h264', availableModes: candidate.availableModes, sessionGeneration: candidate.sessionGeneration, }; diff --git a/packages/@expo/hub-client/src/android-video-codec.ts b/packages/@expo/hub-client/src/android-video-codec.ts new file mode 100644 index 00000000..9b9b3be7 --- /dev/null +++ b/packages/@expo/hub-client/src/android-video-codec.ts @@ -0,0 +1,139 @@ +import { type DeviceGrpcVideoCodec } from './types'; +import { buildCodecString, scanAU } from './h264'; + +export type AndroidVideoSession = { + size: { width: number; height: number }; + codec: DeviceGrpcVideoCodec; +}; + +export function isDeviceGrpcVideoCodec(value: unknown): value is DeviceGrpcVideoCodec { + return value === 'h264' || value === 'vp8' || value === 'vp9'; +} + +/** Parse serve-emu's generation boundary before accepting encoded WebSocket frames. */ +export function parseAndroidVideoSession(value: unknown): AndroidVideoSession | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const candidate = value as Record; + if ( + candidate.type !== 'video-session' || + (candidate.codec !== undefined && !isDeviceGrpcVideoCodec(candidate.codec)) + ) { + return null; + } + if (!candidate.size || typeof candidate.size !== 'object' || Array.isArray(candidate.size)) { + return null; + } + const size = candidate.size as Record; + if ( + typeof size.width !== 'number' || + !Number.isFinite(size.width) || + size.width <= 0 || + typeof size.height !== 'number' || + !Number.isFinite(size.height) || + size.height <= 0 + ) { + return null; + } + return { + size: { width: size.width, height: size.height }, + // Older serve-emu releases emitted H.264 and omitted the codec field. + codec: candidate.codec ?? 'h264', + }; +} + +/** VPx codec strings accepted by WebCodecs; H.264 is derived from its SPS instead. */ +export function fixedWebCodecsCodec(codec: DeviceGrpcVideoCodec): string | null { + if (codec === 'vp8') return 'vp8'; + if (codec === 'vp9') return 'vp09.00.10.08'; + return null; +} + +/** Resolve H.264's in-band profile or a VPx codec's fixed WebCodecs identifier. */ +export function webCodecsCodec( + codec: DeviceGrpcVideoCodec, + h264Sps: Uint8Array | null = null, +): string | null { + if (codec !== 'h264') return fixedWebCodecsCodec(codec); + return h264Sps && h264Sps.byteLength >= 4 ? buildCodecString(h264Sps) : null; +} + +export function grpcVideoCodecLabel(codec: DeviceGrpcVideoCodec): string { + if (codec === 'h264') return 'H.264'; + return codec.toUpperCase(); +} + +/** Report why a codec cannot use the browser's non-WebCodecs fallback. */ +export function mseFallbackCodecError( + codec: DeviceGrpcVideoCodec, + mseSupported = true, +): string | null { + if (codec !== 'h264') { + return `${grpcVideoCodecLabel(codec)} WebSocket video requires WebCodecs.`; + } + return mseSupported ? null : 'This browser cannot decode H.264 (WebCodecs unavailable).'; +} + +export function isWebCodecsUnsupportedError(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'name' in error && + error.name === 'NotSupportedError' + ); +} + +class BitReader { + #offset = 0; + + constructor(private readonly bytes: Uint8Array) {} + + read(bits: number): number | null { + if (!Number.isSafeInteger(bits) || bits < 0 || bits > 31) return null; + if (this.#offset + bits > this.bytes.byteLength * 8) return null; + let value = 0; + for (let index = 0; index < bits; index++) { + const position = this.#offset++; + value = value * 2 + ((this.bytes[position >> 3]! >> (7 - (position & 7))) & 1); + } + return value; + } +} + +function isRawVp8KeyFrame(data: Uint8Array): boolean { + return ( + data.byteLength >= 10 && + (data[0]! & 1) === 0 && + data[3] === 0x9d && + data[4] === 0x01 && + data[5] === 0x2a + ); +} + +function isRawVp9KeyFrame(data: Uint8Array): boolean { + const bits = new BitReader(data); + if (bits.read(2) !== 0b10) return false; + const profileLow = bits.read(1); + const profileHigh = bits.read(1); + if (profileLow === null || profileHigh === null) return false; + const profile = profileLow | (profileHigh << 1); + if (profile === 3 && bits.read(1) !== 0) return false; + if (bits.read(1) !== 0 || bits.read(1) !== 0) return false; + if (bits.read(1) === null || bits.read(1) === null) return false; + return bits.read(24) === 0x498342; +} + +/** Detect a keyframe when legacy/raw video arrives without SEMU metadata. */ +export function isRawVideoKeyFrame(codec: DeviceGrpcVideoCodec, data: Uint8Array): boolean { + if (codec === 'h264') return scanAU(data).isKey; + if (codec === 'vp8') return isRawVp8KeyFrame(data); + return isRawVp9KeyFrame(data); +} + +/** SEMU's key bit is authoritative; inspect bytes only for metadata-free frames. */ +export function resolveVideoKeyFrame( + codec: DeviceGrpcVideoCodec, + metadataKey: boolean | null, + data: Uint8Array, +): boolean { + return metadataKey ?? isRawVideoKeyFrame(codec, data); +} diff --git a/packages/@expo/hub-client/src/h264.ts b/packages/@expo/hub-client/src/h264.ts index f5269ec6..69ac16dc 100644 --- a/packages/@expo/hub-client/src/h264.ts +++ b/packages/@expo/hub-client/src/h264.ts @@ -42,11 +42,11 @@ export function scanAU(buf: Uint8Array): ScanResult { return { isKey, spsBytes }; } -// serve-emu prefixes each WS frame with a 16-byte "SEMU" header carrying the -// keyframe flag + presentation timestamp (see server.ts `withFrameMeta`). +// serve-emu prefixes each WS frame with a versioned "SEMU" header carrying the +// keyframe flag + presentation timestamp (see shared/frame-meta.ts). const FRAME_META_MAGIC = 0x53454d55; // "SEMU" -const FRAME_META_VERSION = 1; -const FRAME_META_HEADER_BYTES = 16; +const FRAME_META_V1_HEADER_BYTES = 16; +const FRAME_META_V2_HEADER_BYTES = 24; const FRAME_FLAG_KEY = 1 << 0; export interface FramePacket { @@ -55,15 +55,25 @@ export interface FramePacket { timestamp: number | null; } -/** Split a `/ws?frame-meta=1` binary message into its meta + Annex-B payload. */ +/** Split a `/ws?frame-meta=1` binary message into its metadata and encoded payload. */ export function parseFramePacket(raw: ArrayBuffer | Uint8Array): FramePacket { const bytes = raw instanceof Uint8Array ? raw : new Uint8Array(raw); - if (bytes.byteLength > FRAME_META_HEADER_BYTES) { - const view = new DataView(bytes.buffer, bytes.byteOffset, FRAME_META_HEADER_BYTES); - if (view.getUint32(0, false) === FRAME_META_MAGIC && view.getUint8(4) === FRAME_META_VERSION) { + if (bytes.byteLength > FRAME_META_V1_HEADER_BYTES) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if (view.getUint32(0, false) === FRAME_META_MAGIC) { + const version = view.getUint8(4); + const headerBytes = + version === 1 + ? FRAME_META_V1_HEADER_BYTES + : version === 2 && bytes.byteLength > FRAME_META_V2_HEADER_BYTES + ? FRAME_META_V2_HEADER_BYTES + : null; + if (headerBytes === null || bytes.byteLength <= headerBytes) { + return { data: bytes, isKey: null, timestamp: null }; + } const pts = view.getBigUint64(8, false); return { - data: bytes.subarray(FRAME_META_HEADER_BYTES), + data: bytes.subarray(headerBytes), isKey: (view.getUint8(5) & FRAME_FLAG_KEY) !== 0, timestamp: pts <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(pts) : null, }; diff --git a/packages/@expo/hub-client/src/types.ts b/packages/@expo/hub-client/src/types.ts index d7b17986..f85dcf63 100644 --- a/packages/@expo/hub-client/src/types.ts +++ b/packages/@expo/hub-client/src/types.ts @@ -155,12 +155,16 @@ export type DeviceGrpcImageMode = 'png' | 'mmap'; /** Input transport used while gRPC provides emulator video. */ export type DeviceInputSource = 'scrcpy' | 'grpc'; +/** Host-side video encoder used by the emulator gRPC screenshot source. */ +export type DeviceGrpcVideoCodec = 'h264' | 'vp8' | 'vp9'; + /** Authoritative source state for the selected Android device session. */ export interface DeviceStreamSourceStatus { mode: DeviceStreamSource; grpcImageMode: DeviceGrpcImageMode; inputSource: DeviceInputSource; availableInputSources: readonly DeviceInputSource[]; + grpcVideoCodec: DeviceGrpcVideoCodec; availableModes: readonly DeviceStreamSource[]; sessionGeneration: number; } @@ -475,6 +479,8 @@ export interface DeviceClient { setGrpcImageMode: (mode: DeviceGrpcImageMode) => void; /** Restart gRPC streaming with scrcpy or emulator-gRPC input delivery. */ setGrpcInputSource: (source: DeviceInputSource) => void; + /** Restart the gRPC source with the selected host-side video encoder. */ + setGrpcVideoCodec: (codec: DeviceGrpcVideoCodec) => void; /** Live WebRTC stream telemetry; null for HTTP/WebSocket transports. */ streamStats: DeviceStreamStats | null; /** Enable telemetry polling while a consumer is displaying WebRTC statistics. */ diff --git a/packages/@expo/hub-client/src/useAndroidDevice.ts b/packages/@expo/hub-client/src/useAndroidDevice.ts index 529123df..5662a998 100644 --- a/packages/@expo/hub-client/src/useAndroidDevice.ts +++ b/packages/@expo/hub-client/src/useAndroidDevice.ts @@ -2,13 +2,13 @@ * serve-emu (Android) implementation of the {@link DeviceClient} interface. * * Wire protocol (see serve-emu `src/middleware.ts` / `src/input.ts`): - * - H.264 video + input share one WebSocket at `/ws?frame-meta=1`. + * - Encoded video + input share one WebSocket at `/ws?frame-meta=1`. * With WebRTC video, input stays on `/ws?video=0`; signaling uses * `/webrtc/{offer,close}`. serve-emu is multi-device: `?device=` * selects the target (omitted → first available). - * - Binary inbound messages are H.264 access units, each prefixed with a - * 16-byte "SEMU" header (keyframe flag + PTS); decoded with WebCodecs into a - * ``. + * - Binary inbound messages are H.264, VP8, or VP9 access units, each prefixed + * with a versioned "SEMU" header (keyframe flag + PTS); decoded with + * WebCodecs into a ``. H.264 retains an MSE fallback. * - Outbound input is JSON on the same socket: `{type:'touch',action,x,y}`, * `{type:'home'|'back'|'recents'|'power'}`, `{type:'reset-video'}`. * - Screen size comes from the decoded frames; logcat is an SSE feed at @@ -36,14 +36,24 @@ import { parseAndroidStreamSettings, } from './android-stream-settings'; import { + androidStreamSourceSupportsWebRtc, androidStreamSourceErrorMessage, parseAndroidStreamSource, } from './android-stream-source'; +import { + fixedWebCodecsCodec, + grpcVideoCodecLabel, + isWebCodecsUnsupportedError, + mseFallbackCodecError, + parseAndroidVideoSession, + resolveVideoKeyFrame, + webCodecsCodec, +} from './android-video-codec'; import { DeviceSettingWriteTracker, mergeAuthoritativeDeviceSetting, } from './device-setting-writes'; -import { buildCodecString, isWebCodecsSupported, parseFramePacket, scanAU } from './h264'; +import { isWebCodecsSupported, parseFramePacket, scanAU } from './h264'; import { androidMessageForKeyboardInput } from './keyboard'; import { MsePlayer } from './mse-player'; import { useStreamSettingsResource } from './useStreamSettingsResource'; @@ -56,6 +66,7 @@ import { type DeviceConnectionOptions, type DeviceEvent, type DeviceGrpcImageMode, + type DeviceGrpcVideoCodec, type DeviceInputSource, type DeviceLog, type DeviceSettingKey, @@ -162,6 +173,7 @@ type ServeEmuStreamSettings = type ServeEmuApiInfo = { size?: { width?: unknown; height?: unknown }; stream?: unknown; + viewerTransports?: unknown; }; function isIceServer(value: unknown): value is WebRtcIceServer { @@ -198,6 +210,24 @@ export function parseServeEmuStreamSettings(value: unknown): ServeEmuStreamSetti }; } +/** Prefer the active source's viewer catalog over the host's configured default transport. */ +export function parseServeEmuViewerTransports(value: unknown): ServeEmuStreamSettings | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const candidate = value as Record; + if ( + !Array.isArray(candidate.available) || + !candidate.available.every( + (transport) => transport === 'websocket' || transport === 'webrtc', + ) + ) { + return null; + } + if (candidate.available.includes('webrtc')) { + return parseServeEmuStreamSettings(candidate.webrtc); + } + return candidate.available.includes('websocket') ? { transport: 'websocket' } : null; +} + export function useAndroidDeviceClient(options: DeviceConnectionOptions): DeviceClient { const { baseUrl, enabled = true, device: targetDevice = null, streamMode } = options; const active = enabled && !!baseUrl; @@ -453,6 +483,7 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device mode: DeviceStreamSource; grpcImageMode?: DeviceGrpcImageMode; inputSource?: DeviceInputSource; + grpcVideoCodec?: DeviceGrpcVideoCodec; }) => { if (!streamSourceUrl || streamSourcePendingRef.current) return; const request = ++streamSourceRequestRef.current; @@ -534,6 +565,7 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device mode: previous.mode, grpcImageMode, inputSource: previous.inputSource, + grpcVideoCodec: previous.grpcVideoCodec, }); }, [putStreamMode], @@ -554,6 +586,27 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device mode: previous.mode, grpcImageMode: previous.grpcImageMode, inputSource, + grpcVideoCodec: previous.grpcVideoCodec, + }); + }, + [putStreamMode], + ); + + const setGrpcVideoCodec = useCallback( + (grpcVideoCodec: DeviceGrpcVideoCodec) => { + const previous = streamSourceRef.current; + if ( + !previous || + previous.mode !== 'grpc-screenshot' || + previous.grpcVideoCodec === grpcVideoCodec + ) { + return; + } + putStreamMode({ + mode: previous.mode, + grpcImageMode: previous.grpcImageMode, + inputSource: previous.inputSource, + grpcVideoCodec, }); }, [putStreamMode], @@ -585,6 +638,7 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device current?.mode === next.mode && current.grpcImageMode === next.grpcImageMode && current.inputSource === next.inputSource && + current.grpcVideoCodec === next.grpcVideoCodec && current.sessionGeneration === next.sessionGeneration && current.availableModes.join() === next.availableModes.join() && current.availableInputSources.join() === next.availableInputSources.join() @@ -671,7 +725,10 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device if (!response.ok) return; const info = (await response.json()) as ServeEmuApiInfo; if (cancelled) return; - const next = parseServeEmuStreamSettings(info.stream) ?? { transport: 'websocket' }; + const next = + parseServeEmuViewerTransports(info.viewerTransports) ?? + parseServeEmuStreamSettings(info.stream) ?? + { transport: 'websocket' }; setServerStreamSettings((current) => JSON.stringify(current) === JSON.stringify(next) ? current : next, ); @@ -700,9 +757,11 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device }, [active, baseUrl, targetDevice]); const webRtcRequested = streamMode === 'webrtc'; - const waitingForWebRtcMetadata = webRtcRequested && serverStreamSettings === null; + const sourceSupportsWebRtc = androidStreamSourceSupportsWebRtc(streamSource); + const waitingForWebRtcMetadata = + webRtcRequested && sourceSupportsWebRtc && serverStreamSettings === null; const useWebRtc = - webRtcRequested && serverStreamSettings?.transport === 'webrtc'; + webRtcRequested && sourceSupportsWebRtc && serverStreamSettings?.transport === 'webrtc'; const requestWebRtcKeyframe = useCallback(() => { send({ type: 'reset-video' }); }, [send]); @@ -833,7 +892,7 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device }; }, [useWebRtc, webRtcStream, webRtcVideoElement, markWebRtcFrameDecoded]); - // ── H.264 video + input WebSocket (with reconnect) ── + // ── Encoded video + input WebSocket (with reconnect) ── useEffect(() => { if (!active || !baseUrl) { setStatus('idle'); @@ -850,11 +909,7 @@ export function useAndroidDeviceClient(options: DeviceConnectionOptions): Device // Media Source Extensions — not secure-context gated — which decodes the same // H.264 through a