Add VP8 and VP9 Android gRPC stream options

.changeset/calm-codecs-stream.md

@@ -0,0 +1,5 @@
+---
+'expo-device-hub': minor
+---
+
+Add selectable H.264, VP8, and VP9 host encoding for Android emulator gRPC WebSocket streams.

README.md

@@ -123,8 +123,8 @@ registers the plugin against itself, so the standalone app still gets the real
 
 [`@expo/hub-client`](packages/@expo/hub-client) is the **device-client layer**. The
 two backends speak very different wire protocols — serve-sim streams MJPEG/H.264 and
-takes binary touch packets, while serve-emu streams H.264 (WebCodecs) and takes JSON
-gestures — so this package hides that behind one shared contract:
+takes binary touch packets, while serve-emu streams H.264, VP8, or VP9 through
+WebCodecs and takes JSON gestures — so this package hides that behind one shared contract:
 
 - a hook (`useIosDeviceClient` / `useAndroidDeviceClient` and general
   `useActiveDeviceClient`) that owns the WebSocket connection and exposes the live

packages/@expo/hub-client/src/__tests__/android-stream-source.test.ts

@@ -1,6 +1,9 @@
 import { describe, expect, test } from 'bun:test';
 
-import { parseAndroidStreamSource } from '../android-stream-source';
+import {
+  androidStreamSourceSupportsWebRtc,
+  parseAndroidStreamSource,
+} from '../android-stream-source';
 
 describe('parseAndroidStreamSource', () => {
   test('parses the authoritative gRPC image mode', () => {
@@ -11,6 +14,7 @@ describe('parseAndroidStreamSource', () => {
         grpcImageMode: 'mmap',
         inputSource: 'scrcpy',
         availableInputSources: ['scrcpy', 'grpc'],
+        grpcVideoCodec: 'vp9',
         availableModes: ['scrcpy', 'grpc-screenshot'],
         sessionGeneration: 4,
       }),
@@ -19,11 +23,74 @@ describe('parseAndroidStreamSource', () => {
       grpcImageMode: 'mmap',
       inputSource: 'scrcpy',
       availableInputSources: ['scrcpy', 'grpc'],
+      grpcVideoCodec: 'vp9',
       availableModes: ['scrcpy', 'grpc-screenshot'],
       sessionGeneration: 4,
     });
   });
 
+  test('uses WebRTC only for sources backed by H.264', () => {
+    expect(androidStreamSourceSupportsWebRtc(null)).toBeTrue();
+    expect(
+      androidStreamSourceSupportsWebRtc({
+        mode: 'scrcpy',
+        grpcImageMode: 'png',
+        inputSource: 'scrcpy',
+        availableInputSources: ['scrcpy'],
+        grpcVideoCodec: 'vp9',
+        availableModes: ['scrcpy', 'grpc-screenshot'],
+        sessionGeneration: 1,
+      }),
+    ).toBeTrue();
+    expect(
+      androidStreamSourceSupportsWebRtc({
+        mode: 'grpc-screenshot',
+        grpcImageMode: 'mmap',
+        inputSource: 'scrcpy',
+        availableInputSources: ['scrcpy', 'grpc'],
+        grpcVideoCodec: 'h264',
+        availableModes: ['scrcpy', 'grpc-screenshot'],
+        sessionGeneration: 2,
+      }),
+    ).toBeTrue();
+    expect(
+      androidStreamSourceSupportsWebRtc({
+        mode: 'grpc-screenshot',
+        grpcImageMode: 'mmap',
+        inputSource: 'scrcpy',
+        availableInputSources: ['scrcpy', 'grpc'],
+        grpcVideoCodec: 'vp8',
+        availableModes: ['scrcpy', 'grpc-screenshot'],
+        sessionGeneration: 3,
+      }),
+    ).toBeFalse();
+    expect(
+      androidStreamSourceSupportsWebRtc({
+        mode: 'grpc-screenshot',
+        grpcImageMode: 'mmap',
+        inputSource: 'grpc',
+        availableInputSources: ['scrcpy', 'grpc'],
+        grpcVideoCodec: 'vp9',
+        availableModes: ['scrcpy', 'grpc-screenshot'],
+        sessionGeneration: 4,
+      }),
+    ).toBeFalse();
+  });
+
+  test('defaults legacy responses without a video codec to H.264', () => {
+    expect(
+      parseAndroidStreamSource({
+        ok: true,
+        mode: 'grpc-screenshot',
+        grpcImageMode: 'png',
+        inputSource: 'scrcpy',
+        availableInputSources: ['scrcpy', 'grpc'],
+        availableModes: ['scrcpy', 'grpc-screenshot'],
+        sessionGeneration: 1,
+      }),
+    ).toMatchObject({ grpcVideoCodec: 'h264' });
+  });
+
   test('rejects missing or unsupported image modes', () => {
     const response = {
       ok: true,
@@ -35,6 +102,13 @@ describe('parseAndroidStreamSource', () => {
     };
     expect(parseAndroidStreamSource(response)).toBeNull();
     expect(parseAndroidStreamSource({ ...response, grpcImageMode: 'rgb' })).toBeNull();
+    expect(
+      parseAndroidStreamSource({
+        ...response,
+        grpcImageMode: 'png',
+        grpcVideoCodec: 'av1',
+      }),
+    ).toBeNull();
   });
 
   test('rejects unavailable or unsupported input sources', () => {

packages/@expo/hub-client/src/__tests__/android-stream.test.ts

@@ -8,7 +8,21 @@ import {
   androidStreamSourceErrorMessage,
   parseAndroidStreamSource,
 } from '../android-stream-source';
-import { androidWsUrlFor, parseServeEmuStreamSettings } from '../useAndroidDevice';
+import {
+  fixedWebCodecsCodec,
+  isRawVideoKeyFrame,
+  isWebCodecsUnsupportedError,
+  mseFallbackCodecError,
+  parseAndroidVideoSession,
+  resolveVideoKeyFrame,
+  webCodecsCodec,
+} from '../android-video-codec';
+import { parseFramePacket } from '../h264';
+import {
+  androidWsUrlFor,
+  parseServeEmuStreamSettings,
+  parseServeEmuViewerTransports,
+} from '../useAndroidDevice';
 
 describe('serve-emu stream contract', () => {
   test('uses a metadata video socket for H.264 and an input-only socket for WebRTC', () => {
@@ -20,6 +34,78 @@ describe('serve-emu stream contract', () => {
     );
   });
 
+  test('parses authoritative codec generation boundaries for WebSocket video', () => {
+    expect(
+      parseAndroidVideoSession({
+        type: 'video-session',
+        size: { width: 1080, height: 2400 },
+        codec: 'vp9',
+      }),
+    ).toEqual({ size: { width: 1080, height: 2400 }, codec: 'vp9' });
+    expect(
+      parseAndroidVideoSession({
+        type: 'video-session',
+        size: { width: 1080, height: 2400 },
+      }),
+    ).toEqual({ size: { width: 1080, height: 2400 }, codec: 'h264' });
+    expect(
+      parseAndroidVideoSession({
+        type: 'video-session',
+        size: { width: 0, height: 2400 },
+        codec: 'vp8',
+      }),
+    ).toBeNull();
+    expect(fixedWebCodecsCodec('h264')).toBeNull();
+    expect(fixedWebCodecsCodec('vp8')).toBe('vp8');
+    expect(fixedWebCodecsCodec('vp9')).toBe('vp09.00.10.08');
+    expect(webCodecsCodec('h264', Uint8Array.of(0x67, 0x64, 0, 0x29))).toBe('avc1.640029');
+    expect(webCodecsCodec('h264', Uint8Array.of(0x67, 0x64))).toBeNull();
+    expect(mseFallbackCodecError('h264')).toBeNull();
+    expect(mseFallbackCodecError('h264', false)).toBe(
+      'This browser cannot decode H.264 (WebCodecs unavailable).',
+    );
+    expect(mseFallbackCodecError('vp8')).toBe(
+      'VP8 WebSocket video requires WebCodecs.',
+    );
+    expect(isWebCodecsUnsupportedError({ name: 'NotSupportedError' })).toBe(true);
+    expect(isWebCodecsUnsupportedError(new Error('decode failed'))).toBe(false);
+  });
+
+  test('recovers VPx keyframes from raw payloads only when SEMU metadata is absent', () => {
+    const vp8Key = Uint8Array.of(0xf0, 0x02, 0, 0x9d, 0x01, 0x2a, 0x10, 0, 0x10, 0);
+    const vp9Key = Uint8Array.of(0x82, 0x49, 0x83, 0x42);
+
+    expect(isRawVideoKeyFrame('vp8', vp8Key)).toBe(true);
+    expect(isRawVideoKeyFrame('vp8', Uint8Array.of(0xb1, 1, 0, 5))).toBe(false);
+    expect(isRawVideoKeyFrame('vp9', vp9Key)).toBe(true);
+    expect(isRawVideoKeyFrame('vp9', Uint8Array.of(0x82, 0, 0, 0))).toBe(false);
+    expect(resolveVideoKeyFrame('vp8', false, vp8Key)).toBe(false);
+    expect(resolveVideoKeyFrame('vp9', true, Uint8Array.of(1, 2, 3))).toBe(true);
+    expect(resolveVideoKeyFrame('vp8', null, vp8Key)).toBe(true);
+  });
+
+  test('reads keyframe and PTS metadata from SEMU v1 and v2 packets', () => {
+    const packet = (version: 1 | 2, payload: number[]) => {
+      const headerBytes = version === 1 ? 16 : 24;
+      const bytes = new Uint8Array(headerBytes + payload.length);
+      const view = new DataView(bytes.buffer);
+      view.setUint32(0, 0x53454d55, false);
+      view.setUint8(4, version);
+      view.setUint8(5, 1);
+      view.setBigUint64(8, 123_456n, false);
+      if (version === 2) view.setBigUint64(16, 987_654_321n, false);
+      bytes.set(payload, headerBytes);
+      return bytes;
+    };
+
+    for (const version of [1, 2] as const) {
+      const parsed = parseFramePacket(packet(version, [1, 2, 3]));
+      expect(parsed.isKey).toBe(true);
+      expect(parsed.timestamp).toBe(123_456);
+      expect([...parsed.data]).toEqual([1, 2, 3]);
+    }
+  });
+
   test('accepts the host-owned H.264 WebRTC configuration', () => {
     expect(
       parseServeEmuStreamSettings({
@@ -50,6 +136,29 @@ describe('serve-emu stream contract', () => {
     });
   });
 
+  test('disables WebRTC when the active source codec only supports WebSocket viewing', () => {
+    expect(
+      parseServeEmuViewerTransports({
+        default: 'websocket',
+        available: ['websocket'],
+        webrtc: null,
+      }),
+    ).toEqual({ transport: 'websocket' });
+    expect(
+      parseServeEmuViewerTransports({
+        default: 'webrtc',
+        available: ['websocket', 'webrtc'],
+        webrtc: {
+          transport: 'webrtc',
+          codec: 'h264',
+          iceServers: [{ urls: ['stun:stun.example.test:3478'] }],
+          iceTransportPolicy: 'all',
+        },
+      }),
+    ).toMatchObject({ transport: 'webrtc', codec: 'h264' });
+    expect(parseServeEmuViewerTransports({ available: ['webtransport'] })).toBeNull();
+  });
+
   test('rejects unsupported codecs and malformed ICE settings', () => {
     expect(parseServeEmuStreamSettings({ transport: 'websocket' })).toEqual({
       transport: 'websocket',
@@ -130,6 +239,7 @@ describe('serve-emu capture source contract', () => {
         grpcImageMode: 'mmap',
         inputSource: 'scrcpy',
         availableInputSources: ['scrcpy', 'grpc'],
+        grpcVideoCodec: 'vp8',
         availableModes: ['scrcpy', 'grpc-screenshot'],
         sessionGeneration: 2,
       }),
@@ -138,6 +248,7 @@ describe('serve-emu capture source contract', () => {
       grpcImageMode: 'mmap',
       inputSource: 'scrcpy',
       availableInputSources: ['scrcpy', 'grpc'],
+      grpcVideoCodec: 'vp8',
       availableModes: ['scrcpy', 'grpc-screenshot'],
       sessionGeneration: 2,
     });
@@ -157,6 +268,7 @@ describe('serve-emu capture source contract', () => {
       grpcImageMode: 'png',
       inputSource: 'scrcpy',
       availableInputSources: ['scrcpy'],
+      grpcVideoCodec: 'h264',
       availableModes: ['scrcpy'],
       sessionGeneration: 0,
     });

packages/@expo/hub-client/src/android-stream-source.ts

@@ -1,5 +1,6 @@
 import {
   type DeviceGrpcImageMode,
+  type DeviceGrpcVideoCodec,
   type DeviceInputSource,
   type DeviceStreamSource,
   type DeviceStreamSourceStatus,
@@ -39,6 +40,17 @@ function isInputSource(value: unknown): value is DeviceInputSource {
   return value === 'scrcpy' || value === 'grpc';
 }
 
+function isGrpcVideoCodec(value: unknown): value is DeviceGrpcVideoCodec {
+  return value === 'h264' || value === 'vp8' || value === 'vp9';
+}
+
+/** Whether the active source can feed serve-emu's H.264-only WebRTC publisher. */
+export function androidStreamSourceSupportsWebRtc(
+  source: DeviceStreamSourceStatus | null,
+): boolean {
+  return source?.mode !== 'grpc-screenshot' || source.grpcVideoCodec === 'h264';
+}
+
 /** Parse serve-emu's authoritative device-scoped `/api/stream-mode` response. */
 export function parseAndroidStreamSource(value: unknown): DeviceStreamSourceStatus | null {
   if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
@@ -47,7 +59,8 @@ export function parseAndroidStreamSource(value: unknown): DeviceStreamSourceStat
     candidate.ok !== true ||
     !isAndroidStreamSource(candidate.mode) ||
     !isGrpcImageMode(candidate.grpcImageMode) ||
-    !isInputSource(candidate.inputSource)
+    !isInputSource(candidate.inputSource) ||
+    (candidate.grpcVideoCodec !== undefined && !isGrpcVideoCodec(candidate.grpcVideoCodec))
   ) {
     return null;
   }
@@ -81,6 +94,8 @@ export function parseAndroidStreamSource(value: unknown): DeviceStreamSourceStat
     grpcImageMode: candidate.grpcImageMode,
     inputSource: candidate.inputSource,
     availableInputSources: candidate.availableInputSources,
+    // Older serve-emu versions always emitted H.264 and omit this field.
+    grpcVideoCodec: candidate.grpcVideoCodec ?? 'h264',
     availableModes: candidate.availableModes,
     sessionGeneration: candidate.sessionGeneration,
   };