Pinch to zoom in & out (#384)

New VideoView.pinchToZoom and pinchToZoomAutoZoomOut properties which
work for iOS only. Seamless zoom in/out will work for supported devices
such as .builtInTripleCamera. This is currently only for local tracks
(capturing).


https://github.com/livekit/client-sdk-swift/assets/548776/2bc5fabb-3184-4613-9d3e-c8a236a15189

Author: hiroshihorie

Package.swift

@@ -18,7 +18,7 @@ let package = Package(
     ],
     dependencies: [
         // LK-Prefixed Dynamic WebRTC XCFramework
-        .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "114.5735.16"),
+        .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "114.5735.18"),
         .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"),
         .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"),
         // Only used for DocC generation

Sources/LiveKit/Extensions/AVCaptureDevice.swift

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2024 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import AVFoundation
+
+public extension AVCaptureDevice {
+    /// Helper extension to return the acual direction the camera is facing.
+    /// In macOS, the Facetime camera's position is .unspecified but this property will return .front for such cases.
+    var facingPosition: AVCaptureDevice.Position {
+        if deviceType == .builtInWideAngleCamera, position == .unspecified {
+            return .front
+        }
+
+        return position
+    }
+}
+
+public extension Collection where Element: AVCaptureDevice {
+    /// Helper extension to return only a single suggested device for each position.
+    func singleDeviceforEachPosition() -> [AVCaptureDevice] {
+        let front = first { $0.facingPosition == .front }
+        let back = first { $0.facingPosition == .back }
+        return [front, back].compactMap { $0 }
+    }
+}

Sources/LiveKit/Support/DeviceManager.swift

@@ -28,25 +28,26 @@ class DeviceManager: Loggable {
     }
 
     // Async version, waits until inital device fetch is complete
-    public func devices(types: [AVCaptureDevice.DeviceType] = [.builtInWideAngleCamera]) async throws -> [AVCaptureDevice] {
-        try await devicesCompleter.wait().filter { types.contains($0.deviceType) }
+    public func devices() async throws -> [AVCaptureDevice] {
+        try await devicesCompleter.wait()
     }
 
     // Sync version
-    public func devices(types: [AVCaptureDevice.DeviceType] = [.builtInWideAngleCamera]) -> [AVCaptureDevice] {
-        _state.devices.filter { types.contains($0.deviceType) }
+    public func devices() -> [AVCaptureDevice] {
+        _state.devices
     }
 
     private lazy var discoverySession: AVCaptureDevice.DiscoverySession = {
         var deviceTypes: [AVCaptureDevice.DeviceType]
         #if os(iOS)
+        // In order of priority
         deviceTypes = [
-            .builtInWideAngleCamera, // General purpose use
-            .builtInTelephotoCamera,
-            .builtInUltraWideCamera,
-            .builtInTripleCamera,
-            .builtInDualCamera,
-            .builtInDualWideCamera,
+            .builtInTripleCamera, // Virtual, switchOver: [2, 6], default: 2
+            .builtInDualCamera, // Virtual, switchOver: [3], default: 1
+            .builtInDualWideCamera, // Virtual, switchOver: [2], default: 2
+            .builtInWideAngleCamera, // Physical, General purpose use
+            .builtInTelephotoCamera, // Physical
+            .builtInUltraWideCamera, // Physical
         ]
         // Xcode 15.0 Swift 5.9 (iOS 17)
         #if compiler(>=5.9)

Sources/LiveKit/Track/Capturers/CameraCapturer.swift

@@ -34,8 +34,8 @@ public class CameraCapturer: VideoCapturer {
     public var options: CameraCaptureOptions { _cameraCapturerState.options }
 
     @objc
-    public static func captureDevices(types: [AVCaptureDevice.DeviceType] = [.builtInWideAngleCamera]) async throws -> [AVCaptureDevice] {
-        try await DeviceManager.shared.devices(types: types)
+    public static func captureDevices() async throws -> [AVCaptureDevice] {
+        try await DeviceManager.shared.devices()
     }
 
     /// Checks whether both front and back capturing devices exist, and can be switched.
@@ -80,7 +80,7 @@ public class CameraCapturer: VideoCapturer {
         var device: AVCaptureDevice?
     }
 
-    private var _cameraCapturerState: StateSync<State>
+    var _cameraCapturerState: StateSync<State>
 
     // Used to hide LKRTCVideoCapturerDelegate symbol
     private lazy var adapter: VideoCapturerDelegateAdapter = .init(cameraCapturer: self)

Sources/LiveKit/Views/VideoView.swift

@@ -140,6 +140,18 @@ public class VideoView: NativeView, Loggable {
         set { _state.mutate { $0.transitionDuration = newValue } }
     }
 
+    @objc
+    public var isPinchToZoomEnabled: Bool {
+        get { _state.isPinchToZoomEnabled }
+        set { _state.mutate { $0.isPinchToZoomEnabled = newValue } }
+    }
+
+    @objc
+    public var isAutoZoomResetEnabled: Bool {
+        get { _state.isAutoZoomResetEnabled }
+        set { _state.mutate { $0.isAutoZoomResetEnabled = newValue } }
+    }
+
     @objc
     public var isDebugMode: Bool {
         get { _state.isDebugMode }
@@ -192,9 +204,12 @@ public class VideoView: NativeView, Loggable {
         var renderTarget: RenderTarget = .primary
         var isSwapping: Bool = false
         var remainingRenderCountBeforeSwap: Int = 0 // Number of frames to be rendered on secondary until swap is initiated
-        var transitionMode: TransitionMode = .flip
+        var transitionMode: TransitionMode = .crossDissolve
         var transitionDuration: TimeInterval = 0.3
 
+        var isPinchToZoomEnabled: Bool = false
+        var isAutoZoomResetEnabled: Bool = true
+
         // Only used for rendering local tracks
         var captureOptions: VideoCaptureOptions? = nil
         var captureDevice: AVCaptureDevice? = nil
@@ -219,6 +234,12 @@ public class VideoView: NativeView, Loggable {
     private var _currentFPS: Int = 0
     private var _frameCount: Int = 0
 
+    #if os(iOS) || os(visionOS)
+    private lazy var _pinchGestureRecognizer = UIPinchGestureRecognizer(target: self, action: #selector(_handlePinchGesture(_:)))
+    // This should be thread safe so it's not required to be guarded by the lock
+    private var _pinchStartZoomFactor: CGFloat = 0.0
+    #endif
+
     override public init(frame: CGRect = .zero) {
         // initial state
         _state = StateSync(State(viewSize: frame.size))
@@ -326,6 +347,15 @@ public class VideoView: NativeView, Loggable {
                 }
             }
 
+            #if os(iOS) || os(visionOS)
+            let newIsPinchToZoomEnabled = newState.isPinchToZoomEnabled
+            if newIsPinchToZoomEnabled != oldState.isPinchToZoomEnabled {
+                Task.detached { @MainActor in
+                    self._pinchGestureRecognizer.isEnabled = newIsPinchToZoomEnabled
+                }
+            }
+            #endif
+
             if newState.isDebugMode != oldState.isDebugMode {
                 // fps timer
                 if newState.isDebugMode {
@@ -359,8 +389,44 @@ public class VideoView: NativeView, Loggable {
 
             await self._renderTimer.restart()
         }
+
+        #if os(iOS) || os(visionOS)
+        // Add pinch gesture recognizer
+        addGestureRecognizer(_pinchGestureRecognizer)
+        _pinchGestureRecognizer.isEnabled = _state.isPinchToZoomEnabled
+        #endif
     }
 
+    #if os(iOS) || os(visionOS)
+    @objc func _handlePinchGesture(_ sender: UIPinchGestureRecognizer) {
+        if let track = _state.track as? LocalVideoTrack,
+           let capturer = track.capturer as? CameraCapturer,
+           let device = capturer.device
+        {
+            if sender.state == .began {
+                _pinchStartZoomFactor = device.videoZoomFactor
+            } else {
+                do {
+                    try device.lockForConfiguration()
+                    defer { device.unlockForConfiguration() }
+
+                    if sender.state == .changed {
+                        let minZoom = device.minAvailableVideoZoomFactor
+                        let maxZoom = device.maxAvailableVideoZoomFactor
+                        device.videoZoomFactor = (_pinchStartZoomFactor * sender.scale).clamped(to: minZoom ... maxZoom)
+                    } else if sender.state == .ended || sender.state == .cancelled, _state.isAutoZoomResetEnabled {
+                        // Zoom to default zoom factor
+                        let defaultZoomFactor = LKRTCCameraVideoCapturer.defaultZoomFactor(forDeviceType: device.deviceType)
+                        device.ramp(toVideoZoomFactor: defaultZoomFactor, withRate: 32.0)
+                    }
+                } catch {
+                    log("Failed to adjust videoZoomFactor", .warning)
+                }
+            }
+        }
+    }
+    #endif
+
     @available(*, unavailable)
     required init?(coder _: NSCoder) {
         fatalError("init(coder:) has not been implemented")
@@ -527,7 +593,7 @@ private extension VideoView {
 
     func _shouldMirror() -> Bool {
         switch _state.mirrorMode {
-        case .auto: return _state.captureDevice?.realPosition == .front
+        case .auto: return _state.captureDevice?.facingPosition == .front
         case .off: return false
         case .mirror: return true
         }
@@ -661,7 +727,7 @@ extension VideoView: VideoRenderer {
 
         // Currently only for iOS
         #if os(iOS)
-        let (mode, duration, position) = _state.read { ($0.transitionMode, $0.transitionDuration, $0.captureDevice?.realPosition) }
+        let (mode, duration, position) = _state.read { ($0.transitionMode, $0.transitionDuration, $0.captureDevice?.facingPosition) }
         if let transitionOption = mode.toAnimationOption(fromPosition: position) {
             UIView.transition(with: self, duration: duration, options: transitionOption, animations: block, completion: nil)
         } else {
@@ -801,16 +867,6 @@ private extension VideoView {
     }
 }
 
-extension AVCaptureDevice {
-    var realPosition: AVCaptureDevice.Position {
-        if deviceType == .builtInWideAngleCamera, position == .unspecified {
-            return .front
-        }
-
-        return position
-    }
-}
-
 #if os(iOS)
 extension VideoView.TransitionMode {
     func toAnimationOption(fromPosition position: AVCaptureDevice.Position? = nil) -> UIView.AnimationOptions? {