[macos] Camera improvements. Use pixel format in camera, and improve …

…display
ossia · Apr 7, 2024 · c042527 · c042527
1 parent 7682d20
commit c042527
Show file tree

Hide file tree

Showing 7 changed files with 148 additions and 50 deletions.
diff --git a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.avf.mm b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.avf.mm
@@ -1,50 +1,143 @@
-#import <AVFoundation/AVFoundation.h>
+#include "CameraDevice.hpp"
 
 #include "CameraSettings.hpp"
-#include <functional>
+
+#include <Gfx/CameraDeviceEnumerator.hpp>
+
+#import <AVFoundation/AVFoundation.h>
 #include <libavutil/pixfmt.h>
 
-#include <QDebug>
+#include <functional>
+#include <iostream>
+#include <memory>
 
 namespace Gfx
 {
-static void iterateCameraFormats(AVCaptureDevice* device, std::function<void(CameraSettings, QString)> func)
+static constexpr int avf_pixelformat_to_ffmpeg(uint32_t fourcc)
 {
-  const char *name = [[device localizedName] UTF8String];
-  for (id format in [device valueForKey:@"formats"]) {
-    CMFormatDescriptionRef formatDescription{};
-    CMVideoDimensions dimensions{};
+  switch(fourcc)
+  {
+    case kCMPixelFormat_32ARGB:
+      return AV_PIX_FMT_ARGB;
+    case kCMPixelFormat_32BGRA:
+      return AV_PIX_FMT_BGRA;
+    case kCMPixelFormat_24RGB:
+      return AV_PIX_FMT_RGB24;
+    case kCMPixelFormat_16BE555:
+      return AV_PIX_FMT_RGB555BE;
+    case kCMPixelFormat_16BE565:
+      return AV_PIX_FMT_RGB565BE;
+    case kCMPixelFormat_16LE555:
+      return AV_PIX_FMT_RGB555LE;
+    case kCMPixelFormat_16LE565:
+      return AV_PIX_FMT_RGB565LE;
+    case kCMPixelFormat_16LE5551:
+      return 0;
+    case 'yuv2':
+    case 'YUV2':
+    case 'uyvy':
+    case 'UYVY':
+    case kCMPixelFormat_422YpCbCr8:
+      return AV_PIX_FMT_UYVY422;
+    case kCMPixelFormat_422YpCbCr8_yuvs:
+      return AV_PIX_FMT_YUYV422;
+    case kCMPixelFormat_444YpCbCr8:
+      return AV_PIX_FMT_YUV444P; // WARNING ! this is planar, macOS's is packed. check that it's ok and ffmpeg assumes the conversion
+    case kCMPixelFormat_4444YpCbCrA8:
+      return AV_PIX_FMT_YUVA444P;
+    case kCMPixelFormat_422YpCbCr16:
+      return AV_PIX_FMT_YUV422P16LE;
+    case kCMPixelFormat_422YpCbCr10:
+      return AV_PIX_FMT_YUV422P10LE;
+    case kCMPixelFormat_444YpCbCr10:
+      return AV_PIX_FMT_YUV444P10LE;
+    case kCMPixelFormat_8IndexedGray_WhiteIsZero:
+    default:
+      return -1;
+  }
+}
 
-    formatDescription = (__bridge CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
-    dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
+struct AVFCameraEnumerator : public Device::DeviceEnumerator
+{
+  explicit AVFCameraEnumerator(
+      std::shared_ptr<CameraDeviceEnumerator> parent, AVCaptureDevice* dev)
+      : parent{parent}
+      , device{dev}
+  {
+  }
+  std::shared_ptr<CameraDeviceEnumerator> parent;
+  AVCaptureDevice* device{};
+  void enumerate(std::function<void(const QString&, const Device::DeviceSettings&)> func)
+      const override
+  {
+    const char* name = [[device localizedName] UTF8String];
+    for(id format in [device valueForKey:@"formats"])
+    {
+      CMFormatDescriptionRef formatDescription{};
+      CMVideoDimensions dimensions{};
 
-    for (id range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
-      double fps{};
+      formatDescription = (__bridge CMFormatDescriptionRef)
+          [format performSelector:@selector(formatDescription)];
+      dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
+      auto fourcc = CMFormatDescriptionGetMediaSubType(formatDescription);
+      std::string_view fcc((const char*)&fourcc, 4);
+      for(id range in [format valueForKey:@"videoSupportedFrameRateRanges"])
+      {
+        double fps{};
 
-      [[range valueForKey:@"maxFrameRate"] getValue:&fps];
+        [[range valueForKey:@"maxFrameRate"] getValue:&fps];
 
-      const auto prettyName = QString("%1 (%2x%3@%4)").arg(name).arg(dimensions.width).arg(dimensions.height).arg(fps);
+        const auto mode = QString("%1x%2@%3 %4")
+                              .arg(dimensions.width)
+                              .arg(dimensions.height)
+                              .arg(fps)
+                              .arg(std::string(fcc).c_str());
 
-      func({.input = "avfoundation", .device = name, .size = { dimensions.width, dimensions.height}, .fps = fps }, prettyName);
+        Device::DeviceSettings s;
+        CameraSettings set;
+        set.input = "avfoundation";
+        set.device = name;
+        set.size = {dimensions.width, dimensions.height};
+        set.fps = fps;
+        set.pixelformat = avf_pixelformat_to_ffmpeg(fourcc);
+        s.name = name;
+        s.protocol = CameraProtocolFactory::static_concreteKey();
+        s.deviceSpecificSettings = QVariant::fromValue(set);
+        func(mode, s);
+      }
     }
   }
-}
+};
 
-void enumerateCameraDevices(std::function<void(CameraSettings, QString)> func)
+struct AVFCameraDeviceEnumerator : public CameraDeviceEnumerator
 {
+  void registerAllEnumerators(Device::DeviceEnumerators& enums) override
   {
-    NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
-    for (AVCaptureDevice *device in devices) {
-      iterateCameraFormats(device, func);
+    auto self = shared_from_this();
+    SCORE_ASSERT(self);
+
+    {
+      NSArray* devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
+      for(AVCaptureDevice* device in devices)
+      {
+        const char* name = [[device localizedName] UTF8String];
+        enums.push_back({name, new AVFCameraEnumerator{self, device}});
+      }
     }
-  }
 
-  {
-    NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
-    for (AVCaptureDevice *device in devices) {
-      iterateCameraFormats(device, func);
+    {
+      NSArray* devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
+      for(AVCaptureDevice* device in devices)
+      {
+        const char* name = [[device localizedName] UTF8String];
+        enums.push_back({name, new AVFCameraEnumerator{self, device}});
+      }
     }
   }
-}
-}
+};
 
+std::shared_ptr<CameraDeviceEnumerator> make_camera_enumerator()
+{
+  return std::make_shared<AVFCameraDeviceEnumerator>();
+}
+} // namespace Gfx
diff --git a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.cpp b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.cpp
@@ -1,5 +1,8 @@
 #include "CameraDevice.hpp"
 
+#include <Gfx/Graph/VideoNode.hpp>
+#include <Video/CameraInput.hpp>
+
 #include <State/MessageListSerialization.hpp>
 #include <State/Widgets/AddressFragmentLineEdit.hpp>
 
@@ -117,7 +120,7 @@ Device::DeviceEnumerators
 CameraProtocolFactory::getEnumerators(const score::DocumentContext& ctx) const
 {
   Device::DeviceEnumerators enums;
-#if !defined(__linux__)
+#if !defined(__linux__) && !defined(__APPLE__)
   enums.push_back({"Cameras", new CameraEnumerator});
 #else
   auto devices = Gfx::make_camera_enumerator();

diff --git a/src/plugins/score-plugin-gfx/Gfx/CameraDevice.hpp b/src/plugins/score-plugin-gfx/Gfx/CameraDevice.hpp
@@ -1,26 +1,15 @@
 #pragma once
-#include <Gfx/CameraSettings.hpp>
-#include <Gfx/GfxDevice.hpp>
-#include <Gfx/GfxExecContext.hpp>
-#include <Gfx/GfxInputDevice.hpp>
-#include <Gfx/Graph/VideoNode.hpp>
-#include <Video/CameraInput.hpp>
-
-#include <ossia/gfx/texture_parameter.hpp>
-#include <ossia/network/base/device.hpp>
-#include <ossia/network/base/protocol.hpp>
-
-#include <QLineEdit>
-
-class QComboBox;
-
-// Score part
-
 #include <Device/Protocol/DeviceInterface.hpp>
 #include <Device/Protocol/DeviceSettings.hpp>
 #include <Device/Protocol/ProtocolFactoryInterface.hpp>
 #include <Device/Protocol/ProtocolSettingsWidget.hpp>
 
+#include <Gfx/CameraSettings.hpp>
+#include <Gfx/GfxInputDevice.hpp>
+
+class QComboBox;
+class QLineEdit;
+
 namespace Gfx
 {
 class CameraProtocolFactory final : public Device::ProtocolFactory

diff --git a/src/plugins/score-plugin-gfx/Gfx/GfxInputDevice.cpp b/src/plugins/score-plugin-gfx/Gfx/GfxInputDevice.cpp
@@ -1,5 +1,8 @@
 #include "GfxInputDevice.hpp"
 
+#include <Gfx/Graph/VideoNode.hpp>
+#include <Video/CameraInput.hpp>
+
 namespace Gfx
 {
 

diff --git a/src/plugins/score-plugin-gfx/Gfx/GfxInputDevice.hpp b/src/plugins/score-plugin-gfx/Gfx/GfxInputDevice.hpp
@@ -1,13 +1,16 @@
 #pragma once
 #include <Gfx/GfxDevice.hpp>
 #include <Gfx/GfxExecContext.hpp>
-#include <Gfx/Graph/VideoNode.hpp>
-#include <Video/CameraInput.hpp>
-
 #include <ossia/gfx/texture_parameter.hpp>
 #include <ossia/network/base/device.hpp>
 #include <ossia/network/base/protocol.hpp>
-
+namespace Video
+{
+class ExternalInput;
+}
+namespace score::gfx {
+class CameraNode;
+}
 namespace Gfx
 {
 

diff --git a/src/plugins/score-plugin-gfx/Gfx/Shmdata/ShmdataInputDevice.cpp b/src/plugins/score-plugin-gfx/Gfx/Shmdata/ShmdataInputDevice.cpp
@@ -1,5 +1,8 @@
 #include "ShmdataInputDevice.hpp"
 
+#include <Gfx/Graph/VideoNode.hpp>
+#include <Video/CameraInput.hpp>
+
 #include <State/MessageListSerialization.hpp>
 #include <State/Widgets/AddressFragmentLineEdit.hpp>
 

diff --git a/src/plugins/score-plugin-media/Video/CameraInput.cpp b/src/plugins/score-plugin-media/Video/CameraInput.cpp
@@ -70,7 +70,11 @@ bool CameraInput::start() noexcept
   if(auto codec_name = avcodec_get_name(this->m_requestedCodec))
     av_dict_set(&options, "input_format", codec_name, 0);
 
-  // FIXME support pixel format choosing
+  if(m_requestedPixfmt != -1)
+  {
+    if(auto name = av_get_pix_fmt_name(m_requestedPixfmt))
+      av_dict_set(&options, "pixel_format", name, 0);
+  }
 
   if(fps > 0.)
     av_dict_set(&options, "framerate", std::to_string(fps).c_str(), 0);