RASPBERRYPI ONLY: Handle mandatory stream flags

Look for the RAW mandatory stream flag in the pipeline handler config
file. If this flag is set, it guarantees that the application will
provide buffers for Unicam Image, so override the minUnicamBuffers and
minTotalUnicamBuffers config parameters in the following way:

- If startup drop frames are required, allocate at least 1 internal buffer.
- If no startup drop frames are required, do not allocate any internal buffers.

Look for the Output 0 mandatory stream flag in in the pipeline handler
config file. If this flag is set, it guarantees that the application
will provide buffers for the ISP, do not allocate any internal buffers
for the device.

Add a new rpi_apps.yaml pipeline handler config file that enables both
these flags.  To use the file, set the following env variable for a
custom build:

export LIBCAMERA_RPI_CONFIG_FILE=/usr/local/share/libcamera/pipeline/rpi/vc4/rpi_apps.yaml

or for a packaged install:

export LIBCAMERA_RPI_CONFIG_FILE=/usr/share/libcamera/pipeline/rpi/vc4/rpi_apps.yaml

Signed-off-by: Naushir Patuck <[email protected]>

Author: naushir

src/libcamera/pipeline/rpi/vc4/data/meson.build

@@ -2,6 +2,7 @@
 
 conf_files = files([
     'example.yaml',
+    'rpi_apps.yaml',
 ])
 
 install_data(conf_files,

src/libcamera/pipeline/rpi/vc4/data/rpi_apps.yaml

@@ -0,0 +1,45 @@
+{
+        "version": 1.0,
+        "target": "bcm2835",
+
+        "pipeline_handler":
+        {
+                # The minimum number of internal buffers to be allocated for
+                # Unicam. This value must be greater than 0, but less than or
+                # equal to min_total_unicam_buffers.
+                #
+                # A larger number of internal buffers can reduce the occurrence
+                # of frame drops during high CPU loads, but might also cause
+                # additional latency in the system.
+                #
+                # Note that the pipeline handler might override this value and
+                # not allocate any internal buffers if it knows they will never
+                # be used. For example if the RAW stream is marked as mandatory
+                # and there are no dropped frames signalled for algorithm
+                # convergence.
+                #
+                "min_unicam_buffers": 2,
+
+                # The minimum total (internal + external) buffer count used for
+                # Unicam. The number of internal buffers allocated for Unicam is
+                # given by:
+                #
+                # internal buffer count = max(min_unicam_buffers,
+                #         min_total_unicam_buffers - external buffer count)
+                #
+                "min_total_unicam_buffers": 4,
+
+                # Override any request from the IPA to drop a number of startup
+                # frames.
+                #
+                # "disable_startup_frame_drops": false,
+
+                # The application will always provide a request buffer for the
+                # RAW stream, if it has been configured.
+                "raw_mandatory_stream": true,
+
+                # The application will always provide a request buffer for the
+                # Output 0 stream, if it has been configured.
+                "output0_mandatory_stream": true,
+        }
+}

src/libcamera/pipeline/rpi/vc4/vc4.cpp

@@ -105,6 +105,16 @@ class Vc4CameraData final : public RPi::CameraData
 		 * minTotalUnicamBuffers >= minUnicamBuffers
 		 */
 		unsigned int minTotalUnicamBuffers;
+		/*
+		 * The application will always provide a request buffer for the
+		 * RAW stream, if it has been configured.
+		 */
+		bool rawMandatoryStream;
+		/*
+		 * The application will always provide a request buffer for the
+		 * Output 0 stream, if it has been configured.
+		 */
+		bool output0MandatoryStream;
 	};
 
 	Config config_;
@@ -219,42 +229,74 @@ bool PipelineHandlerVc4::match(DeviceEnumerator *enumerator)
 int PipelineHandlerVc4::prepareBuffers(Camera *camera)
 {
 	Vc4CameraData *data = cameraData(camera);
-	unsigned int numRawBuffers = 0;
+	unsigned int minUnicamBuffers = data->config_.minUnicamBuffers;
+	unsigned int minTotalUnicamBuffers = data->config_.minTotalUnicamBuffers;
+	unsigned int numRawBuffers = 0, minIspBuffers = 1;
 	int ret;
 
-	for (Stream *s : camera->streams()) {
-		if (BayerFormat::fromPixelFormat(s->configuration().pixelFormat).isValid()) {
-			numRawBuffers = s->configuration().bufferCount;
-			break;
+	if (data->unicam_[Unicam::Image].getFlags() & StreamFlag::External) {
+		numRawBuffers = data->unicam_[Unicam::Image].getBuffers().size();
+		/*
+		 * If the application provides a guarantees that Unicam
+		 * image buffers will always be provided for the RAW stream
+		 * in a Request, we need:
+		 * - at least 1 internal Unicam buffer to handle startup frame drops,
+		 * - no internal Unicam buffers if there are no startup frame drops.
+		 */
+		if (data->config_.rawMandatoryStream) {
+			if (data->dropFrameCount_) {
+				minUnicamBuffers = 2;
+				minTotalUnicamBuffers = 2;
+			} else {
+				minUnicamBuffers = 0;
+				minTotalUnicamBuffers = 0;
+			}
 		}
 	}
 
+	if (data->isp_[Isp::Output0].getFlags() & StreamFlag::External) {
+		/*
+		 * Since the ISP runs synchronous with the IPA and requests,
+		 * we only ever need a maximum of one internal buffer. Any
+		 * buffers the application wants to hold onto will already
+		 * be exported through PipelineHandlerRPi::exportFrameBuffers().
+		 *
+		 * However, as above, if the application provides a guarantee
+		 * that the buffer will always be provided for the ISP Output0
+		 * stream in a Request, we don't need any internal buffers
+		 * allocated.
+		 */
+		if (!data->dropFrameCount_ && data->config_.output0MandatoryStream)
+			minIspBuffers = 0;
+	}
+
 	/* Decide how many internal buffers to allocate. */
 	for (auto const stream : data->streams_) {
 		unsigned int numBuffers;
 		/*
 		 * For Unicam, allocate a minimum number of buffers for internal
 		 * use as we want to avoid any frame drops.
 		 */
-		const unsigned int minBuffers = data->config_.minTotalUnicamBuffers;
 		if (stream == &data->unicam_[Unicam::Image]) {
 			/*
 			 * If an application has configured a RAW stream, allocate
 			 * additional buffers to make up the minimum, but ensure
 			 * we have at least minUnicamBuffers of internal buffers
 			 * to use to minimise frame drops.
 			 */
-			numBuffers = std::max<int>(data->config_.minUnicamBuffers,
-						   minBuffers - numRawBuffers);
+			numBuffers = std::max<int>(minUnicamBuffers,
+						   minTotalUnicamBuffers - numRawBuffers);
+			LOG(RPI, Debug) << "Unicam::Image numBuffers " << numBuffers;
 		} else if (stream == &data->isp_[Isp::Input]) {
 			/*
 			 * ISP input buffers are imported from Unicam, so follow
 			 * similar logic as above to count all the RAW buffers
 			 * available.
 			 */
 			numBuffers = numRawBuffers +
-				     std::max<int>(data->config_.minUnicamBuffers,
-						   minBuffers - numRawBuffers);
+				     std::max<int>(minUnicamBuffers,
+						   minTotalUnicamBuffers - numRawBuffers);
+			LOG(RPI, Debug) << "Isp::Input numBuffers " << numBuffers;
 
 		} else if (stream == &data->unicam_[Unicam::Embedded]) {
 			/*
@@ -273,14 +315,18 @@ int PipelineHandlerVc4::prepareBuffers(Camera *camera)
 			 * buffers, as these will be recycled quicker.
 			 */
 			numBuffers = 12;
+		} else if (stream == &data->isp_[Isp::Output0]) {
+			/* Buffer count for this is handled in the earlier loop above. */
+			numBuffers = minIspBuffers;
+			LOG(RPI, Debug) << "Isp::Output0 numBuffers " << numBuffers;
 		} else {
 			/*
-			 * Since the ISP runs synchronous with the IPA and requests,
-			 * we only ever need one set of internal buffers. Any buffers
-			 * the application wants to hold onto will already be exported
-			 * through PipelineHandlerRPi::exportFrameBuffers().
+			 * Same reasoning as for ISP Output 0, we only ever need
+			 * a maximum of one internal buffer for Output1 (required
+			 * for colour denoise) and ISP statistics.
 			 */
 			numBuffers = 1;
+			LOG(RPI, Debug) << "Other numBuffers " << numBuffers;
 		}
 
 		ret = stream->prepareBuffers(numBuffers);
@@ -484,6 +530,8 @@ int Vc4CameraData::platformPipelineConfigure(const std::unique_ptr<YamlObject> &
 	config_ = {
 		.minUnicamBuffers = 2,
 		.minTotalUnicamBuffers = 4,
+		.rawMandatoryStream = false,
+		.output0MandatoryStream = false,
 	};
 
 	if (!root)
@@ -507,6 +555,10 @@ int Vc4CameraData::platformPipelineConfigure(const std::unique_ptr<YamlObject> &
 		phConfig["min_unicam_buffers"].get<unsigned int>(config_.minUnicamBuffers);
 	config_.minTotalUnicamBuffers =
 		phConfig["min_total_unicam_buffers"].get<unsigned int>(config_.minTotalUnicamBuffers);
+	config_.rawMandatoryStream =
+		phConfig["raw_mandatory_stream"].get<bool>(config_.rawMandatoryStream);
+	config_.output0MandatoryStream =
+		phConfig["output0_mandatory_stream"].get<bool>(config_.output0MandatoryStream);
 
 	if (config_.minTotalUnicamBuffers < config_.minUnicamBuffers) {
 		LOG(RPI, Error) << "Invalid configuration: min_total_unicam_buffers must be >= min_unicam_buffers";