pipeline: rpi: Report which control lists apply to each request

This forms part of Raspberry Pi's implementation of "per frame
control".

Control lists that are queued to the pipeline handler through
Camera::queueControls are assigned a sequence number and get applied
to the next possible request. They are tracked through the pipeline
using "delayed contexts". Finally, the control list sequence number
that is applied to the request is reported back through the
"ControlListSequence" metadata.

Signed-off-by: David Plowman <david.plowman@raspberrypi.com>

Author: davidplowman

src/ipa/rpi/common/ipa_base.cpp

@@ -440,6 +440,9 @@ void IpaBase::prepareIsp(const PrepareParams &params)
 	fillDeviceStatus(params.sensorControls, ipaContext);
 	fillSyncParams(params, ipaContext);
 
+	if (!params.requestControls.empty())
+		rpiMetadata.set("ipa.request_controls", true);
+
 	if (params.buffers.embedded) {
 		/*
 		 * Pipeline handler has supplied us with an embedded data buffer,
@@ -460,7 +463,7 @@ void IpaBase::prepareIsp(const PrepareParams &params)
 	 */
 	AgcStatus agcStatus;
 	bool hdrChange = false;
-	RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext];
+	RPiController::Metadata &delayedMetadata = rpiMetadata_[params.delayContext % rpiMetadata_.size()];
 	if (!delayedMetadata.get<AgcStatus>("agc.status", agcStatus)) {
 		rpiMetadata.set("agc.delayed_status", agcStatus);
 		hdrChange = agcStatus.hdr.mode != hdrStatus_.mode;
@@ -473,9 +476,13 @@ void IpaBase::prepareIsp(const PrepareParams &params)
 	 */
 	helper_->prepare(embeddedBuffer, rpiMetadata);
 
+	bool delayedRequestControls = false;
+	delayedMetadata.get<bool>("ipa.request_controls", delayedRequestControls);
+
 	/* Allow a 10% margin on the comparison below. */
 	Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns;
-	if (lastRunTimestamp_ && frameCount_ > invalidCount_ &&
+	if (!delayedRequestControls && params.requestControls.empty() &&
+	    lastRunTimestamp_ && frameCount_ > invalidCount_ &&
 	    delta < controllerMinFrameDuration * 0.9 && !hdrChange) {
 		/*
 		 * Ensure we merge the previous frame's metadata with the current
@@ -558,7 +565,7 @@ void IpaBase::processStats(const ProcessParams &params)
 		ControlList ctrls(sensorCtrls_);
 		applyAGC(&agcStatus, ctrls, offset);
 		rpiMetadata.set("agc.status", agcStatus);
-		setDelayedControls.emit(ctrls, ipaContext);
+		setDelayedControls.emit(ctrls, params.ipaContext);
 		setCameraTimeoutValue();
 	}
 
@@ -975,8 +982,6 @@ void IpaBase::applyControls(const ControlList &controls)
 
 			/* The control provides units of microseconds. */
 			agc->setFixedExposureTime(0, ctrl.second.get<int32_t>() * 1.0us);
-
-			libcameraMetadata_.set(controls::ExposureTime, ctrl.second.get<int32_t>());
 			break;
 		}
 
@@ -1000,9 +1005,6 @@ void IpaBase::applyControls(const ControlList &controls)
 				break;
 
 			agc->setFixedGain(0, ctrl.second.get<float>());
-
-			libcameraMetadata_.set(controls::AnalogueGain,
-					       ctrl.second.get<float>());
 			break;
 		}
 

src/libcamera/pipeline/rpi/common/pipeline_base.cpp

@@ -1531,4 +1531,88 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request
 	}
 }
 
+static bool isControlDelayed(unsigned int id)
+{
+	return id == controls::ExposureTime ||
+	       id == controls::AnalogueGain ||
+	       id == controls::FrameDurationLimits ||
+	       id == controls::AeEnable ||
+	       id == controls::ExposureTimeMode ||
+	       id == controls::AnalogueGainMode;
+}
+
+void CameraData::handleControlLists(uint32_t delayContext)
+{
+	/*
+	 * If any control lists have been queued for us, merge them into the
+	 * next request that we're going to process. This route saves controls
+	 * from being queued always with the last request, meaning they get
+	 * picked up here more quickly.
+	 */
+	Request *request = requestQueue_.front();
+	if (!controlsQueue_.empty()) {
+		request->controls().merge(std::move(controlsQueue_.front()),
+					  ControlList::MergePolicy::OverwriteExisting);
+		controlsQueue_.pop();
+		controlListId_++;
+		LOG(RPI, Debug) << "Popped control list " << controlListId_ << " from queue";
+	}
+
+	/*
+	 * Record which control list corresponds to this ipaCookie. Because setDelayedControls
+	 * now gets called by the IPA from the start of the following frame, we must record
+	 * the previous control list id.
+	 */
+	syncTable_.emplace(SyncTableEntry{ request->sequence(), controlListId_ });
+	LOG(RPI, Debug) << "Add sync table entry: sequence " << request->sequence()
+			<< " control list id " << controlListId_;
+
+	/*
+	 * We know we that we added an entry for every delayContext, so we can
+	 * find the one for this Bayer frame, and this links us to the correct
+	 * control list. Anything ahead of "our" entry in the queue is old, so
+	 * can be dropped.
+	 */
+	while (!syncTable_.empty() &&
+	       syncTable_.front().ipaCookie != delayContext) {
+		LOG(RPI, Debug) << "Pop sync entry: ipa cookie "
+				<< syncTable_.front().ipaCookie << " control id "
+				<< syncTable_.front().controlListId << " for job "
+				<< delayContext;
+		syncTable_.pop();
+	}
+
+	if (syncTable_.empty())
+		LOG(RPI, Warning) << "Unable to find ipa cookie for PFC";
+	else {
+		LOG(RPI, Debug) << "Using sync control id " << syncTable_.front().controlListId;
+		requestControlId_ = syncTable_.front().controlListId;
+	}
+
+	/* The control list id is reported as the "ControlListSequence" in the metadata. */
+	request->_d()->metadata().set(controls::rpi::ControlListSequence, requestControlId_);
+
+	/*
+	 * Controls that take effect immediately (typically ISP controls) have to be
+	 * delayed so as to synchronise with those controls that do get delayed. So we
+	 * must remove them from the current request, and push them onto a queue so
+	 * that they can be used later.
+	 */
+	ControlList controls = std::move(request->controls());
+	immediateControls_.push({ controlListId_, request->controls() });
+	for (const auto &ctrl : controls) {
+		if (isControlDelayed(ctrl.first))
+			request->controls().set(ctrl.first, ctrl.second);
+		else
+			immediateControls_.back().controls.set(ctrl.first, ctrl.second);
+	}
+	/* "Immediate" controls that have become due are now merged back into this request. */
+	while (!immediateControls_.empty() &&
+	       immediateControls_.front().controlListId <= requestControlId_) {
+		request->controls().merge(immediateControls_.front().controls,
+					  ControlList::MergePolicy::OverwriteExisting);
+		immediateControls_.pop();
+	}
+}
+
 } /* namespace libcamera */

src/libcamera/pipeline/rpi/common/pipeline_base.h

@@ -48,7 +48,7 @@ class CameraData : public Camera::Private
 {
 public:
 	CameraData(PipelineHandler *pipe)
-		: Camera::Private(pipe), state_(State::Stopped),
+		: Camera::Private(pipe), state_(State::Stopped), controlListId_(0),
 		  startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false)
 	{
 	}
@@ -131,6 +131,8 @@ class CameraData : public Camera::Private
 
 	std::queue<Request *> requestQueue_;
 	std::queue<ControlList> controlsQueue_;
+	uint64_t controlListId_;
+	uint64_t requestControlId_;
 
 	/* For handling digital zoom. */
 	IPACameraSensorInfo sensorInfo_;
@@ -176,10 +178,24 @@ class CameraData : public Camera::Private
 
 	ClockRecovery wallClockRecovery_;
 
+	struct SyncTableEntry {
+		uint32_t ipaCookie;
+		uint64_t controlListId;
+	};
+	std::queue<SyncTableEntry> syncTable_;
+
+	struct ImmediateControlsEntry {
+		uint64_t controlListId;
+		ControlList controls;
+	};
+	std::queue<ImmediateControlsEntry> immediateControls_;
+
 protected:
 	void fillRequestMetadata(const ControlList &bufferControls,
 				 Request *request);
 
+	void handleControlLists(uint32_t delayContext);
+
 	virtual void tryRunPipeline() = 0;
 
 private:

src/libcamera/pipeline/rpi/pisp/pisp.cpp

@@ -2311,6 +2311,9 @@ void PiSPCameraData::tryRunPipeline()
 
 	fillRequestMetadata(job.sensorControls, request);
 
+	/* This sorts out synchronisation with the ControlList queue. */
+	handleControlLists(job.delayContext);
+
 	/* Set our state to say the pipeline is active. */
 	state_ = State::Busy;
 
@@ -2327,7 +2330,7 @@ void PiSPCameraData::tryRunPipeline()
 	params.buffers.bayer = RPi::MaskBayerData | bayerId;
 	params.buffers.stats = RPi::MaskStats | statsId;
 	params.buffers.embedded = 0;
-	params.ipaContext = requestQueue_.front()->sequence();
+	params.ipaContext = request->sequence();
 	params.delayContext = job.delayContext;
 	params.sensorControls = std::move(job.sensorControls);
 	params.requestControls = request->controls();

src/libcamera/pipeline/rpi/vc4/vc4.cpp

@@ -984,6 +984,9 @@ void Vc4CameraData::tryRunPipeline()
 
 	fillRequestMetadata(bayerFrame.controls, request);
 
+	/* This sorts out synchronisation with the ControlList queue. */
+	handleControlLists(bayerFrame.delayContext);
+
 	/* Set our state to say the pipeline is active. */
 	state_ = State::Busy;