diff --git a/drivers/media/platform/raspberrypi/pisp_be/pisp_be.c b/drivers/media/platform/raspberrypi/pisp_be/pisp_be.c
index 7dce1f83de4b18..2db4211b2b35a1 100644
--- a/drivers/media/platform/raspberrypi/pisp_be/pisp_be.c
+++ b/drivers/media/platform/raspberrypi/pisp_be/pisp_be.c
@@ -594,9 +594,7 @@ static int pispbe_prepare_job(struct pispbe_node_group *node_group)
 	return -ENODEV;
 }
 
-static void pispbe_schedule(struct pispbe_dev *pispbe,
-			    struct pispbe_node_group *node_group,
-			    bool clear_hw_busy)
+static void pispbe_schedule(struct pispbe_dev *pispbe, bool clear_hw_busy)
 {
 	struct pispbe_job_descriptor *job;
 
@@ -613,9 +611,6 @@ static void pispbe_schedule(struct pispbe_dev *pispbe,
 		if (!job)
 			return;
 
-		if (node_group && job->node_group != node_group)
-			continue;
-
 		list_del(&job->queue);
 
 		for (unsigned int i = 0; i < PISPBE_NUM_NODES; i++)
@@ -703,7 +698,7 @@ static irqreturn_t pispbe_isr(int irq, void *dev)
 	}
 
 	/* check if there's more to do before going to sleep */
-	pispbe_schedule(pispbe, NULL, can_queue_another);
+	pispbe_schedule(pispbe, can_queue_another);
 
 	return IRQ_HANDLED;
 }
@@ -894,7 +889,7 @@ static void pispbe_node_buffer_queue(struct vb2_buffer *buf)
 	 * to do, but only for this client.
 	 */
 	if (!pispbe_prepare_job(node_group))
-		pispbe_schedule(pispbe, node_group, false);
+		pispbe_schedule(pispbe, false);
 }
 
 static int pispbe_node_start_streaming(struct vb2_queue *q, unsigned int count)
@@ -921,7 +916,7 @@ static int pispbe_node_start_streaming(struct vb2_queue *q, unsigned int count)
 
 	/* Maybe we're ready to run. */
 	if (!pispbe_prepare_job(node_group))
-		pispbe_schedule(pispbe, node_group, false);
+		pispbe_schedule(pispbe, false);
 
 	return 0;
 
@@ -941,7 +936,6 @@ static void pispbe_node_stop_streaming(struct vb2_queue *q)
 	struct pispbe_dev *pispbe = node_group->pispbe;
 	struct pispbe_job_descriptor *job, *temp;
 	struct pispbe_buffer *buf;
-	LIST_HEAD(tmp_list);
 
 	/*
 	 * Now this is a bit awkward. In a simple M2M device we could just wait
@@ -968,20 +962,18 @@ static void pispbe_node_stop_streaming(struct vb2_queue *q)
 	spin_lock_irq(&pispbe->hw_lock);
 	node_group->streaming_map &= ~BIT(node->id);
 
-	if (node_group->streaming_map == 0) {
-		/*
-		 * If all nodes have stopped streaming release all jobs
-		 * without holding the lock.
-		 */
-		list_splice_init(&pispbe->job_queue, &tmp_list);
+	/*
+	 * If a node has stopped streaming release all jobs belonging to the
+	 * node group immediately.
+	 */
+	list_for_each_entry_safe(job, temp, &pispbe->job_queue, queue) {
+		if (job->node_group == node->node_group) {
+			list_del(&job->queue);
+			kfree(job);
+		}
 	}
 	spin_unlock_irq(&pispbe->hw_lock);
 
-	list_for_each_entry_safe(job, temp, &tmp_list, queue) {
-		list_del(&job->queue);
-		kfree(job);
-	}
-
 	pm_runtime_mark_last_busy(pispbe->dev);
 	pm_runtime_put_autosuspend(pispbe->dev);