Implement RunPipelineQueue function to manage worker execution and pipeline processing

Author: jravani

internal/process/worker.go

@@ -28,3 +28,26 @@ func (w *Worker) Start(pipelineChan <-chan *Pipeline, conn *pgxpool.Pool, wg *sy
 		}
 	}
 }
+
+// RunPipelineQueue drains a PipelineQueue into a channel and runs it with workers.
+func RunPipelineQueue(queue *PipelineQueue, conn *pgxpool.Pool, cfg *config.Config) error {
+	if queue.IsEmpty() {
+		return nil
+	}
+
+	pipChan := make(chan *Pipeline, queue.Len())
+	for !queue.IsEmpty() {
+		if p := queue.Dequeue(); p != nil {
+			pipChan <- p
+		}
+	}
+	close(pipChan)
+
+	var wg sync.WaitGroup
+	for i := 1; i <= cfg.Batch.Threads; i++ {
+		wg.Add(1)
+		go NewWorker(i).Start(pipChan, conn, &wg, cfg)
+	}
+	wg.Wait()
+	return nil
+}