fix: add a maxsize argumen to the saver queue to make sure we don't overload the ram and crash

thiswillbeyourgithub · thiswillbeyourgithub · commit bfd4dd0edf3e · 2024-06-29T13:30:06.000+02:00
diff --git a/DocToolsLLM/utils/embeddings.py b/DocToolsLLM/utils/embeddings.py
@@ -322,7 +322,7 @@ def load_embeddings(
                 for i in range(len(to_embed) // batch_size + 1)
                 ]
         n_saver = 10
-        saver_queues = [(queue.Queue(), queue.Queue()) for i in range(n_saver)]
+        saver_queues = [(queue.Queue(maxsize=10), queue.Queue()) for i in range(n_saver)]
         saver_workers = [
                 threading.Thread(
                     target=faiss_saver,

Original file line number	Diff line number	Diff line change
`@@ -322,7 +322,7 @@ def load_embeddings(`
`322`	`322`	`for i in range(len(to_embed) // batch_size + 1)`
`323`	`323`	`]`
`324`	`324`	`n_saver = 10`
`325`		`- saver_queues = [(queue.Queue(), queue.Queue()) for i in range(n_saver)]`
	`325`	`+ saver_queues = [(queue.Queue(maxsize=10), queue.Queue()) for i in range(n_saver)]`
`326`	`326`	`saver_workers = [`
`327`	`327`	`threading.Thread(`
`328`	`328`	`target=faiss_saver,`