WIP: try different machines

Author: tellet-q

.github/workflows/continuous-benchmark-transfer.yaml

@@ -17,11 +17,11 @@ on:
         type: string
       server_type:
         description: 'Hetzner server type'
-        default: 'cpx42'
+        default: 'ccx13'
         type: string
       client_type:
         description: 'Hetzner client type'
-        default: 'cpx42'
+        default: 'cx32'
         type: string
   push:
     branches:

ansible/playbooks/roles/run-transfer-speed/files/shard_transfer.py

@@ -88,8 +88,8 @@ def setup_collection(self, dims: int):
         self.primary.create_collection(
             COLLECTION_NAME,
             vectors_config=models.VectorParams(size=dims, distance=models.Distance.COSINE, on_disk=True),
-            # Set to 0 to disable automatic indexing during upload
-            optimizers_config=models.OptimizersConfigDiff(indexing_threshold=0),
+            # Disable optimization threads during upload for faster ingestion
+            optimizers_config=models.OptimizersConfigDiff(max_optimization_threads=0),
         )
 
     def upload_vectors(self, vectors: np.ndarray):
@@ -98,10 +98,13 @@ def upload_vectors(self, vectors: np.ndarray):
             collection_name=COLLECTION_NAME,
             vectors=vectors,
             ids=range(len(vectors)),
+            batch_size=1024,
+            parallel=16,
         )
+        # Re-enable optimization after upload
         self.primary.update_collection(
             collection_name=COLLECTION_NAME,
-            optimizer_config=models.OptimizersConfigDiff(indexing_threshold=1),
+            optimizer_config=models.OptimizersConfigDiff(max_optimization_threads=100_000),
         )
 
     def wait_for_green(self, timeout: int = 1800):