WIP: try different machines

Author: tellet-q

.github/workflows/continuous-benchmark-transfer.yaml

@@ -35,8 +35,8 @@ concurrency:
 env:
   HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
   # Defaults for push trigger (inputs.* is empty on push)
-  SERVER_TYPE: ${{ inputs.server_type || 'cpx42' }}
-  CLIENT_TYPE: ${{ inputs.client_type || 'cpx42' }}
+  SERVER_TYPE: ${{ inputs.server_type || 'ccx13' }}
+  CLIENT_TYPE: ${{ inputs.client_type || 'cx32' }}
   REGION: ${{ inputs.region || 'fsn1' }}
   DATASET_NAME: ${{ inputs.dataset_name || 'dbpedia-openai-100K-1536-angular' }}
   QDRANT_VERSIONS: ${{ inputs.qdrant_versions || 'ghcr/dev,docker/master' }}

ansible/playbooks/files/transfer-speed/docker-compose-cluster.yaml

@@ -1,7 +1,7 @@
 services:
   qdrant:
     image: ${CONTAINER_REGISTRY}/qdrant/qdrant:${QDRANT_VERSION}
-    container_name: qdrant-node
+    container_name: qdrant-continuous
     restart: unless-stopped
     ports:
       - "6333:6333"

ansible/playbooks/playbook-transfer-speed.yml

@@ -48,7 +48,7 @@
 #        dataset_name=$(echo "$results" | jq -r '.params.dataset')
 #        measure_timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
 #
-#        pg_query="INSERT INTO transfer_speed (
+#        pg_query="INSERT INTO benchmark_shard_transfer (
 #            engine, engine_version, dataset, measure_timestamp, throughput_pts_s, duration_s
 #        ) VALUES (
 #            '{{ server.name }}', '{{ server.version }}', '\${dataset_name}', '\${measure_timestamp}', \${throughput_pts_s}, \${duration_s}

ansible/playbooks/roles/run-transfer-speed/files/shard_transfer.py

@@ -88,8 +88,12 @@ def setup_collection(self, dims: int):
         self.primary.create_collection(
             COLLECTION_NAME,
             vectors_config=models.VectorParams(size=dims, distance=models.Distance.COSINE, on_disk=True),
-            # Disable optimization threads during upload for faster ingestion
-            optimizers_config=models.OptimizersConfigDiff(max_optimization_threads=0),
+            # Match framework settings - allow optimization during upload
+            optimizers_config=models.OptimizersConfigDiff(
+                default_segment_number=3,
+                max_segment_size=1_000_000,
+                memmap_threshold=10_000_000,
+            ),
         )
 
     def upload_vectors(self, vectors: np.ndarray):
@@ -101,22 +105,24 @@ def upload_vectors(self, vectors: np.ndarray):
             batch_size=1024,
             parallel=16,
         )
-        # Re-enable optimization after upload
-        self.primary.update_collection(
-            collection_name=COLLECTION_NAME,
-            optimizer_config=models.OptimizersConfigDiff(max_optimization_threads=100_000),
-        )
 
     def wait_for_green(self, timeout: int = 1800):
         print("Waiting for green status...", end="", flush=True)
-        start = time.time()
-        while time.time() - start < timeout:
+        wait_time = 5.0
+        total = 0
+        while total < timeout:
+            time.sleep(wait_time)
+            total += wait_time
+            info = self.primary.get_collection(COLLECTION_NAME)
+            if info.status != models.CollectionStatus.GREEN:
+                print(".", end="", flush=True)
+                continue
+            # Double-check: status can briefly flip to GREEN during optimization
+            time.sleep(wait_time)
             info = self.primary.get_collection(COLLECTION_NAME)
             if info.status == models.CollectionStatus.GREEN:
-                print(f" done ({time.time() - start:.1f}s)")
+                print(f" done ({total:.1f}s)")
                 return
-            time.sleep(1)
-            print(".", end="", flush=True)
         print(f" timeout after {timeout}s")
 
     def get_collection_cluster_info(self) -> dict:

ansible/playbooks/roles/setup-qdrant-cluster/tasks/main.yml

@@ -17,7 +17,6 @@
 - name: Stop existing containers
   ansible.builtin.shell: |
     cd {{ working_dir }} && docker compose down 2>/dev/null || true
-    docker rm -f qdrant-node 2>/dev/null || true
     docker rm -f qdrant-continuous 2>/dev/null || true
     # Kill any container using port 6333
     docker ps -q --filter "publish=6333" | xargs -r docker rm -f 2>/dev/null || true