WIP: debug

Author: tellet-q

.github/workflows/continuous-benchmark-transfer.yaml

@@ -9,7 +9,7 @@ on:
         type: string
       qdrant_versions:
         description: 'Comma-separated versions (ghcr/dev, docker/master, docker/v1.13.0)'
-        default: 'ghcr/dev,docker/master'
+        default: 'ghcr/dev'
         type: string
       region:
         description: 'Hetzner region'
@@ -39,7 +39,7 @@ env:
   CLIENT_TYPE: ${{ inputs.client_type || 'cx33' }}
   REGION: ${{ inputs.region || 'fsn1' }}
   DATASET_NAME: ${{ inputs.dataset_name || 'dbpedia-openai-100K-1536-angular' }}
-  QDRANT_VERSIONS: ${{ inputs.qdrant_versions || 'ghcr/dev,docker/master' }}
+  QDRANT_VERSIONS: ${{ inputs.qdrant_versions || 'ghcr/dev' }}
 
 jobs:
   setupCluster:
@@ -128,6 +128,7 @@ jobs:
         env:
           ANSIBLE_HOST_KEY_CHECKING: "False"
           ANSIBLE_SSH_ARGS: "-o ServerAliveInterval=30 -o ServerAliveCountMax=10 -o ControlMaster=no"
+          ANSIBLE_STDOUT_CALLBACK: yaml
           QDRANT_VERSIONS: ${{ env.QDRANT_VERSIONS }}
           DATASET_NAME: ${{ env.DATASET_NAME }}
 

ansible/playbooks/roles/run-transfer-speed/files/shard_transfer.py

@@ -107,23 +107,30 @@ def upload_vectors(self, vectors: np.ndarray):
         )
 
     def wait_for_green(self, timeout: int = 1800):
-        print("Waiting for green status...", end="", flush=True)
+        print("Waiting for green status...")
         wait_time = 5.0
         total = 0
+        info = None
         while total < timeout:
             time.sleep(wait_time)
             total += wait_time
             info = self.primary.get_collection(COLLECTION_NAME)
-            if info.status != models.CollectionStatus.GREEN:
-                print(".", end="", flush=True)
+            status = info.status
+            indexed = info.indexed_vectors_count
+            total_vectors = info.vectors_count
+            segments = info.segments_count
+            print(f"  [{total:.0f}s] status={status}, indexed={indexed}/{total_vectors}, segments={segments}")
+
+            if status != models.CollectionStatus.GREEN:
                 continue
             # Double-check: status can briefly flip to GREEN during optimization
             time.sleep(wait_time)
             info = self.primary.get_collection(COLLECTION_NAME)
             if info.status == models.CollectionStatus.GREEN:
-                print(f" done ({total:.1f}s)")
+                print(f"Collection is GREEN after {total:.1f}s")
                 return
-        print(f" timeout after {timeout}s")
+        final_status = info.status if info else 'unknown'
+        print(f"WARNING: timeout after {timeout}s, status={final_status}")
 
     def get_collection_cluster_info(self) -> dict:
         return self.primary.http.distributed_api.collection_cluster_info(COLLECTION_NAME).dict()["result"]

ansible/playbooks/roles/run-transfer-speed/tasks/main.yml

@@ -56,13 +56,14 @@
 
 - name: Run transfer benchmark
   ansible.builtin.shell: |
-    {{ working_dir }}/run-bench.sh
+    {{ working_dir }}/run-bench.sh 2>&1
   environment:
     QDRANT_URIS: "{{ qdrant_uris }}"
     DATASET_NAME: "{{ dataset_name }}"
     RUNS: "{{ runs }}"
     OUTPUT_FILENAME: "{{ working_dir }}/results.json"
     WORK_DIR: "{{ working_dir }}"
+    PYTHONUNBUFFERED: "1"
   register: benchmark_result
 
 - name: Display benchmark output