WIP: refactor

Author: tellet-q

.github/workflows/continuous-benchmark-transfer.yaml

@@ -8,7 +8,7 @@ on:
         default: 'laion-small-clip'
         type: string
       qdrant_versions:
-        description: 'Comma-separated versions to compare (ghcr/dev, docker/master, docker/v1.13.0)'
+        description: 'Comma-separated versions to compare (ghcr/dev, docker/master, docker/v1.13.0). Can be just single version.'
         default: 'ghcr/dev,docker/v1.16.3'
         type: string
       region:

ansible/playbooks/roles/run-transfer-speed/files/shard_transfer.py

@@ -81,15 +81,18 @@ def storage_types(self, uri: str) -> dict:
         return {}
 
     def wait_mmap(self, uri: str, timeout=180):
+        print(f"Waiting for Mmap segments...")
         start = time.time()
+        types = {}
         while time.time() - start < timeout:
             types = self.storage_types(uri)
             mmap = types.get("Mmap", 0)
             chunked = types.get("ChunkedMmap", 0) + types.get("InRamChunkedMmap", 0)
+            print(f"  Storage types: {types}")
             if mmap > 0 and mmap >= chunked:
                 return
-            time.sleep(2)
-        raise TimeoutError("Mmap segments timeout")
+            time.sleep(5)
+        print(f"Warning: Mmap timeout, continuing anyway. Types: {types}")
 
     def wait_transfer(self, timeout=600):
         start = time.time()

ansible/playbooks/roles/run-transfer-speed/tasks/main.yml

@@ -1,4 +1,9 @@
 ---
+- name: Install python3-venv
+  ansible.builtin.package:
+    name: python3-venv
+    state: present
+
 - name: Setup
   ansible.builtin.shell: mkdir -p {{ working_dir }}/data/{{ dataset_name }}
   args:
@@ -18,19 +23,23 @@
     - {src: "run-bench.sh", dest: "{{ working_dir }}/run-bench.sh", mode: '0755'}
     - {src: "{{ playbook_dir }}/files/transfer-speed/requirements.txt", dest: "{{ working_dir }}/requirements.txt"}
 
+- name: Check dataset
+  stat:
+    path: "{{ working_dir }}/data/{{ dataset_name }}/vectors.npy"
+  register: vectors_file
+
 - name: Download dataset
   ansible.builtin.get_url:
     url: "{{ dataset.link }}"
     dest: "{{ working_dir }}/data/{{ dataset_name }}.tgz"
-  args:
-    creates: "{{ working_dir }}/data/{{ dataset_name }}.tgz"
+  when: not vectors_file.stat.exists
 
 - name: Extract dataset
   ansible.builtin.unarchive:
     src: "{{ working_dir }}/data/{{ dataset_name }}.tgz"
     dest: "{{ working_dir }}/data/{{ dataset_name }}"
     remote_src: yes
-    creates: "{{ working_dir }}/data/{{ dataset_name }}/vectors.npy"
+  when: not vectors_file.stat.exists
 
 - name: Run benchmark
   ansible.builtin.shell: "{{ working_dir }}/run-bench.sh"