WIP

Author: tellet-q

.github/workflows/continuous-benchmark-transfer.yaml

@@ -7,9 +7,9 @@ on:
         description: 'Dataset name'
         default: 'dbpedia-openai-1M-1536-angular'
         type: string
-      qdrant_version:
-        description: 'Version of qdrant to benchmark (ghcr/dev, docker/v1.7.0)'
-        default: 'ghcr/dev'
+      qdrant_versions:
+        description: 'Comma-separated versions (ghcr/dev, docker/master, docker/v1.13.0)'
+        default: 'ghcr/dev,docker/master'
         type: string
       region:
         description: 'Hetzner region'
@@ -19,11 +19,18 @@ on:
         description: 'Hetzner server type'
         default: 'cpx41'
         type: string
-  schedule:
-    - cron: "0 4 * * *"  # Daily at 4am UTC
+      client_type:
+        description: 'Hetzner client type'
+        default: 'cpx32'
+        type: string
+  push:
+    branches:
+      - transfer-speed-benchmark
+#  schedule:
+#    - cron: "0 4 * * *"  # Daily at 4am UTC
 
 concurrency:
-  group: transfer-benchmark
+  group: hetzner-machines
 
 env:
   HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
@@ -69,7 +76,7 @@ jobs:
         uses: ./.github/workflows/actions/create-server-with-retry
         with:
           server_name: transfer-bench-client-${{ github.run_id }}
-          server_type: cpx32
+          server_type: ${{ inputs.client_type }}
           region: ${{ inputs.region }}
 
   runBenchmark:
@@ -82,44 +89,49 @@ jobs:
       - uses: webfactory/ssh-agent@v0.8.0
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-      - name: Parse qdrant version
-        id: parse_version
-        run: |
-          QDRANT_VERSION="${{ inputs.qdrant_version }}"
-
-          case "${QDRANT_VERSION}" in
-            docker/*)
-              CONTAINER_REGISTRY="docker.io"
-              VERSION=${QDRANT_VERSION#docker/}
-              ;;
-            ghcr/*)
-              CONTAINER_REGISTRY="ghcr.io"
-              VERSION=${QDRANT_VERSION#ghcr/}
-              ;;
-            *)
-              echo "Error: unknown version ${QDRANT_VERSION}. Version name should start with 'docker/' or 'ghcr/'"
-              exit 1
-              ;;
-          esac
-
-          echo "registry=${CONTAINER_REGISTRY}" >> $GITHUB_OUTPUT
-          echo "version=${VERSION}" >> $GITHUB_OUTPUT
       - name: Create inventory
         uses: ./.github/workflows/actions/create-inventory
         with:
           hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
           server_names: ${{ needs.setupCluster.outputs.node_names }}
           client_names: ${{ needs.setupCluster.outputs.client_name }}
           db_host: ${{ secrets.POSTGRES_HOST }}
-      - name: Run bench
+      - name: Run benchmarks for all versions
         run: |
-          cd ansible/playbooks
-          ansible-playbook playbook-transfer-speed.yml \
-            -i inventory.ini \
-            --extra-vars "
-              dataset_name=${{ inputs.dataset_name }}
-              servers=[{'name':'qdrant','registry':'${{ steps.parse_version.outputs.registry }}','image':'qdrant/qdrant','version':'${{ steps.parse_version.outputs.version }}'}]
-            "
+          VERSIONS="${{ inputs.qdrant_versions }}"
+          VERSIONS="${VERSIONS:-ghcr/dev,docker/latest}"
+
+          IFS=',' read -ra VERSION_ARRAY <<< "$VERSIONS"
+          for QDRANT_VERSION in "${VERSION_ARRAY[@]}"; do
+            QDRANT_VERSION=$(echo "$QDRANT_VERSION" | xargs)  # trim whitespace
+            echo "=========================================="
+            echo "Benchmarking Qdrant version: $QDRANT_VERSION"
+            echo "=========================================="
+
+            case "${QDRANT_VERSION}" in
+              docker/*)
+                CONTAINER_REGISTRY="docker.io"
+                VERSION=${QDRANT_VERSION#docker/}
+                ;;
+              ghcr/*)
+                CONTAINER_REGISTRY="ghcr.io"
+                VERSION=${QDRANT_VERSION#ghcr/}
+                ;;
+              *)
+                echo "Error: unknown version ${QDRANT_VERSION}. Version name should start with 'docker/' or 'ghcr/'"
+                exit 1
+                ;;
+            esac
+
+            cd ansible/playbooks
+            ansible-playbook playbook-transfer-speed.yml \
+              -i inventory.ini \
+              --extra-vars "
+                dataset_name=${{ inputs.dataset_name }}
+                servers=[{'name':'qdrant','registry':'${CONTAINER_REGISTRY}','image':'qdrant/qdrant','version':'${VERSION}'}]
+              "
+            cd ../..
+          done
         env:
           ANSIBLE_HOST_KEY_CHECKING: "False"
 

.github/workflows/manual-benchmarks-cascade.yaml

@@ -40,6 +40,10 @@ on:
         server_machine_type:
             description: "Hetzner server machine type to run the benchmarks in"
             default: "cpx41"
+
+concurrency:
+  group: hetzner-machines
+
 env:
   # Common environment variables
   HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}