manual-benchmark.yaml

name: Manual Benchmark

on:
  workflow_dispatch:
    inputs:
        qdrant_version:
            description: "Version of qdrant to benchmark (tags/v1.6.1, <commit-id>, my-branch, docker/v1.5.1, ghcr/dev)"
            default: ghcr/dev
        dataset:
            description: "Dataset to benchmark"
            default: laion-small-clip
        engine_config:
            description: "Engine config to benchmark"
            default: qdrant-continuous-benchmark
        feature_flags_all:
            type: boolean
            description: "Enable all feature flags (false by default)"
            default: false
        build_vector_db_image:
            required: false
            description: "Build vector DB image from source branch and use it (false by default)"
            default: "false"

# Restrict to only running workflows of this group one at a time.
# Any new runs will be queued until the previous run is complete.
# Any existing pending runs will be cancelled and replaced with current run.
concurrency:
  group: continuous-benchmark

jobs:
  runManualBenchmark:
    name: manual benchmark - ${{ inputs.qdrant_version }} - ${{ inputs.dataset }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
      - uses: webfactory/ssh-agent@d4b9b8ff72958532804b70bbe600ad43b36d5f2e # v0.8.0
        with:
          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
      - name: prepare image tag
        id: prepare-tag
        if: ${{ inputs.build_vector_db_image == 'true' }}
        shell: bash
        run: |
          branch_tmp=$(git rev-parse --abbrev-ref HEAD)
          branch=${branch_tmp//\//-} # replace all / with -
          tag="ghcr.io/${{ github.repository_owner }}/vector-db-benchmark:${branch}"
          echo "Use tag ${tag}"
          echo "tag=${tag}" >> $GITHUB_OUTPUT
      - name: Set up Docker Buildx
        if: ${{ inputs.build_vector_db_image == 'true' }}
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0
      - name: Login to ghcr.io
        if: ${{ inputs.build_vector_db_image == 'true' }}
        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build Vector DB image
        if: ${{ inputs.build_vector_db_image == 'true' }}
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2
        with:
          context: .
          push: true
          tags: ${{ steps.prepare-tag.outputs.tag }}
          provenance: false
          cache-from: type=gha
          cache-to: type=gha,mode=max
      - name: Benches
        run: |
          export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
          export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
          export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
          export QDRANT_VERSION=${{ inputs.qdrant_version }}
          export DATASETS=${{ inputs.dataset }}
          export ENGINE_NAME=${{ inputs.engine_config }}
          export POSTGRES_TABLE=benchmark_manual
          export QDRANT__FEATURE_FLAGS__ALL=${{ inputs.feature_flags_all }}
          if [ "${{ inputs.build_vector_db_image }}" = "true" ]; then
            export VECTOR_DB_BENCHMARK_IMAGE=${{ steps.prepare-tag.outputs.tag }}
            export GHCR_USERNAME=${{ github.repository_owner }}
            export GHCR_PASSWORD=${{ secrets.GITHUB_TOKEN }}
          fi
          bash -x tools/setup_ci.sh
          bash -x tools/run_ci.sh