Skip to content

Commit c984695

Browse files
tellet-qtellet-q
committed
WIP: h-and-m-2048-angular-filters dataset
1 parent 02fd38c commit c984695

3 files changed

Lines changed: 75 additions & 64 deletions

File tree

‎.github/workflows/continuous-benchmark-transfer.yaml‎

Lines changed: 50 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,16 @@ on:
55
inputs:
66
dataset_name:
77
description: 'Dataset name'
8-
default: 'laion-small-clip'
8+
default: 'h-and-m-2048-angular-filters'
99
type: string
1010
qdrant_versions:
1111
description: 'Comma-separated versions to compare (ghcr/dev, docker/master, docker/v1.13.0). Can be just single version.'
1212
default: 'ghcr/dev,docker/v1.16.3'
1313
type: string
14+
cluster_nodes:
15+
description: 'Number of cluster nodes'
16+
default: '2'
17+
type: string
1418
region:
1519
description: 'Hetzner region'
1620
default: 'fsn1'
@@ -38,50 +42,64 @@ env:
3842
SERVER_TYPE: ${{ inputs.server_type || 'ccx13' }}
3943
CLIENT_TYPE: ${{ inputs.client_type || 'cx33' }}
4044
REGION: ${{ inputs.region || 'fsn1' }}
41-
DATASET_NAME: ${{ inputs.dataset_name || 'laion-small-clip' }}
45+
DATASET_NAME: ${{ inputs.dataset_name || 'h-and-m-2048-angular-filters' }}
4246
QDRANT_VERSIONS: ${{ inputs.qdrant_versions || 'ghcr/dev,docker/v1.16.3' }}
47+
CLUSTER_NODES: ${{ inputs.cluster_nodes || '2' }}
4348

4449
jobs:
50+
generateMatrix:
51+
name: Generate Matrix
52+
runs-on: ubuntu-latest
53+
outputs:
54+
matrix: ${{ steps.generate.outputs.matrix }}
55+
node_names: ${{ steps.generate.outputs.node_names }}
56+
client_name: ${{ steps.generate.outputs.client_name }}
57+
steps:
58+
- name: Generate machine matrix
59+
id: generate
60+
run: |
61+
NODES=${{ env.CLUSTER_NODES }}
62+
RUN_ID=${{ github.run_id }}
63+
64+
MACHINES='['
65+
NODE_NAMES=''
66+
for i in $(seq 0 $((NODES - 1))); do
67+
[ $i -gt 0 ] && MACHINES+=','
68+
MACHINES+="{\"name\":\"node-${i}\",\"suffix\":\"node-${i}\",\"type\":\"SERVER_TYPE\"}"
69+
[ -n "$NODE_NAMES" ] && NODE_NAMES+=','
70+
NODE_NAMES+="transfer-bench-node-${i}-${RUN_ID}"
71+
done
72+
MACHINES+=",{\"name\":\"client\",\"suffix\":\"client\",\"type\":\"CLIENT_TYPE\"}]"
73+
74+
echo "matrix={\"machine\":${MACHINES}}" >> $GITHUB_OUTPUT
75+
echo "node_names=${NODE_NAMES}" >> $GITHUB_OUTPUT
76+
echo "client_name=transfer-bench-client-${RUN_ID}" >> $GITHUB_OUTPUT
77+
4578
setupCluster:
4679
name: Setup ${{ matrix.machine.name }}
80+
needs: generateMatrix
4781
runs-on: ubuntu-latest
4882
strategy:
4983
fail-fast: true
50-
matrix:
51-
machine:
52-
- name: node-0
53-
server_name_suffix: node-0
54-
server_type_env: SERVER_TYPE
55-
- name: node-1
56-
server_name_suffix: node-1
57-
server_type_env: SERVER_TYPE
58-
- name: client
59-
server_name_suffix: client
60-
server_type_env: CLIENT_TYPE
61-
outputs:
62-
node_names: transfer-bench-node-0-${{ github.run_id }},transfer-bench-node-1-${{ github.run_id }}
63-
client_name: transfer-bench-client-${{ github.run_id }}
84+
matrix: ${{ fromJSON(needs.generateMatrix.outputs.matrix) }}
6485
steps:
6586
- uses: actions/checkout@v4
66-
6787
- uses: webfactory/ssh-agent@v0.8.0
6888
with:
6989
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
70-
7190
- name: Setup CI tools
7291
run: bash -x tools/setup_ci.sh
73-
7492
- name: Create machine
7593
uses: ./.github/workflows/actions/create-server-with-retry
7694
with:
77-
server_name: transfer-bench-${{ matrix.machine.server_name_suffix }}-${{ github.run_id }}
78-
server_type: ${{ matrix.machine.server_type_env == 'CLIENT_TYPE' && env.CLIENT_TYPE || env.SERVER_TYPE }}
95+
server_name: transfer-bench-${{ matrix.machine.suffix }}-${{ github.run_id }}
96+
server_type: ${{ matrix.machine.type == 'CLIENT_TYPE' && env.CLIENT_TYPE || env.SERVER_TYPE }}
7997
region: ${{ env.REGION }}
8098
max_retries: 2
8199

82100
runBenchmark:
83101
name: Run Transfer Benchmark
84-
needs: setupCluster
102+
needs: [generateMatrix, setupCluster]
85103
runs-on: ubuntu-latest
86104
container: alpine/ansible:2.18.1
87105
steps:
@@ -93,13 +111,13 @@ jobs:
93111
uses: ./.github/workflows/actions/create-inventory
94112
with:
95113
hcloud_token: ${{ secrets.HCLOUD_TOKEN }}
96-
server_names: ${{ needs.setupCluster.outputs.node_names }}
97-
client_names: ${{ needs.setupCluster.outputs.client_name }}
114+
server_names: ${{ needs.generateMatrix.outputs.node_names }}
115+
client_names: ${{ needs.generateMatrix.outputs.client_name }}
98116
db_host: ${{ secrets.POSTGRES_HOST }}
99117
- name: Run benchmarks for all versions
100118
run: |
101119
echo "$QDRANT_VERSIONS" | tr ',' '\n' | while read -r QDRANT_VERSION; do
102-
QDRANT_VERSION=$(echo "$QDRANT_VERSION" | xargs) # trim whitespace
120+
QDRANT_VERSION=$(echo "$QDRANT_VERSION" | xargs)
103121
[ -z "$QDRANT_VERSION" ] && continue
104122
105123
echo "=========================================="
@@ -134,7 +152,7 @@ jobs:
134152

135153
cleanup:
136154
name: Cleanup Cluster
137-
needs: [setupCluster, runBenchmark]
155+
needs: [generateMatrix, runBenchmark]
138156
if: always()
139157
runs-on: ubuntu-latest
140158
steps:
@@ -143,26 +161,20 @@ jobs:
143161
run: bash -x tools/setup_ci.sh
144162
env:
145163
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
146-
- name: Delete cluster nodes and client
164+
- name: Delete machines
147165
run: |
148-
NODE_NAMES="${{ needs.setupCluster.outputs.node_names }}"
149-
CLIENT_NAME="${{ needs.setupCluster.outputs.client_name }}"
166+
NODE_NAMES="${{ needs.generateMatrix.outputs.node_names }}"
167+
CLIENT_NAME="${{ needs.generateMatrix.outputs.client_name }}"
150168
151-
# Build list of all machines to delete
152169
ALL_MACHINES=()
153170
IFS=',' read -ra NODES <<< "$NODE_NAMES"
154171
for NODE in "${NODES[@]}"; do
155-
if [ -n "$NODE" ]; then
156-
ALL_MACHINES+=("$NODE")
157-
fi
172+
[ -n "$NODE" ] && ALL_MACHINES+=("$NODE")
158173
done
159-
if [ -n "$CLIENT_NAME" ]; then
160-
ALL_MACHINES+=("$CLIENT_NAME")
161-
fi
174+
[ -n "$CLIENT_NAME" ] && ALL_MACHINES+=("$CLIENT_NAME")
162175
163176
echo "Deleting ${#ALL_MACHINES[@]} machines in parallel..."
164177
165-
# Launch all deletions in parallel
166178
PIDS=()
167179
for MACHINE in "${ALL_MACHINES[@]}"; do
168180
(
@@ -172,7 +184,6 @@ jobs:
172184
PIDS+=($!)
173185
done
174186
175-
# Wait for all deletions to complete
176187
FAILED=0
177188
for i in "${!PIDS[@]}"; do
178189
if wait ${PIDS[$i]}; then
@@ -183,8 +194,6 @@ jobs:
183194
fi
184195
done
185196
186-
if [ $FAILED -gt 0 ]; then
187-
echo "Warning: $FAILED deletion(s) failed"
188-
fi
197+
[ $FAILED -gt 0 ] && echo "Warning: $FAILED deletion(s) failed"
189198
env:
190199
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}

‎ansible/playbooks/playbook-transfer-speed.yml‎

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,25 +20,23 @@
2020
roles:
2121
- run-transfer-speed
2222

23-
#- name: Export data into postgres
24-
# hosts: db_hosts
25-
# tasks:
26-
# - name: Insert data into table
27-
# ansible.builtin.shell: |
28-
# results='{{ hostvars[groups["client_machines"][0]]["transfer_results"] | to_json }}'
29-
#
30-
# throughput_pts_s=$(echo "$results" | jq -r '.stats.throughput_mean')
31-
# duration_s=$(echo "$results" | jq -r '.stats.duration_mean')
32-
# dataset_name=$(echo "$results" | jq -r '.params.dataset')
33-
# measure_timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
34-
#
35-
# pg_query="INSERT INTO benchmark_shard_transfer (
36-
# engine, engine_version, dataset, measure_timestamp, throughput_pts_s, duration_s
37-
# ) VALUES (
38-
# '{{ server.name }}', '{{ server.version }}', '\${dataset_name}', '\${measure_timestamp}', \${throughput_pts_s}, \${duration_s}
39-
# );"
40-
#
41-
# docker exec -i qdrant-postgres psql -U qdrant -d postgres -c "${pg_query}"
42-
# loop: "{{ servers }}"
43-
# loop_control:
44-
# loop_var: "server"
23+
- name: Export data into postgres
24+
hosts: db_hosts
25+
vars_files: ["group_vars/transfer-speed.yml"]
26+
tasks:
27+
- name: Insert data into table
28+
ansible.builtin.shell: |
29+
results='{{ hostvars[groups["client_machines"][0]]["transfer_results"] | to_json }}'
30+
31+
throughput_pts_s=$(echo "$results" | jq -r '.stats.throughput_mean')
32+
duration_s=$(echo "$results" | jq -r '.stats.duration_mean')
33+
dataset_name=$(echo "$results" | jq -r '.params.dataset')
34+
measure_timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
35+
36+
pg_query="INSERT INTO benchmark_shard_transfer (
37+
engine, engine_version, dataset, measure_timestamp, throughput_pts_s, duration_s
38+
) VALUES (
39+
'qdrant', '{{ server_registry }}/{{ server_version }}', '\${dataset_name}', '\${measure_timestamp}', \${throughput_pts_s}, \${duration_s}
40+
);"
41+
42+
docker exec -i qdrant-postgres psql -U qdrant -d postgres -c "${pg_query}"

‎ansible/playbooks/roles/run-transfer-speed/tasks/main.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@
6161
src: "{{ working_dir }}/results.json"
6262
register: results_file
6363

64+
- name: Set transfer_results fact
65+
set_fact:
66+
transfer_results: "{{ results_file.content | b64decode | from_json }}"
67+
6468
- name: Output
6569
debug:
66-
msg: "{{ (results_file.content | b64decode | from_json).stats }}"
70+
msg: "{{ transfer_results.stats }}"

0 commit comments

Comments
 (0)