From f27007ab59f8c19c37d06a77b712e9ef01eb43b6 Mon Sep 17 00:00:00 2001 From: Benn Otieno Date: Thu, 30 Apr 2026 18:42:41 +0200 Subject: [PATCH 1/6] feat: backup-qdrant-collection-aliases --- qdrant-backup-restore/.env.sample | 2 + qdrant-backup-restore/README.md | 21 +++++- qdrant-backup-restore/k8s/README.md | 25 +++++++ .../qdrant_backup_recovery.sh | 73 +++++++++++++++++-- 4 files changed, 109 insertions(+), 12 deletions(-) diff --git a/qdrant-backup-restore/.env.sample b/qdrant-backup-restore/.env.sample index a5ba56a..5ab9901 100644 --- a/qdrant-backup-restore/.env.sample +++ b/qdrant-backup-restore/.env.sample @@ -8,3 +8,5 @@ export QDRANT_S3_ENDPOINT_URL="http://minio.default.svc.cluster.local:9000" # up export QDRANT_RESTORE_HOSTS="http://localhost:6334" # update to target restore host(s) (for restore only otherwise leave as empty string). export GET_PEERS_FROM_CLUSTER_INFO="false" # only enable for kubernetes cluster deployment with qdrant cluster.enabled option set as true. export MC_CONFIG_DIR="mc" # controls where mc S3 client store its configurations (for restore or get_snap_s3 only). +export BACKUP_COLLECTION_ALIASES_ON_S3="false" # enable to backup collection aliases to s3 bucket. +export QDRANT_COLLECTION_ALIASES_STABLE_FILE="collection_aliases" # override the stable version of collection_aliases file to restore. diff --git a/qdrant-backup-restore/README.md b/qdrant-backup-restore/README.md index 0c78af7..78cfd67 100644 --- a/qdrant-backup-restore/README.md +++ b/qdrant-backup-restore/README.md @@ -63,6 +63,19 @@ Create a `.env` file based on `.env.sample` with the following variables: | `QDRANT_RESTORE_HOSTS` | Comma-separated list of destination Qdrant hosts set as `""` | `""` | | `GET_PEERS_FROM_CLUSTER_INFO` | Auto-discover peers from Qdrant cluster info endpoint (only for Kubernetes) | `false` | +if `BACKUP_COLLECTION_ALIASES_ON_S3` is `true` include the S3 credentials. When `true` ensure S3 credentials are set i.e. `QDRANT_S3_SECRET_ACCESS_KEY`, `QDRANT_S3_SECRET_ACCESS_KEY`, `QDRANT_S3_ENDPOINT_URL`, `QDRANT_S3_BUCKET_NAME` (use same bucket name on qdrant configuration). + +#### Required Variables For Optional Collection Aliases Backup +| Variable | Description | Default | +|----------|-------------|---------| +| `QDRANT_SOURCE_HOSTS` | Comma-separated list of source Qdrant hosts | `http://localhost:6333` | +| `QDRANT_RESTORE_HOSTS` | Comma-separated list of destination Qdrant hosts set as `""` | `""` | +| `QDRANT_S3_ENDPOINT_URL` | S3-compatible storage endpoint URL | `http://minio:9000` | +| `QDRANT_S3_ACCESS_KEY_ID` | S3 access key ID | `your-access-key` | +| `QDRANT_S3_SECRET_ACCESS_KEY` | S3 secret access key | `your-secret-key` | +| `QDRANT_S3_BUCKET_NAME` | S3 bucket name where snapshots are stored | `bucket-name` | +| `BACKUP_COLLECTION_ALIASES_ON_S3` | set `true` to toggle backing up collection aliases on S3 | `false` | + #### Required Variables For Restore | Variable | Description | Default | @@ -308,10 +321,10 @@ source .env.dest - `QDRANT_API_KEY` - set your Qdrant api key if it exists otherwise leave as is. - `QDRANT_SOURCE_HOSTS` - set your Qdrant source host. if you are connecting to your a qdrant cluster deployed on kubernetes use port forwarding. Ensure **all** the pods/containers can be reached locally. Add these comma seperated hosts in this config .e.g `"http://qdrant-source-1:6333,http://qdrant-source-1:6334"`. This is required only for the backup process. In Kubernetes, service/peer discovery is done automatically by enabling `GET_PEERS_FROM_CLUSTER_INFO`. - `QDRANT_RESTORE_HOSTS` - set your Qdrant target restore host **(for restore only)** set as `""` when backing up. - - `QDRANT_S3_ENDPOINT_URL` - set it to your s3 endpoint url **(for restore only)**. - - `QDRANT_S3_ACCESS_KEY_ID` - set it to your s3 access key id credentials **(for restore only)**. - - `QDRANT_S3_SECRET_ACCESS_KEY`- set it to your s3 secret access key credentials **(for restore only)**. - - `QDRANT_S3_BUCKET_NAME`- set it to your s3 bucket name **(for restore only)**. + - `QDRANT_S3_ENDPOINT_URL` - set it to your s3 endpoint url. + - `QDRANT_S3_ACCESS_KEY_ID` - set it to your s3 access key id credentials. + - `QDRANT_S3_SECRET_ACCESS_KEY`- set it to your s3 secret access key credentials. + - `QDRANT_S3_BUCKET_NAME`- set it to your s3 bucket name. - `GET_PEERS_FROM_CLUSTER_INFO`- leave as is (`false`) for non-cluster usecases. - Run below to make the environment variables available. diff --git a/qdrant-backup-restore/k8s/README.md b/qdrant-backup-restore/k8s/README.md index 4943847..3f1c137 100644 --- a/qdrant-backup-restore/k8s/README.md +++ b/qdrant-backup-restore/k8s/README.md @@ -120,6 +120,29 @@ Update the following environment varibles in your copy of `backup-cronjob.yaml` 5. `CURL_TIMEOUT` is set at `3000` seconds. This is the max time curl will wait for request to complete. Its increased in scenarios where backup takes a while. 6. `QDRANT_S3_LINK_EXPIRY_DURATION` is set at `3600`. This is the duration that an s3 presigned url will be active. The url is used during the recovery process. 7. `QDRANT_WAIT_ON_TASK` is set as `true`. This configuation make backup process synchronous meaning 'wait for snapshot process to finish successfully before moving on'. Its used during backup and recovery. +8. `BACKUP_COLLECTION_ALIASES_ON_S3` is set as `false`. This configuration is used to toggle backup of collection aliases to S3. The collection_aliases are versioned by the timestamp. When `true` ensure S3 credentials are set; + ````yaml + env: + - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ACCESS_KEY + valueFrom: + secretKeyRef: + name: + key: QDRANT_S3_ACCESS_KEY_ID + - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__SECRET_KEY + valueFrom: + secretKeyRef: + name: + key: QDRANT_S3_SECRET_ACCESS_KEY + - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__BUCKET + valueFrom: + secretKeyRef: + name: + key: QDRANT_S3_BUCKET_NAME + - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ENDPOINT_URL + value: "" + - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__SNAPSHOTS_STORAGE + value: "s3" + ```` Deploy the job! @@ -148,6 +171,8 @@ Update the following environment varibles accordingly; 7. `QDRANT_WAIT_ON_TASK` is set as `true`. This configuation make restoration process synchronous meaning 'wait for snapshot process to finish successfully before moving on'. Its used during backup and recovery. 8. `QDRANT_SNAPSHOT_DATETIME_FILTER` is empty. Setting this filters out snapshot/backups belonging to a certain date and time using glob pattern matching. e.g `"2026-01-29"` = all snapshots in 29th January 2026, `2026-01` = all backups in January 2026. 9. `MC_CONFIG_DIR` is `mc`. This overrides the default storage location ($HOME) for mc s3 client configurations. Essential in set ups that use stricter securityContext configuration like `readOnlyRootFilesystem: true`. +10. `BACKUP_COLLECTION_ALIASES_ON_S3` is `false`. When this is set to `true` during restoration it will fetch the lastest `collection_alias` from S3. +11. `QDRANT_COLLECTION_ALIASES_STABLE_FILE` is `collection_aliases`. This configuration is used to override the stable version of collection_aliases file to restore. e.g. `collection_aliases-2026-04-30_18-31-04`. The file has to exist on s3. Deploy the job! diff --git a/qdrant-backup-restore/qdrant_backup_recovery.sh b/qdrant-backup-restore/qdrant_backup_recovery.sh index 18554e3..7cdd4ce 100755 --- a/qdrant-backup-restore/qdrant_backup_recovery.sh +++ b/qdrant-backup-restore/qdrant_backup_recovery.sh @@ -7,11 +7,14 @@ QDRANT_FAILED_RECOVERY_FILE="failed_snapshot_recovery" QDRANT_SNAPSHOT_RECOVERY_HISTORY_FILE="snapshot_recovery_history" QDRANT_ALIAS_RECOVERY_HISTORY_FILE="alias_recovery_history" QDRANT_COLLECTION_ALIASES="collection_aliases" +BACKUP_COLLECTION_ALIASES_ON_S3=${BACKUP_COLLECTION_ALIASES_ON_S3:-false} +QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER="collection_aliases_store" QDRANT_WAIT_ON_TASK=${QDRANT_WAIT_ON_TASK:-true} CURL_TIMEOUT="${CURL_TIMEOUT:-1800}" QDRANT_S3_ALIAS="qdrant_s3_snaphost" QDRANT_S3_LINK_EXPIRY_DURATION="${QDRANT_S3_LINK_EXPIRY_DURATION:-3600s}" QDRANT_HTTP_PORT="${QDRANT_HTTP_PORT:-6333}" +QDRANT_COLLECTION_ALIASES_STABLE_FILE="${QDRANT_COLLECTION_ALIASES_STABLE_FILE:-collection_aliases}" declare -a peer_uri_map; # printf wrapper helper @@ -496,6 +499,8 @@ get_collection_aliases() { exit 1 fi + : > "$QDRANT_COLLECTION_ALIASES" + local colla_length=0 colla_length=$(jq -c '(.result.aliases // []) | length' <<< "$result") @@ -510,14 +515,41 @@ get_collection_aliases() { done <<< "$_result" fi +if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then + + setup_s3_storage + + local ts + ts=$(date '+%Y-%m-%d_%H-%M-%S') + + local base_path="$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER" + + local versioned_dest="$base_path/${QDRANT_COLLECTION_ALIASES}-$ts" + local stable_dest="$base_path/${QDRANT_COLLECTION_ALIASES}" + + _printf "backing up collection aliases to s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME" + + if ! mc cp "$QDRANT_COLLECTION_ALIASES" "$versioned_dest"; then + _printf "failed to upload versioned collection aliases backup to %s\n" "$QDRANT_S3_BUCKET_NAME" + exit 1 + fi + + if ! mc cp "$versioned_dest" "$stable_dest"; then + _printf "failed to update collection aliases stable backup pointer in %s\n" "$QDRANT_S3_BUCKET_NAME" + exit 1 + fi + + _printf "successfully backed up collection aliases to s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME" + fi + _printf "%s file updated,found %d collection alias(es)!\n" "$QDRANT_COLLECTION_ALIASES" "$colla_length" } # restores collection alias to a qdrant node and appends progress to $QDRANT_ALIAS_RECOVERY_HISTORY_FILE recover_collection_alias() { - local host="${source_hosts[0]}" local collection_name="$1" local alias_name="$2" + local host="$3" local status="" local result="" @@ -540,7 +572,7 @@ recover_collection_alias() { if [ "$status" != "ok" ]; then ((failed_recovered_colla_count=failed_recovered_colla_count+1)) - _printf "[%s] failed to fetch collection alias %s:%s, got this instead %s\n" "$host" "$collection_name" "$alias_name" "${result//[[:space:]]/}" + _printf "[%s] failed to restore collection alias %s:%s, got this instead %s\n" "$host" "$collection_name" "$alias_name" "${result//[[:space:]]/}" return fi @@ -550,6 +582,22 @@ recover_collection_alias() { # restores collection aliases to a qdrant node and appends progress to $QDRANT_ALIAS_RECOVERY_HISTORY_FILE recover_collection_aliases() { + + local host="${restore_hosts[0]}" + + + if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then + + setup_s3_storage + + if mc cp "$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER/$QDRANT_COLLECTION_ALIASES_STABLE_FILE" "$QDRANT_COLLECTION_ALIASES"; then + _printf "collection aliases restored from s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME" + else + _printf "failed to restore collection aliases file (%s) from s3 bucket %s\n" "$QDRANT_COLLECTION_ALIASES_STABLE_FILE" "$QDRANT_S3_BUCKET_NAME" + exit 1 + fi + fi + if [[ ! -s "$QDRANT_COLLECTION_ALIASES" ]]; then get_collection_aliases fi @@ -562,8 +610,7 @@ recover_collection_aliases() { colla_count=$(wc -l < $QDRANT_COLLECTION_ALIASES) recovered_colla_count=0 failed_recovered_colla_count=0 - - local host="${restore_hosts[0]}" + colla_skipped_count=0 touch $QDRANT_ALIAS_RECOVERY_HISTORY_FILE @@ -577,12 +624,13 @@ recover_collection_aliases() { if [ "$history_count" -gt 0 ]; then _printf "[%s] collection alias %s:%s already recovered, skipping!\n" "$host" "$collection_name" "$alias_name" + ((colla_skipped_count=colla_skipped_count+1)) continue fi - recover_collection_alias "$collection_name" "$alias_name" + recover_collection_alias "$collection_name" "$alias_name" "$host" done < $QDRANT_COLLECTION_ALIASES - _printf "recovery summary: %d/%d collection aliases recovered, %d failed.\n" "$recovered_colla_count" "$colla_count" "$failed_recovered_colla_count" + _printf "recovery summary: %d/%d collection aliases recovered, %d failed, %d skipped.\n" "$recovered_colla_count" "$colla_count" "$failed_recovered_colla_count" "$colla_skipped_count" colla_count=0 recovered_colla_count=0 @@ -686,9 +734,9 @@ Environment variables: QDRANT_S3_SECRET_ACCESS_KEY - S3 secret access key (for S3 operations) QDRANT_S3_BUCKET_NAME - S3 bucket name (for S3 operations) GET_PEERS_FROM_CLUSTER_INFO - Set to "true" to auto-discover peers - CURL_TIMEOUT - Curl timeout in seconds (default: 300) + CURL_TIMEOUT - Curl timeout in seconds (default: 1800) QDRANT_WAIT_ON_TASK - Waits for async tasks to finish - QDRANT_SNAPSHOT_DATETIME_FILTER - Specify the datea and time filter for snapshots to be fetched and/or restored, format YYYY-mm-dd, e,g "2026-01-29-11-44", default value is empty so it will fetch every snapshot!! + QDRANT_SNAPSHOT_DATETIME_FILTER - Specify the date and time filter for snapshots to be fetched and/or restored, format YYYY-mm-dd, e,g "2026-01-29-11-44", default value is empty so it will fetch every snapshot!! Examples: $0 get_snap $0 recover_snap @@ -738,11 +786,20 @@ run() { local DATETIME="${QDRANT_SNAPSHOT_DATETIME_FILTER:-}" generate_snapshot_file_from_s3 "$DATETIME" elif [ "$command" = "create_snap" ] || [ "$command" = "create_snapshot" ]; then + # create collection snapshots create_collection_snapshot + + # backup collection aliases to s3 bucket + get_collection_aliases + elif [ "$command" = "recover_snap" ]; then setup_s3_storage local DATETIME="${QDRANT_SNAPSHOT_DATETIME_FILTER:-}" recover_collection_snapshots "$DATETIME" + + # restore collection aliases from s3 bucket + recover_collection_aliases + elif [ "$command" = "get_colla" ] || [ "$command" = "get_collection_alias" ]; then get_collection_aliases elif [ "$command" = "recover_colla" ] || [ "$command" = "recover_collection_alias" ]; then From 81aa99e776e61382e434470b0122235d01ca2a3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 5 May 2026 11:58:42 +0000 Subject: [PATCH 2/6] Update ConfigMap with latest script changes --- .../k8s/configmap-script.yaml | 73 +++++++++++++++++-- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/qdrant-backup-restore/k8s/configmap-script.yaml b/qdrant-backup-restore/k8s/configmap-script.yaml index 797ddca..c10db83 100644 --- a/qdrant-backup-restore/k8s/configmap-script.yaml +++ b/qdrant-backup-restore/k8s/configmap-script.yaml @@ -16,11 +16,14 @@ data: QDRANT_SNAPSHOT_RECOVERY_HISTORY_FILE="snapshot_recovery_history" QDRANT_ALIAS_RECOVERY_HISTORY_FILE="alias_recovery_history" QDRANT_COLLECTION_ALIASES="collection_aliases" + BACKUP_COLLECTION_ALIASES_ON_S3=${BACKUP_COLLECTION_ALIASES_ON_S3:-false} + QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER="collection_aliases_store" QDRANT_WAIT_ON_TASK=${QDRANT_WAIT_ON_TASK:-true} CURL_TIMEOUT="${CURL_TIMEOUT:-1800}" QDRANT_S3_ALIAS="qdrant_s3_snaphost" QDRANT_S3_LINK_EXPIRY_DURATION="${QDRANT_S3_LINK_EXPIRY_DURATION:-3600s}" QDRANT_HTTP_PORT="${QDRANT_HTTP_PORT:-6333}" + QDRANT_COLLECTION_ALIASES_STABLE_FILE="${QDRANT_COLLECTION_ALIASES_STABLE_FILE:-collection_aliases}" declare -a peer_uri_map; # printf wrapper helper @@ -505,6 +508,8 @@ data: exit 1 fi + : > "$QDRANT_COLLECTION_ALIASES" + local colla_length=0 colla_length=$(jq -c '(.result.aliases // []) | length' <<< "$result") @@ -519,14 +524,41 @@ data: done <<< "$_result" fi + if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then + + setup_s3_storage + + local ts + ts=$(date '+%Y-%m-%d_%H-%M-%S') + + local base_path="$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER" + + local versioned_dest="$base_path/${QDRANT_COLLECTION_ALIASES}-$ts" + local stable_dest="$base_path/${QDRANT_COLLECTION_ALIASES}" + + _printf "backing up collection aliases to s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME" + + if ! mc cp "$QDRANT_COLLECTION_ALIASES" "$versioned_dest"; then + _printf "failed to upload versioned collection aliases backup to %s\n" "$QDRANT_S3_BUCKET_NAME" + exit 1 + fi + + if ! mc cp "$versioned_dest" "$stable_dest"; then + _printf "failed to update collection aliases stable backup pointer in %s\n" "$QDRANT_S3_BUCKET_NAME" + exit 1 + fi + + _printf "successfully backed up collection aliases to s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME" + fi + _printf "%s file updated,found %d collection alias(es)!\n" "$QDRANT_COLLECTION_ALIASES" "$colla_length" } # restores collection alias to a qdrant node and appends progress to $QDRANT_ALIAS_RECOVERY_HISTORY_FILE recover_collection_alias() { - local host="${source_hosts[0]}" local collection_name="$1" local alias_name="$2" + local host="$3" local status="" local result="" @@ -549,7 +581,7 @@ data: if [ "$status" != "ok" ]; then ((failed_recovered_colla_count=failed_recovered_colla_count+1)) - _printf "[%s] failed to fetch collection alias %s:%s, got this instead %s\n" "$host" "$collection_name" "$alias_name" "${result//[[:space:]]/}" + _printf "[%s] failed to restore collection alias %s:%s, got this instead %s\n" "$host" "$collection_name" "$alias_name" "${result//[[:space:]]/}" return fi @@ -559,6 +591,22 @@ data: # restores collection aliases to a qdrant node and appends progress to $QDRANT_ALIAS_RECOVERY_HISTORY_FILE recover_collection_aliases() { + + local host="${restore_hosts[0]}" + + + if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then + + setup_s3_storage + + if mc cp "$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER/$QDRANT_COLLECTION_ALIASES_STABLE_FILE" "$QDRANT_COLLECTION_ALIASES"; then + _printf "collection aliases restored from s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME" + else + _printf "failed to restore collection aliases file (%s) from s3 bucket %s\n" "$QDRANT_COLLECTION_ALIASES_STABLE_FILE" "$QDRANT_S3_BUCKET_NAME" + exit 1 + fi + fi + if [[ ! -s "$QDRANT_COLLECTION_ALIASES" ]]; then get_collection_aliases fi @@ -571,8 +619,7 @@ data: colla_count=$(wc -l < $QDRANT_COLLECTION_ALIASES) recovered_colla_count=0 failed_recovered_colla_count=0 - - local host="${restore_hosts[0]}" + colla_skipped_count=0 touch $QDRANT_ALIAS_RECOVERY_HISTORY_FILE @@ -586,12 +633,13 @@ data: if [ "$history_count" -gt 0 ]; then _printf "[%s] collection alias %s:%s already recovered, skipping!\n" "$host" "$collection_name" "$alias_name" + ((colla_skipped_count=colla_skipped_count+1)) continue fi - recover_collection_alias "$collection_name" "$alias_name" + recover_collection_alias "$collection_name" "$alias_name" "$host" done < $QDRANT_COLLECTION_ALIASES - _printf "recovery summary: %d/%d collection aliases recovered, %d failed.\n" "$recovered_colla_count" "$colla_count" "$failed_recovered_colla_count" + _printf "recovery summary: %d/%d collection aliases recovered, %d failed, %d skipped.\n" "$recovered_colla_count" "$colla_count" "$failed_recovered_colla_count" "$colla_skipped_count" colla_count=0 recovered_colla_count=0 @@ -695,9 +743,9 @@ data: QDRANT_S3_SECRET_ACCESS_KEY - S3 secret access key (for S3 operations) QDRANT_S3_BUCKET_NAME - S3 bucket name (for S3 operations) GET_PEERS_FROM_CLUSTER_INFO - Set to "true" to auto-discover peers - CURL_TIMEOUT - Curl timeout in seconds (default: 300) + CURL_TIMEOUT - Curl timeout in seconds (default: 1800) QDRANT_WAIT_ON_TASK - Waits for async tasks to finish - QDRANT_SNAPSHOT_DATETIME_FILTER - Specify the datea and time filter for snapshots to be fetched and/or restored, format YYYY-mm-dd, e,g "2026-01-29-11-44", default value is empty so it will fetch every snapshot!! + QDRANT_SNAPSHOT_DATETIME_FILTER - Specify the date and time filter for snapshots to be fetched and/or restored, format YYYY-mm-dd, e,g "2026-01-29-11-44", default value is empty so it will fetch every snapshot!! Examples: $0 get_snap $0 recover_snap @@ -747,11 +795,20 @@ data: local DATETIME="${QDRANT_SNAPSHOT_DATETIME_FILTER:-}" generate_snapshot_file_from_s3 "$DATETIME" elif [ "$command" = "create_snap" ] || [ "$command" = "create_snapshot" ]; then + # create collection snapshots create_collection_snapshot + + # backup collection aliases to s3 bucket + get_collection_aliases + elif [ "$command" = "recover_snap" ]; then setup_s3_storage local DATETIME="${QDRANT_SNAPSHOT_DATETIME_FILTER:-}" recover_collection_snapshots "$DATETIME" + + # restore collection aliases from s3 bucket + recover_collection_aliases + elif [ "$command" = "get_colla" ] || [ "$command" = "get_collection_alias" ]; then get_collection_aliases elif [ "$command" = "recover_colla" ] || [ "$command" = "recover_collection_alias" ]; then From 6e2a0f352120f1876be781d1b7952f57176f3342 Mon Sep 17 00:00:00 2001 From: Benn Otieno Date: Tue, 5 May 2026 17:59:06 +0200 Subject: [PATCH 3/6] fix collection alias recovery logic update doc --- qdrant-backup-restore/k8s/README.md | 12 +++++----- .../qdrant_backup_recovery.sh | 23 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/qdrant-backup-restore/k8s/README.md b/qdrant-backup-restore/k8s/README.md index 3f1c137..c870402 100644 --- a/qdrant-backup-restore/k8s/README.md +++ b/qdrant-backup-restore/k8s/README.md @@ -123,25 +123,25 @@ Update the following environment varibles in your copy of `backup-cronjob.yaml` 8. `BACKUP_COLLECTION_ALIASES_ON_S3` is set as `false`. This configuration is used to toggle backup of collection aliases to S3. The collection_aliases are versioned by the timestamp. When `true` ensure S3 credentials are set; ````yaml env: - - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ACCESS_KEY + - name: QDRANT_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: name: key: QDRANT_S3_ACCESS_KEY_ID - - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__SECRET_KEY + - name: QDRANT_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: key: QDRANT_S3_SECRET_ACCESS_KEY - - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__BUCKET + - name: QDRANT_S3_BUCKET_NAME valueFrom: secretKeyRef: name: key: QDRANT_S3_BUCKET_NAME - - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ENDPOINT_URL + - name: QDRANT_S3_ENDPOINT_URL value: "" - - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__SNAPSHOTS_STORAGE - value: "s3" + - name: BACKUP_COLLECTION_ALIASES_ON_S3 + value: "true" ```` Deploy the job! diff --git a/qdrant-backup-restore/qdrant_backup_recovery.sh b/qdrant-backup-restore/qdrant_backup_recovery.sh index 7cdd4ce..cd6e459 100755 --- a/qdrant-backup-restore/qdrant_backup_recovery.sh +++ b/qdrant-backup-restore/qdrant_backup_recovery.sh @@ -259,8 +259,6 @@ fetch_collection_snapshot() { return fi - item=$(jq -c '.result[0] // empty' <<< "$result") - _result=$(jq -c '.result // [] | .[]' <<< "$result") while read -r item; do local snapshot_name="" @@ -402,10 +400,12 @@ get_s3_url() { local status="" status=$(jq -r '.status?' <<< "$result") if [ "$status" != "success" ]; then - _printf "[%s] failed to fetch snapshots for collections %s, got this instead %s\n" "$host" "$collection_name" "${result//[[:space:]]/}" + _printf "failed to fetch snapshots for collections %s, got this instead %s\n" "$collection_name" "${result//[[:space:]]/}" + return 1 fi s3_presigned_url=$(jq -r '.share' <<< "$result") + return 0 } # restores an collection snapshot from an s3 url updates the $QDRANT_SNAPSHOT_RECOVERY_HISTORY_FILE and $QDRANT_FAILED_RECOVERY_FILE @@ -418,7 +418,12 @@ recover_collection_snapshot() { _printf "[%s] started to recover %s snapshot of %s collection...\n" "$host" "$snapshot_name" "$collection_name" - get_s3_url "$collection_name" "$snapshot_name" + if ! get_s3_url "$collection_name" "$snapshot_name"; then + ((fail_recovered_count=fail_recovered_count+1)) + _printf "[%s] failed to presign S3 URL for %s snapshot of %s collection, skipping recover request\n" "$host" "$snapshot_name" "$collection_name" + printf '%s,%s,%s\n' "$host" "$snapshot_name" "presign_failed" >> $QDRANT_FAILED_RECOVERY_FILE + return + fi result=$(_curl PUT \ "$host/collections/$collection_name/snapshots/recover?wait=$QDRANT_WAIT_ON_TASK" \ @@ -599,12 +604,8 @@ recover_collection_aliases() { fi if [[ ! -s "$QDRANT_COLLECTION_ALIASES" ]]; then - get_collection_aliases - fi - - if [ ! -f "$QDRANT_COLLECTION_ALIASES" ]; then - _printf "[%s] collection aliases do not exist on source!\n" "$host" - return + _printf "collection aliases file does not exist or is empty, run 'get_colla' task to fetch collection aliases from source hosts or enable BACKUP_COLLECTION_ALIASES_ON_S3 to fetch collection aliases from S3 bucket\n" + exit 1 fi colla_count=$(wc -l < $QDRANT_COLLECTION_ALIASES) @@ -669,7 +670,7 @@ get_peers_from_cluster_info() { done <<< "$peer_uri_entries" if [ ${#peer_uri_map[@]} -eq 0 ]; then - _printf "no registered host peers in %s... exiting" "$host" + _printf "no registered host peers in %s... exiting\n" "$host" exit 1 fi } From db4cbb56323502c3475d029e0334e0acb3f9be11 Mon Sep 17 00:00:00 2001 From: Benn Otieno Date: Wed, 6 May 2026 12:30:31 +0200 Subject: [PATCH 4/6] update exit code when collection_aliases have not been generated --- qdrant-backup-restore/qdrant_backup_recovery.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qdrant-backup-restore/qdrant_backup_recovery.sh b/qdrant-backup-restore/qdrant_backup_recovery.sh index cd6e459..6597e56 100755 --- a/qdrant-backup-restore/qdrant_backup_recovery.sh +++ b/qdrant-backup-restore/qdrant_backup_recovery.sh @@ -603,9 +603,9 @@ recover_collection_aliases() { fi fi - if [[ ! -s "$QDRANT_COLLECTION_ALIASES" ]]; then - _printf "collection aliases file does not exist or is empty, run 'get_colla' task to fetch collection aliases from source hosts or enable BACKUP_COLLECTION_ALIASES_ON_S3 to fetch collection aliases from S3 bucket\n" - exit 1 + if [ ! -f "$QDRANT_COLLECTION_ALIASES" ]; then + _printf "collection aliases file does not exist, run 'get_colla' task to fetch collection aliases from source hosts or enable BACKUP_COLLECTION_ALIASES_ON_S3 to fetch collection aliases from S3 bucket\n" + return fi colla_count=$(wc -l < $QDRANT_COLLECTION_ALIASES) From bd9636b70e522d63c14d5696af4951dca876ae9a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 6 May 2026 10:33:04 +0000 Subject: [PATCH 5/6] Update ConfigMap with latest script changes --- .../k8s/configmap-script.yaml | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/qdrant-backup-restore/k8s/configmap-script.yaml b/qdrant-backup-restore/k8s/configmap-script.yaml index c10db83..88a4419 100644 --- a/qdrant-backup-restore/k8s/configmap-script.yaml +++ b/qdrant-backup-restore/k8s/configmap-script.yaml @@ -268,8 +268,6 @@ data: return fi - item=$(jq -c '.result[0] // empty' <<< "$result") - _result=$(jq -c '.result // [] | .[]' <<< "$result") while read -r item; do local snapshot_name="" @@ -411,10 +409,12 @@ data: local status="" status=$(jq -r '.status?' <<< "$result") if [ "$status" != "success" ]; then - _printf "[%s] failed to fetch snapshots for collections %s, got this instead %s\n" "$host" "$collection_name" "${result//[[:space:]]/}" + _printf "failed to fetch snapshots for collections %s, got this instead %s\n" "$collection_name" "${result//[[:space:]]/}" + return 1 fi s3_presigned_url=$(jq -r '.share' <<< "$result") + return 0 } # restores an collection snapshot from an s3 url updates the $QDRANT_SNAPSHOT_RECOVERY_HISTORY_FILE and $QDRANT_FAILED_RECOVERY_FILE @@ -427,7 +427,12 @@ data: _printf "[%s] started to recover %s snapshot of %s collection...\n" "$host" "$snapshot_name" "$collection_name" - get_s3_url "$collection_name" "$snapshot_name" + if ! get_s3_url "$collection_name" "$snapshot_name"; then + ((fail_recovered_count=fail_recovered_count+1)) + _printf "[%s] failed to presign S3 URL for %s snapshot of %s collection, skipping recover request\n" "$host" "$snapshot_name" "$collection_name" + printf '%s,%s,%s\n' "$host" "$snapshot_name" "presign_failed" >> $QDRANT_FAILED_RECOVERY_FILE + return + fi result=$(_curl PUT \ "$host/collections/$collection_name/snapshots/recover?wait=$QDRANT_WAIT_ON_TASK" \ @@ -607,12 +612,8 @@ data: fi fi - if [[ ! -s "$QDRANT_COLLECTION_ALIASES" ]]; then - get_collection_aliases - fi - if [ ! -f "$QDRANT_COLLECTION_ALIASES" ]; then - _printf "[%s] collection aliases do not exist on source!\n" "$host" + _printf "collection aliases file does not exist, run 'get_colla' task to fetch collection aliases from source hosts or enable BACKUP_COLLECTION_ALIASES_ON_S3 to fetch collection aliases from S3 bucket\n" return fi @@ -678,7 +679,7 @@ data: done <<< "$peer_uri_entries" if [ ${#peer_uri_map[@]} -eq 0 ]; then - _printf "no registered host peers in %s... exiting" "$host" + _printf "no registered host peers in %s... exiting\n" "$host" exit 1 fi } From 1d30f9b1a1a2ffd23bc2dbb699aeec486d0b1420 Mon Sep 17 00:00:00 2001 From: Benn Otieno Date: Thu, 7 May 2026 08:37:46 +0200 Subject: [PATCH 6/6] fix yaml formatting --- qdrant-backup-restore/k8s/README.md | 44 +++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/qdrant-backup-restore/k8s/README.md b/qdrant-backup-restore/k8s/README.md index c870402..909f2e7 100644 --- a/qdrant-backup-restore/k8s/README.md +++ b/qdrant-backup-restore/k8s/README.md @@ -37,24 +37,24 @@ Important steps to note: ````yaml env: - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ACCESS_KEY - valueFrom: + valueFrom: secretKeyRef: name: key: QDRANT_S3_ACCESS_KEY_ID - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__SECRET_KEY - valueFrom: + valueFrom: secretKeyRef: name: key: QDRANT_S3_SECRET_ACCESS_KEY - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__BUCKET - valueFrom: + valueFrom: secretKeyRef: name: key: QDRANT_S3_BUCKET_NAME - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__S3_CONFIG__ENDPOINT_URL - value: "" + value: "" - name: QDRANT__STORAGE__SNAPSHOTS_CONFIG__SNAPSHOTS_STORAGE - value: "s3" + value: "s3" ```` (Re)Deploy the Qdrant cluster! @@ -124,24 +124,24 @@ Update the following environment varibles in your copy of `backup-cronjob.yaml` ````yaml env: - name: QDRANT_S3_ACCESS_KEY_ID - valueFrom: + valueFrom: secretKeyRef: name: key: QDRANT_S3_ACCESS_KEY_ID - name: QDRANT_S3_SECRET_ACCESS_KEY - valueFrom: + valueFrom: secretKeyRef: name: key: QDRANT_S3_SECRET_ACCESS_KEY - name: QDRANT_S3_BUCKET_NAME - valueFrom: + valueFrom: secretKeyRef: name: key: QDRANT_S3_BUCKET_NAME - name: QDRANT_S3_ENDPOINT_URL - value: "" + value: "" - name: BACKUP_COLLECTION_ALIASES_ON_S3 - value: "true" + value: "true" ```` Deploy the job! @@ -171,7 +171,29 @@ Update the following environment varibles accordingly; 7. `QDRANT_WAIT_ON_TASK` is set as `true`. This configuation make restoration process synchronous meaning 'wait for snapshot process to finish successfully before moving on'. Its used during backup and recovery. 8. `QDRANT_SNAPSHOT_DATETIME_FILTER` is empty. Setting this filters out snapshot/backups belonging to a certain date and time using glob pattern matching. e.g `"2026-01-29"` = all snapshots in 29th January 2026, `2026-01` = all backups in January 2026. 9. `MC_CONFIG_DIR` is `mc`. This overrides the default storage location ($HOME) for mc s3 client configurations. Essential in set ups that use stricter securityContext configuration like `readOnlyRootFilesystem: true`. -10. `BACKUP_COLLECTION_ALIASES_ON_S3` is `false`. When this is set to `true` during restoration it will fetch the lastest `collection_alias` from S3. +10. `BACKUP_COLLECTION_ALIASES_ON_S3` is `false`. When this is set to `true` during restoration it will fetch the lastest `collection_alias` from S3. When `true` ensure the following S3 credentials are included. + ````yaml + env: + - name: QDRANT_S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: + key: QDRANT_S3_ACCESS_KEY_ID + - name: QDRANT_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: + key: QDRANT_S3_SECRET_ACCESS_KEY + - name: QDRANT_S3_BUCKET_NAME + valueFrom: + secretKeyRef: + name: + key: QDRANT_S3_BUCKET_NAME + - name: QDRANT_S3_ENDPOINT_URL + value: "" + - name: BACKUP_COLLECTION_ALIASES_ON_S3 + value: "true" + ```` 11. `QDRANT_COLLECTION_ALIASES_STABLE_FILE` is `collection_aliases`. This configuration is used to override the stable version of collection_aliases file to restore. e.g. `collection_aliases-2026-04-30_18-31-04`. The file has to exist on s3. Deploy the job!