feat: backup-qdrant-collection-aliases

Author: bennotieno

qdrant-backup-restore/.env.sample

@@ -8,3 +8,5 @@ export QDRANT_S3_ENDPOINT_URL="http://minio.default.svc.cluster.local:9000" # up
 export QDRANT_RESTORE_HOSTS="http://localhost:6334" # update to target restore host(s) (for restore only otherwise leave as empty string).
 export GET_PEERS_FROM_CLUSTER_INFO="false" # only enable for kubernetes cluster deployment with qdrant cluster.enabled option set as true.
 export MC_CONFIG_DIR="mc" # controls where mc S3 client store its configurations (for restore or get_snap_s3 only).
+export BACKUP_COLLECTION_ALIASES_ON_S3="true" # enable to backup collection aliases to s3 bucket.
+export QDRANT_COLLECTION_ALIASES_STABLE_FILE="collection_aliases" # override the stable version of collection_aliases file to restore.

qdrant-backup-restore/k8s/README.md

@@ -120,6 +120,7 @@ Update the following environment varibles in your copy of `backup-cronjob.yaml`
 5. `CURL_TIMEOUT` is set at `3000` seconds. This is the max time curl will wait for request to complete. Its increased in scenarios where backup takes a while.
 6. `QDRANT_S3_LINK_EXPIRY_DURATION` is set at `3600`. This is the duration that an s3 presigned url will be active. The url is used during the recovery process.
 7. `QDRANT_WAIT_ON_TASK` is set as `true`. This configuation make backup process synchronous meaning 'wait for snapshot process to finish successfully before moving on'. Its used during backup and recovery.
+8. `BACKUP_COLLECTION_ALIASES_ON_S3` is set as `false`. This configuration is used to toggle backup of collection aliases to S3. The collection_aliases are versioned by the timestamp.
 
 Deploy the job!
 
@@ -148,6 +149,8 @@ Update the following environment varibles accordingly;
 7. `QDRANT_WAIT_ON_TASK` is set as `true`. This configuation make restoration process synchronous meaning 'wait for snapshot process to finish successfully before moving on'. Its used during backup and recovery.
 8. `QDRANT_SNAPSHOT_DATETIME_FILTER` is empty. Setting this filters out snapshot/backups belonging to a certain date and time using glob pattern matching. e.g `"2026-01-29"` = all snapshots in 29th January 2026, `2026-01` = all backups in January 2026.
 9. `MC_CONFIG_DIR` is `mc`. This overrides the default storage location ($HOME) for mc s3 client configurations. Essential in set ups that use stricter securityContext configuration like `readOnlyRootFilesystem: true`.
+10. `BACKUP_COLLECTION_ALIASES_ON_S3` is `false`. When this is set to `true` during restoration it will fetch the lastest `collection_alias` from S3.
+11. `QDRANT_COLLECTION_ALIASES_STABLE_FILE` is `collection_aliases`. This configuration is used to override the stable version of collection_aliases file to restore. e.g. `collection_aliases-2026-04-30_18-31-04`. The file has to exist on s3.
 
 Deploy the job!
 

qdrant-backup-restore/qdrant_backup_recovery.sh

@@ -7,11 +7,14 @@ QDRANT_FAILED_RECOVERY_FILE="failed_snapshot_recovery"
 QDRANT_SNAPSHOT_RECOVERY_HISTORY_FILE="snapshot_recovery_history"
 QDRANT_ALIAS_RECOVERY_HISTORY_FILE="alias_recovery_history"
 QDRANT_COLLECTION_ALIASES="collection_aliases"
+BACKUP_COLLECTION_ALIASES_ON_S3=${BACKUP_COLLECTION_ALIASES_ON_S3:-false}
+QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER="collection_aliases_store"
 QDRANT_WAIT_ON_TASK=${QDRANT_WAIT_ON_TASK:-true}
 CURL_TIMEOUT="${CURL_TIMEOUT:-1800}"
 QDRANT_S3_ALIAS="qdrant_s3_snaphost"
 QDRANT_S3_LINK_EXPIRY_DURATION="${QDRANT_S3_LINK_EXPIRY_DURATION:-3600s}"
 QDRANT_HTTP_PORT="${QDRANT_HTTP_PORT:-6333}"
+QDRANT_COLLECTION_ALIASES_STABLE_FILE="${QDRANT_COLLECTION_ALIASES_STABLE_FILE:-collection_aliases}"
 
 declare -a peer_uri_map;
 # printf wrapper helper
@@ -510,6 +513,30 @@ get_collection_aliases() {
     done <<< "$_result"
   fi
 
+if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then
+    local ts
+    ts=$(date '+%Y-%m-%d_%H-%M-%S')
+
+    local base_path="$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER"
+
+    local versioned_dest="$base_path/${QDRANT_COLLECTION_ALIASES}-$ts"
+    local stable_dest="$base_path/${QDRANT_COLLECTION_ALIASES}"
+
+    _printf "backing up collection aliases to s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME"
+
+    if ! mc cp "$QDRANT_COLLECTION_ALIASES" "$versioned_dest"; then
+      _printf "failed to upload versioned backup to %s\n" "$QDRANT_S3_BUCKET_NAME"
+      exit 1
+    fi
+
+    if ! mc cp "$versioned_dest" "$stable_dest"; then
+      _printf "failed to update stable backup pointer in %s\n" "$QDRANT_S3_BUCKET_NAME"
+      exit 1
+    fi
+
+    _printf "successfully backed up collection aliases to s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME"
+  fi
+
   _printf "%s file updated,found %d collection alias(es)!\n" "$QDRANT_COLLECTION_ALIASES" "$colla_length"
 }
 
@@ -550,6 +577,16 @@ recover_collection_alias() {
 
 # restores collection aliases to a qdrant node and appends progress to $QDRANT_ALIAS_RECOVERY_HISTORY_FILE
 recover_collection_aliases() {
+
+  if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then
+    if mc cp "$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$QDRANT_COLLECTION_ALIASES_BACKUP_FOLDER/$QDRANT_COLLECTION_ALIASES_STABLE_FILE" "$QDRANT_COLLECTION_ALIASES"; then
+      _printf "collection aliases restored from s3 bucket %s\n" "$QDRANT_S3_BUCKET_NAME"
+    else
+      _printf "failed to restore collection aliases file (%s) from s3 bucket %s\n" "$QDRANT_COLLECTION_ALIASES_STABLE_FILE" "$QDRANT_S3_BUCKET_NAME"
+      exit 1
+    fi
+  fi
+
   if [[ ! -s "$QDRANT_COLLECTION_ALIASES" ]]; then
       get_collection_aliases
   fi
@@ -562,7 +599,7 @@ recover_collection_aliases() {
   colla_count=$(wc -l < $QDRANT_COLLECTION_ALIASES)
   recovered_colla_count=0
   failed_recovered_colla_count=0
-
+  colla_skipped_count=0
   local host="${restore_hosts[0]}"
 
   touch $QDRANT_ALIAS_RECOVERY_HISTORY_FILE
@@ -577,12 +614,13 @@ recover_collection_aliases() {
 
     if [ "$history_count" -gt 0 ]; then
       _printf "[%s] collection alias %s:%s already recovered, skipping!\n" "$host" "$collection_name" "$alias_name"
+      ((colla_skipped_count=colla_skipped_count+1))
       continue
     fi
     recover_collection_alias "$collection_name" "$alias_name"
   done < $QDRANT_COLLECTION_ALIASES
 
-  _printf "recovery summary: %d/%d collection aliases recovered, %d failed.\n" "$recovered_colla_count" "$colla_count" "$failed_recovered_colla_count"
+  _printf "recovery summary: %d/%d collection aliases recovered, %d failed, %d skipped.\n" "$recovered_colla_count" "$colla_count" "$failed_recovered_colla_count" "$colla_skipped_count"
 
   colla_count=0
   recovered_colla_count=0