diff --git a/UPGRADE-OPERATIONS.md b/UPGRADE-OPERATIONS.md index e5d5bff..5ca62c7 100644 --- a/UPGRADE-OPERATIONS.md +++ b/UPGRADE-OPERATIONS.md @@ -737,11 +737,11 @@ nano .env # Qdrant API key export QDRANT_API_KEY="your-api-key-here" -# Source hosts (where snapshots are stored) -# Port forward qdrant service to local -export QDRANT_SOURCE_HOSTS="http://localhost:6333" +# Source hosts (leave empty for restore - collections are discovered from S3) +export QDRANT_SOURCE_HOSTS="" # Restore hosts (target Qdrant instances) +# Port forward qdrant service to local export QDRANT_RESTORE_HOSTS="http://localhost:6334" # S3 configuration (for fetching snapshots) @@ -750,8 +750,8 @@ export QDRANT_S3_ACCESS_KEY_ID="your-access-key" export QDRANT_S3_SECRET_ACCESS_KEY="your-secret-key" export QDRANT_S3_BUCKET_NAME="qdrant-snapshots" -# Auto-discover peers -export GET_PEERS_FROM_CLUSTER_INFO="true" +# Disable Auto-discover of peers +export GET_PEERS_FROM_CLUSTER_INFO="false" # Timeout settings export CURL_TIMEOUT="1800" diff --git a/qdrant-backup-restore/k8s/README.md b/qdrant-backup-restore/k8s/README.md index 909f2e7..a588d23 100644 --- a/qdrant-backup-restore/k8s/README.md +++ b/qdrant-backup-restore/k8s/README.md @@ -163,9 +163,9 @@ kubectl -n logs -lapp=qdrant-backup -f Update the following environment varibles accordingly; 1. Update the `secretKeyRef.name` on all `env:` entries under `env.[*].valueFrom` to the name of the kubernetes secrets deployed when configuring Qdrant. -2. Update the `QDRANT_SOURCE_HOSTS` to the kubernetes service domain of the source Qdrant deployment. +2. Update the `QDRANT_SOURCE_HOSTS` to empty `""`. 3. Update the `QDRANT_RESTORE_HOSTS` to the target qdrant cluster service domain. -4. `GET_PEERS_FROM_CLUSTER_INFO` should be `true` to discover qdrant cluster peers when backing up collections. +4. `GET_PEERS_FROM_CLUSTER_INFO` should be `false` during restore. 5. `CURL_TIMEOUT` is set at `3000` seconds. This is the max time curl will wait for request to complete. Its increased in scenarios where restoration takes a while. 6. `QDRANT_S3_LINK_EXPIRY_DURATION` is set at `3600`. This is the duration that an s3 presigned url will be active. The url is used during the recovery process. 7. `QDRANT_WAIT_ON_TASK` is set as `true`. This configuation make restoration process synchronous meaning 'wait for snapshot process to finish successfully before moving on'. Its used during backup and recovery. diff --git a/qdrant-backup-restore/k8s/backup-job.yaml b/qdrant-backup-restore/k8s/backup-job.yaml index 1cf3f7e..5e09393 100644 --- a/qdrant-backup-restore/k8s/backup-job.yaml +++ b/qdrant-backup-restore/k8s/backup-job.yaml @@ -26,11 +26,30 @@ spec: - -c - "cd scripts && ./qdrant_backup_recovery.sh create_snap" env: + - name: QDRANT_S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: qdrant-credentials-minio # change this to your qdrant kubernetes secret name + key: QDRANT_S3_ACCESS_KEY_ID + - name: QDRANT_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: qdrant-credentials-minio # change this to your qdrant kubernetes secret name + key: QDRANT_S3_SECRET_ACCESS_KEY + - name: QDRANT_S3_BUCKET_NAME + valueFrom: + secretKeyRef: + name: qdrant-credentials-minio # change this to your qdrant kubernetes secret name + key: QDRANT_S3_BUCKET_NAME - name: QDRANT_API_KEY valueFrom: secretKeyRef: name: qdrant-credentials-minio # change this to your qdrant kubernetes secret name key: QDRANT_API_KEY + - name: QDRANT_S3_ENDPOINT_URL + value: "http://minio.default.svc.cluster.local:9000" # change this to your s3 endpoint url + - name: MC_CONFIG_DIR + value: "mc" # sets the configuration location for mc S3 client - name: QDRANT_SOURCE_HOSTS value: "http://qdrant-headless:6333" # change this to your qdrant kubernetes headless service name - name: QDRANT_RESTORE_HOSTS @@ -39,6 +58,8 @@ spec: value: "true" - name: CURL_TIMEOUT value: "3000" # increase if more time is needed for a single backup + - name: BACKUP_COLLECTION_ALIASES_ON_S3 + value: "true" resources: requests: cpu: "100m" diff --git a/qdrant-backup-restore/k8s/configmap-script.yaml b/qdrant-backup-restore/k8s/configmap-script.yaml index 88a4419..4eccbbb 100644 --- a/qdrant-backup-restore/k8s/configmap-script.yaml +++ b/qdrant-backup-restore/k8s/configmap-script.yaml @@ -83,7 +83,11 @@ data: fi _printf "fetching collections!\n" - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available for fetching collections. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi collections_count=0 @@ -108,6 +112,13 @@ data: local result="$2" _result=$(jq -c '.result.collections // [] | .[]' <<< "$result") + # Guard against the bash `<<<` quirk: a here-string of an empty value still + # produces a single newline, so the loop below would iterate once with an + # empty item and write a bogus `host,` line to $QDRANT_COLLECTIONS_FILE. + if [[ -z "$_result" ]]; then + return + fi + while read -r item; do local collection_name="" collection_name=$(jq -r '.name' <<< "$item") @@ -141,7 +152,11 @@ data: # creates and appends created collection snapshot from a qdrant node to $QDRANT_SNAPSHOTS_FILE track_created_collection_snapshot() { - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available for fetching collection snapshots. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi local collection_name="$1" local result="" local status="" @@ -342,11 +357,61 @@ data: fi } + # discovers collection names from the S3 snapshots prefix and stores them in $QDRANT_COLLECTIONS_FILE + fetch_collections_from_s3() { + local prefix="snapshots" + local result="" + + _printf "fetching collections from s3 bucket %s under %s/\n" "$QDRANT_S3_BUCKET_NAME" "$prefix" + + result=$(mc ls --json "$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$prefix/") + + if [ "$result" = "" ]; then + _printf "no collections found in s3 path: %s/%s/ ...skipping!\n" "$QDRANT_S3_BUCKET_NAME" "$prefix" + return + fi + + collections_count=0 + _result=$(jq -c '.' <<< "$result") + + while read -r item; do + local status="" + local object_key="" + local collection_name="" + + status=$(jq -r '.status?' <<< "$item") + if [ "$status" != "success" ]; then + _printf "failed to list collections from s3, got this instead %s\n" "${item//[[:space:]]/}" + continue + fi + + object_key=$(jq -r '.key' <<< "$item") # e.g. "my_collection/" from mc ls on snapshots/ + object_key="${object_key%/}" # strip trailing slash (folder marker) + collection_name="${object_key##*/}" # use last path segment as collection name + + if [[ -z "$collection_name" || "$collection_name" == "$prefix" ]]; then + continue + fi + + ((collections_count=collections_count+1)) + printf ',%s\n' "$collection_name" >> $QDRANT_COLLECTIONS_FILE + done <<< "$_result" + + _printf "%s file updated, found %d collection(s) from s3!\n" "$QDRANT_COLLECTIONS_FILE" "$collections_count" + collections_count=0 + } + # generates a list of collection snapshots from a s3 and appends it to $QDRANT_SNAPSHOTS_FILE generate_snapshot_file_from_s3() { local datetime="${1:-}" + if [[ ! -s "$QDRANT_COLLECTIONS_FILE" ]]; then - get_collections + fetch_collections_from_s3 + fi + + if [ ! -f "$QDRANT_COLLECTIONS_FILE" ]; then + _printf "non error exit since file %s does not exist.\n" "$QDRANT_COLLECTIONS_FILE" + exit 0 fi snapshot_count=0 @@ -496,7 +561,11 @@ data: # gets the collection aliases from a qdrant node and appends it to $QDRANT_COLLECTION_ALIASES file get_collection_aliases() { - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available for fetching collection aliases. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi local result="" local status="" @@ -597,7 +666,11 @@ data: # restores collection aliases to a qdrant node and appends progress to $QDRANT_ALIAS_RECOVERY_HISTORY_FILE recover_collection_aliases() { - local host="${restore_hosts[0]}" + local host="${restore_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "restore host is not available for recovering collection aliases. Please ensure QDRANT_RESTORE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then @@ -667,7 +740,12 @@ data: # gets qdrant peer host url using cluster info endpoint. Sets the port to $QDRANT_HTTP_PORT the http port. get_peers_from_cluster_info() { - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables. If restoring snapshots set QDRANT_SOURCE_HOSTS and GET_PEERS_FROM_CLUSTER_INFO to empty string.\n" + exit 1 + fi + result=$(_curl GET "$host/cluster" --header "api-key: $QDRANT_API_KEY") entries=$(jq -r '.result.peers // {}' <<< "$result") peer_uri_entries=$(jq -r 'to_entries[] | "\(.key) \(.value.uri)"' <<< "$entries") diff --git a/qdrant-backup-restore/k8s/configmap-snapshot.yaml b/qdrant-backup-restore/k8s/configmap-snapshot.yaml index af152a9..31912c6 100644 --- a/qdrant-backup-restore/k8s/configmap-snapshot.yaml +++ b/qdrant-backup-restore/k8s/configmap-snapshot.yaml @@ -6,6 +6,12 @@ metadata: app: qdrant-snapshots-file component: script data: + # this is used to provide a list of snapshots to restore (comma separated list of collection names and snapshot names, host is optional (first column is host)) snapshots: | ,_default_256,_default_256-5528114518172461-2026-01-29-11-44-26.snapshot ,midlib,midlib-567156434043892-2026-01-29-11-44-27.snapshot + + # this is used to provide a list of collections to restore (comma separated list of collection names, host is optional (first column is host)) + collections: | + ,_default_256 + ,_default_128 diff --git a/qdrant-backup-restore/k8s/restore-job.yaml b/qdrant-backup-restore/k8s/restore-job.yaml index 23185fe..a7ca4f0 100644 --- a/qdrant-backup-restore/k8s/restore-job.yaml +++ b/qdrant-backup-restore/k8s/restore-job.yaml @@ -47,19 +47,21 @@ spec: name: qdrant-credentials-minio key: QDRANT_API_KEY - name: QDRANT_SOURCE_HOSTS - value: "http://qdrant-headless:6333" # change this to your source qdrant kubernetes headless service name + value: "" # leave empty if source host is not available - name: QDRANT_S3_ENDPOINT_URL value: "http://minio.default.svc.cluster.local:9000" # change this to your s3 endpoint url - name: QDRANT_RESTORE_HOSTS - value: "http://qdrantr2-headless:6333" # change this to your target qdrant kubernetes headless service name + value: "http://qdrant-headless:6333" # change this to your target qdrant kubernetes headless service name - name: GET_PEERS_FROM_CLUSTER_INFO - value: "true" + value: "false" - name: QDRANT_SNAPSHOT_DATETIME_FILTER - value: "" # change tis to filter snapshts based on date (glob matching is used) + value: "" # set this to filter snapshots based on date (glob matching is used) e.g "2026-01-29" = all snapshots in 29th January 2026, "2026-01" = all snapshots in January 2026. - name: CURL_TIMEOUT value: "3000" # increase if more time is needed for a single restore - name: MC_CONFIG_DIR value: "mc" # sets the configuration location for mc S3 client + - name: BACKUP_COLLECTION_ALIASES_ON_S3 + value: "true" resources: requests: cpu: "100m" diff --git a/qdrant-backup-restore/k8s/restore-specific-snapshot-job.yaml b/qdrant-backup-restore/k8s/restore-specific-snapshot-job.yaml index 0fb8b41..6b2818e 100644 --- a/qdrant-backup-restore/k8s/restore-specific-snapshot-job.yaml +++ b/qdrant-backup-restore/k8s/restore-specific-snapshot-job.yaml @@ -47,19 +47,21 @@ spec: name: qdrant-credentials-minio key: QDRANT_API_KEY - name: QDRANT_SOURCE_HOSTS - value: "http://qdrant-headless:6333" # change this to your source qdrant kubernetes headless service name + value: "" # leave empty if source host is not available - name: QDRANT_S3_ENDPOINT_URL value: "http://minio.default.svc.cluster.local:9000" # change this to your s3 endpoint url - name: QDRANT_RESTORE_HOSTS - value: "http://qdrantr2-headless:6333" # change this to your target qdrant kubernetes headless service name + value: "http://qdrant-headless:6333" # change this to your target qdrant kubernetes headless service name - name: GET_PEERS_FROM_CLUSTER_INFO - value: "true" + value: "false" - name: CURL_TIMEOUT value: "3000" # increase if more time is needed for a single restore - name: QDRANT_SNAPSHOT_DATETIME_FILTER value: "" # change tis to filter snapshts based on date (glob matching is used) - name: MC_CONFIG_DIR value: "mc" # sets the configuration location for mc S3 client + - name: BACKUP_COLLECTION_ALIASES_ON_S3 + value: "true" resources: requests: cpu: "100m" @@ -71,9 +73,14 @@ spec: - name: qdrant-backup-restore-script mountPath: /scripts/qdrant_backup_recovery.sh subPath: qdrant_backup_recovery.sh + # this is used to restore specific collections - name: qdrant-snapshots-file - mountPath: /scripts/snapshots - subPath: snapshots + mountPath: /scripts/collections + subPath: collections + # this is used to restore specific snapshots + # - name: qdrant-snapshots-file + # mountPath: /scripts/snapshots + # subPath: snapshots - name: scripts mountPath: /scripts volumes: diff --git a/qdrant-backup-restore/qdrant_backup_recovery.sh b/qdrant-backup-restore/qdrant_backup_recovery.sh index 6597e56..c5dbd16 100755 --- a/qdrant-backup-restore/qdrant_backup_recovery.sh +++ b/qdrant-backup-restore/qdrant_backup_recovery.sh @@ -74,7 +74,11 @@ get_collections() { fi _printf "fetching collections!\n" - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available for fetching collections. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi collections_count=0 @@ -99,6 +103,13 @@ track_collection(){ local result="$2" _result=$(jq -c '.result.collections // [] | .[]' <<< "$result") + # Guard against the bash `<<<` quirk: a here-string of an empty value still + # produces a single newline, so the loop below would iterate once with an + # empty item and write a bogus `host,` line to $QDRANT_COLLECTIONS_FILE. + if [[ -z "$_result" ]]; then + return + fi + while read -r item; do local collection_name="" collection_name=$(jq -r '.name' <<< "$item") @@ -132,7 +143,11 @@ create_snapshot_from_peer() { # creates and appends created collection snapshot from a qdrant node to $QDRANT_SNAPSHOTS_FILE track_created_collection_snapshot() { - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available for fetching collection snapshots. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi local collection_name="$1" local result="" local status="" @@ -333,11 +348,61 @@ fetch_collection_snapshot_from_s3() { fi } +# discovers collection names from the S3 snapshots prefix and stores them in $QDRANT_COLLECTIONS_FILE +fetch_collections_from_s3() { + local prefix="snapshots" + local result="" + + _printf "fetching collections from s3 bucket %s under %s/\n" "$QDRANT_S3_BUCKET_NAME" "$prefix" + + result=$(mc ls --json "$QDRANT_S3_ALIAS/$QDRANT_S3_BUCKET_NAME/$prefix/") + + if [ "$result" = "" ]; then + _printf "no collections found in s3 path: %s/%s/ ...skipping!\n" "$QDRANT_S3_BUCKET_NAME" "$prefix" + return + fi + + collections_count=0 + _result=$(jq -c '.' <<< "$result") + + while read -r item; do + local status="" + local object_key="" + local collection_name="" + + status=$(jq -r '.status?' <<< "$item") + if [ "$status" != "success" ]; then + _printf "failed to list collections from s3, got this instead %s\n" "${item//[[:space:]]/}" + continue + fi + + object_key=$(jq -r '.key' <<< "$item") # e.g. "my_collection/" from mc ls on snapshots/ + object_key="${object_key%/}" # strip trailing slash (folder marker) + collection_name="${object_key##*/}" # use last path segment as collection name + + if [[ -z "$collection_name" || "$collection_name" == "$prefix" ]]; then + continue + fi + + ((collections_count=collections_count+1)) + printf ',%s\n' "$collection_name" >> $QDRANT_COLLECTIONS_FILE + done <<< "$_result" + + _printf "%s file updated, found %d collection(s) from s3!\n" "$QDRANT_COLLECTIONS_FILE" "$collections_count" + collections_count=0 +} + # generates a list of collection snapshots from a s3 and appends it to $QDRANT_SNAPSHOTS_FILE generate_snapshot_file_from_s3() { local datetime="${1:-}" + if [[ ! -s "$QDRANT_COLLECTIONS_FILE" ]]; then - get_collections + fetch_collections_from_s3 + fi + + if [ ! -f "$QDRANT_COLLECTIONS_FILE" ]; then + _printf "non error exit since file %s does not exist.\n" "$QDRANT_COLLECTIONS_FILE" + exit 0 fi snapshot_count=0 @@ -487,7 +552,11 @@ recover_collection_snapshots(){ # gets the collection aliases from a qdrant node and appends it to $QDRANT_COLLECTION_ALIASES file get_collection_aliases() { - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available for fetching collection aliases. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi local result="" local status="" @@ -588,7 +657,11 @@ recover_collection_alias() { # restores collection aliases to a qdrant node and appends progress to $QDRANT_ALIAS_RECOVERY_HISTORY_FILE recover_collection_aliases() { - local host="${restore_hosts[0]}" + local host="${restore_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "restore host is not available for recovering collection aliases. Please ensure QDRANT_RESTORE_HOSTS is set correctly in your environment variables.\n" + exit 1 + fi if [ "$BACKUP_COLLECTION_ALIASES_ON_S3" = "true" ]; then @@ -658,7 +731,12 @@ delete_files() { # gets qdrant peer host url using cluster info endpoint. Sets the port to $QDRANT_HTTP_PORT the http port. get_peers_from_cluster_info() { - local host="${source_hosts[0]}" + local host="${source_hosts[0]:-}" + if [[ -z "$host" ]]; then + _printf "source host is not available. Please ensure QDRANT_SOURCE_HOSTS is set correctly in your environment variables. If restoring snapshots set QDRANT_SOURCE_HOSTS and GET_PEERS_FROM_CLUSTER_INFO to empty string.\n" + exit 1 + fi + result=$(_curl GET "$host/cluster" --header "api-key: $QDRANT_API_KEY") entries=$(jq -r '.result.peers // {}' <<< "$result") peer_uri_entries=$(jq -r 'to_entries[] | "\(.key) \(.value.uri)"' <<< "$entries")