From 47d68ed69c69f9c347908f35ae3ef17010686ad5 Mon Sep 17 00:00:00 2001 From: Elias Oehen Date: Tue, 28 Apr 2026 09:56:52 +0200 Subject: [PATCH 1/2] fix: Fix download blobxfer when BLOBXFER_REMOTE_PATH has folders --- examples/mssql-blobxfer/compose.yml | 7 ++++--- install/assets/functions/10-db-backup | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/examples/mssql-blobxfer/compose.yml b/examples/mssql-blobxfer/compose.yml index 113a22f0..948a1f4b 100644 --- a/examples/mssql-blobxfer/compose.yml +++ b/examples/mssql-blobxfer/compose.yml @@ -28,11 +28,11 @@ services: # image: tiredofit/db-backup image: tiredofit/db-backup links: - - example-mssql-s3-db + - example-mssql-s3-db volumes: - ./backups:/backup - ./tmp/backups:/tmp/backups # shared tmp backup directory - #- ./post-script.sh:/assets/custom-scripts/post-script.sh + #- ./post-script.sh:/assets/custom-scripts/post-script.sh environment: - TIMEZONE=America/Vancouver - CONTAINER_ENABLE_MONITORING=FALSE @@ -58,7 +58,8 @@ services: - DB01_BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name} # Add here azure storage account key - DB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key} - - DB01_BLOBXFER_REMOTE_PATH=docker-db-backup + # Optional nested path, e.g. test/weekly (leading/trailing slashes are not required) + - DB01_BLOBXFER_REMOTE_PATH=backup01/docker-db-backup restart: always networks: example-mssql-blobxfer-net: diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 63aec237..544551b5 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -1677,6 +1677,18 @@ EOF write_log info "Synchronize local storage from blob container with blobxfer" ${play_fair} blobxfer download --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete + # Avoid recursively re-uploading the downloaded remote path when a nested remote directory is configured. + blobxfer_remote_path_clean=$(echo "${backup_job_blobxfer_remote_path}" | sed -e 's#^/*##' -e 's#/*$##') + case "${blobxfer_remote_path_clean}" in + */*) + blobxfer_local_mirror_path="${backup_job_filesystem_path}/${blobxfer_remote_path_clean}" + if [ -d "${blobxfer_local_mirror_path}" ]; then + write_log debug "Removing local blobxfer mirror path ${blobxfer_local_mirror_path} to prevent recursive nesting" + run_as_user rm -rf "${blobxfer_local_mirror_path}" + fi + ;; + esac + write_log info "Moving backup to external storage with blobxfer" mkdir -p "${backup_job_filesystem_path}" if [ "${backup_job_checksum}" != "none" ] ; then run_as_user mv "${temporary_directory}"/*."${checksum_extension}" "${backup_job_filesystem_path}"/; fi From a3fbbad7034888d5754e942b3397c2bf51d0f105 Mon Sep 17 00:00:00 2001 From: Elias Oehen Date: Tue, 28 Apr 2026 10:00:35 +0200 Subject: [PATCH 2/2] feat: Add BLOBXFER_REMOTE_DOWNLOAD flag --- README.md | 4 +++ examples/mssql-blobxfer/compose.yml | 2 ++ install/assets/defaults/10-db-backup | 1 + install/assets/functions/10-db-backup | 38 +++++++++++++++++---------- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5feda898..3674bf6c 100644 --- a/README.md +++ b/README.md @@ -362,10 +362,12 @@ If `DEFAULT_BACKUP_LOCATION` = `blobxfer` then the following options are used:. | -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- | | `DEFAULT_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | | `DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | +| `DEFAULT_BLOBXFER_REMOTE_DOWNLOAD` | Download remote path before upload to keep local/remote in sync | `TRUE` | x | | `DEFAULT_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | | `DEFAULT_BLOBXFER_MODE` | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto` | x | - When `DEFAULT_BLOBXFER_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. +- Set `DEFAULT_BLOBXFER_REMOTE_DOWNLOAD=FALSE` to skip downloading remote files before upload. Keeping this set to `TRUE` is recommended when you use cleanup sync behavior. > This service uploads files from backup targed directory `DEFAULT_FILESYSTEM_PATH`. > If the a cleanup configuration in `DEFAULT_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. @@ -645,10 +647,12 @@ If `DB01_BACKUP_LOCATION` = `blobxfer` then the following options are used:. | -------------------------------------- | ------------------------------------------------------------------- | ------------------- | ------- | | `DB01_BLOBXFER_STORAGE_ACCOUNT` | Microsoft Azure Cloud storage account name. | | x | | `DB01_BLOBXFER_STORAGE_ACCOUNT_KEY` | Microsoft Azure Cloud storage account key. | | x | +| `DB01_BLOBXFER_REMOTE_DOWNLOAD` | Download remote path before upload to keep local/remote in sync | `TRUE` | x | | `DB01_BLOBXFER_REMOTE_PATH` | Remote Azure path | `/docker-db-backup` | x | | `DB01_BLOBXFER_REMOTE_MODE` | Azure Storage mode e.g. `auto`, `file`, `append`, `block` or `page` | `auto` | x | - When `DEFAULT_BLOBXFER_MODE` is set to auto it will use blob containers by default. If the `DEFAULT_BLOBXFER_REMOTE_PATH` path does not exist a blob container with that name will be created. +- Set `DB01_BLOBXFER_REMOTE_DOWNLOAD=FALSE` to skip downloading remote files before upload. Keeping this set to `TRUE` is recommended when you use cleanup sync behavior. > This service uploads files from backup directory `DB01_BACKUP_FILESYSTEM_PATH`. > If the a cleanup configuration in `DB01_CLEANUP_TIME` is defined, the remote directory on Azure storage will also be cleaned automatically. diff --git a/examples/mssql-blobxfer/compose.yml b/examples/mssql-blobxfer/compose.yml index 948a1f4b..148f62d7 100644 --- a/examples/mssql-blobxfer/compose.yml +++ b/examples/mssql-blobxfer/compose.yml @@ -58,6 +58,8 @@ services: - DB01_BLOBXFER_STORAGE_ACCOUNT={TODO Add Storage Name} # Add here azure storage account key - DB01_BLOBXFER_STORAGE_ACCOUNT_KEY={TODO Add Key} + # Keep TRUE to sync remote/local state before upload + - DB01_BLOBXFER_REMOTE_DOWNLOAD=TRUE # Optional nested path, e.g. test/weekly (leading/trailing slashes are not required) - DB01_BLOBXFER_REMOTE_PATH=backup01/docker-db-backup restart: always diff --git a/install/assets/defaults/10-db-backup b/install/assets/defaults/10-db-backup index 3ad56f65..f8fc0a24 100644 --- a/install/assets/defaults/10-db-backup +++ b/install/assets/defaults/10-db-backup @@ -7,6 +7,7 @@ DBBACKUP_GROUP=${DBBACKUP_GROUP:-"${DBBACKUP_USER}"} # Must go after DBBACKUP_US DEFAULT_BACKUP_BEGIN=${DEFAULT_BACKUP_BEGIN:-+0} DEFAULT_BACKUP_INTERVAL=${DEFAULT_BACKUP_INTERVAL:-1440} DEFAULT_BACKUP_LOCATION=${DEFAULT_BACKUP_LOCATION:-"FILESYSTEM"} +DEFAULT_BLOBXFER_REMOTE_DOWNLOAD=${DEFAULT_BLOBXFER_REMOTE_DOWNLOAD:-"TRUE"} DEFAULT_BLOBXFER_REMOTE_PATH=${DEFAULT_BLOBXFER_REMOTE_PATH:-"/docker-db-backup"} DEFAULT_BLOBXFER_MODE=${DEFAULT_BLOBXFER_MODE:-"auto"} DEFAULT_CHECKSUM=${DEFAULT_CHECKSUM:-"MD5"} diff --git a/install/assets/functions/10-db-backup b/install/assets/functions/10-db-backup index 544551b5..75c50665 100644 --- a/install/assets/functions/10-db-backup +++ b/install/assets/functions/10-db-backup @@ -65,6 +65,7 @@ bootstrap_variables() { DEFAULT_S3_CERT_CA_FILE \ DEFAULT_BLOBXFER_STORAGE_ACCOUNT \ DEFAULT_BLOBXFER_STORAGE_ACCOUNT_KEY \ + DEFAULT_BLOBXFER_REMOTE_DOWNLOAD \ DEFAULT_BLOBXFER_REMOTE_PATH \ DEFAULT_BLOBXFER_MODE \ DB"${backup_instance_number}"_AUTH \ @@ -93,10 +94,12 @@ bootstrap_variables() { DB"${backup_instance_number}"_S3_CERT_CA_FILE \ DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT \ DB"${backup_instance_number}"_BLOBXFER_STORAGE_ACCOUNT_KEY \ + DB"${backup_instance_number}"_BLOBXFER_REMOTE_DOWNLOAD \ DB"${backup_instance_number}"_BLOBXFER_REMOTE_PATH \ DB"${backup_instance_number}"_BLOBXFER_MODE \ BLOBXFER_STORAGE_ACCOUNT \ BLOBXFER_STORAGE_ACCOUNT_KEY \ + BLOBXFER_REMOTE_DOWNLOAD \ DB_HOST \ DB_NAME \ DB_PORT \ @@ -206,6 +209,7 @@ bootstrap_variables() { transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_REMOTE_PATH backup_job_blobxfer_remote_path transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT backup_job_blobxfer_storage_account transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_STORAGE_ACCOUNT_KEY backup_job_blobxfer_storage_account_key + transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_REMOTE_DOWNLOAD backup_job_blobxfer_remote_download transform_backup_instance_variable "${backup_instance_number}" BLOBXFER_MODE backup_job_blobxfer_mode transform_backup_instance_variable "${backup_instance_number}" CHECKSUM backup_job_checksum transform_backup_instance_variable "${backup_instance_number}" CLEANUP_TIME backup_job_cleanup_time @@ -1109,6 +1113,8 @@ cleanup_old_data() { if [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions" + elif var_false "${backup_job_blobxfer_remote_download}"; then + write_log info "Skipping remote cleanup sync because _BLOBXFER_REMOTE_DOWNLOAD is disabled" else write_log info "Syncing changes via blobxfer" silent run_as_user blobxfer upload --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --delete --delete-only @@ -1674,20 +1680,24 @@ EOF if [ -z "${backup_job_blobxfer_storage_account}" ] || [ -z "${backup_job_blobxfer_storage_account_key}" ]; then write_log warn "Variable _BLOBXFER_STORAGE_ACCOUNT or _BLOBXFER_STORAGE_ACCOUNT_KEY is not set. Skipping blobxfer functions" else - write_log info "Synchronize local storage from blob container with blobxfer" - ${play_fair} blobxfer download --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete - - # Avoid recursively re-uploading the downloaded remote path when a nested remote directory is configured. - blobxfer_remote_path_clean=$(echo "${backup_job_blobxfer_remote_path}" | sed -e 's#^/*##' -e 's#/*$##') - case "${blobxfer_remote_path_clean}" in - */*) - blobxfer_local_mirror_path="${backup_job_filesystem_path}/${blobxfer_remote_path_clean}" - if [ -d "${blobxfer_local_mirror_path}" ]; then - write_log debug "Removing local blobxfer mirror path ${blobxfer_local_mirror_path} to prevent recursive nesting" - run_as_user rm -rf "${blobxfer_local_mirror_path}" - fi - ;; - esac + if var_true "${backup_job_blobxfer_remote_download}"; then + write_log info "Synchronize local storage from blob container with blobxfer" + ${play_fair} blobxfer download --no-overwrite --mode ${backup_job_blobxfer_mode} --remote-path ${backup_job_blobxfer_remote_path} --storage-account ${backup_job_blobxfer_storage_account} --storage-account-key ${backup_job_blobxfer_storage_account_key} --local-path ${backup_job_filesystem_path} --restore-file-lmt --delete + + # Avoid recursively re-uploading the downloaded remote path when a nested remote directory is configured. + blobxfer_remote_path_clean=$(echo "${backup_job_blobxfer_remote_path}" | sed -e 's#^/*##' -e 's#/*$##') + case "${blobxfer_remote_path_clean}" in + */*) + blobxfer_local_mirror_path="${backup_job_filesystem_path}/${blobxfer_remote_path_clean}" + if [ -d "${blobxfer_local_mirror_path}" ]; then + write_log debug "Removing local blobxfer mirror path ${blobxfer_local_mirror_path} to prevent recursive nesting" + run_as_user rm -rf "${blobxfer_local_mirror_path}" + fi + ;; + esac + else + write_log info "Skipping blobxfer remote download because _BLOBXFER_REMOTE_DOWNLOAD is disabled" + fi write_log info "Moving backup to external storage with blobxfer" mkdir -p "${backup_job_filesystem_path}"