Skip to content

Commit 2e2742c

Browse files
AlexSkrypnykAlexSkrypnyk
committed
[#1895] Added fresh DB download support for Acquia.
1 parent f7d3ee9 commit 2e2742c

9 files changed

Lines changed: 377 additions & 19 deletions

File tree

‎.ahoy.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,10 @@ commands:
120120

121121
#;< !PROVISION_TYPE_PROFILE
122122
download-db:
123-
usage: Download database. Run with "--no-cache" option to force fresh database backup.
123+
usage: Download database. Run with "--fresh" option to force fresh database backup.
124124
aliases: [fetch-db]
125125
cmd: |
126-
case " $* " in *" --no-cache "*) export VORTEX_DB_DOWNLOAD_NO_CACHE=1;; esac
126+
case " $* " in *" --fresh "*) export VORTEX_DB_DOWNLOAD_FRESH=1;; esac
127127
./scripts/vortex/download-db.sh
128128
#;> !PROVISION_TYPE_PROFILE
129129

‎.vortex/docs/content/workflows/development.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ To fetch the database with the latest data from the production environment,
4747
use the `ahoy fetch-db` command, which will download the latest database
4848
dump from the production environment into a `.data` directory. If there was a
4949
download attempt on the same day (locally or from CI), it will download the
50-
cached database dump. Use `ahoy fetch-db --no-cache` to create a new database
50+
cached database dump. Use `ahoy fetch-db --fresh` to create a new database
5151
dump regardless of the cache.
5252

5353
:::note

‎.vortex/docs/content/workflows/variables.mdx‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -586,6 +586,22 @@ Default value: `./.data`
586586

587587
Defined or used in: `.env`, `scripts/vortex/download-db-acquia.sh`, `scripts/vortex/download-db-container-registry.sh`, `scripts/vortex/download-db-ftp.sh`, `scripts/vortex/download-db-lagoon.sh`, `scripts/vortex/download-db-url.sh`, `scripts/vortex/download-db.sh`, `scripts/vortex/provision.sh`
588588

589+
### `VORTEX_DB_DOWNLOAD_ACQUIA_BACKUP_MAX_WAIT`
590+
591+
Maximum time in seconds to wait for backup completion.
592+
593+
Default value: `600`
594+
595+
Defined or used in: `scripts/vortex/download-db-acquia.sh`
596+
597+
### `VORTEX_DB_DOWNLOAD_ACQUIA_BACKUP_WAIT_INTERVAL`
598+
599+
Interval in seconds to wait between backup status checks.
600+
601+
Default value: `10`
602+
603+
Defined or used in: `scripts/vortex/download-db-acquia.sh`
604+
589605
### `VORTEX_DB_DOWNLOAD_ACQUIA_DB_NAME`
590606

591607
Acquia database name to download the database from.
@@ -612,6 +628,14 @@ Default value: `1`
612628

613629
Defined or used in: `.env.local.example`, `scripts/vortex/download-db.sh`
614630

631+
### `VORTEX_DB_DOWNLOAD_FRESH`
632+
633+
Flag to download a fresh copy of the database by triggering a new backup.
634+
635+
Default value: `UNDEFINED`
636+
637+
Defined or used in: `scripts/vortex/download-db-acquia.sh`, `scripts/vortex/download-db-lagoon.sh`
638+
615639
### `VORTEX_DB_DOWNLOAD_FTP_FILE`
616640

617641
Database dump FTP file name.
@@ -702,14 +726,6 @@ Default value: `${LAGOON_PROJECT}-${VORTEX_DB_DOWNLOAD_ENVIRONMENT}`
702726

703727
Defined or used in: `scripts/vortex/download-db-lagoon.sh`
704728

705-
### `VORTEX_DB_DOWNLOAD_NO_CACHE`
706-
707-
Flag to download a fresh copy of the database.
708-
709-
Default value: `UNDEFINED`
710-
711-
Defined or used in: `scripts/vortex/download-db-lagoon.sh`
712-
713729
### `VORTEX_DB_DOWNLOAD_PROCEED`
714730

715731
Proceed with download.

‎.vortex/installer/tests/Fixtures/install/_baseline/.ahoy.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,10 +112,10 @@ commands:
112112
cmd: ahoy cli ./scripts/vortex/login.sh
113113

114114
download-db:
115-
usage: Download database. Run with "--no-cache" option to force fresh database backup.
115+
usage: Download database. Run with "--fresh" option to force fresh database backup.
116116
aliases: [fetch-db]
117117
cmd: |
118-
case " $* " in *" --no-cache "*) export VORTEX_DB_DOWNLOAD_NO_CACHE=1;; esac
118+
case " $* " in *" --fresh "*) export VORTEX_DB_DOWNLOAD_FRESH=1;; esac
119119
./scripts/vortex/download-db.sh
120120
121121
reload-db:

‎.vortex/installer/tests/Fixtures/install/provision_profile/.ahoy.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
cmd: ahoy cli ./scripts/vortex/login.sh
44

55
- download-db:
6-
- usage: Download database. Run with "--no-cache" option to force fresh database backup.
6+
- usage: Download database. Run with "--fresh" option to force fresh database backup.
77
- aliases: [fetch-db]
88
- cmd: |
9-
- case " $* " in *" --no-cache "*) export VORTEX_DB_DOWNLOAD_NO_CACHE=1;; esac
9+
- case " $* " in *" --fresh "*) export VORTEX_DB_DOWNLOAD_FRESH=1;; esac
1010
- ./scripts/vortex/download-db.sh
1111
-
1212
reload-db:

‎.vortex/tests/bats/unit/download-db-acquia.bats‎

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -445,3 +445,279 @@ load ../_helper.bash
445445

446446
popd >/dev/null
447447
}
448+
449+
@test "download-db-acquia: Create fresh backup when requested" {
450+
pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1
451+
452+
# Clean up any existing test files
453+
rm -rf .data
454+
mkdir -p .data
455+
456+
# Create .env.local with the fresh flag
457+
echo "VORTEX_DB_DOWNLOAD_FRESH=1" >.env.local
458+
459+
declare -a STEPS=(
460+
"[INFO] Started database dump download from Acquia."
461+
462+
# Authentication
463+
"[TASK] Retrieving authentication token."
464+
'@curl -s -L https://accounts.acquia.com/api/auth/oauth/token --data-urlencode client_id=test-key --data-urlencode client_secret=test-secret --data-urlencode grant_type=client_credentials # {"access_token":"test-token","expires_in":3600}'
465+
466+
# Application UUID
467+
"[TASK] Retrieving testapp application UUID."
468+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications?filter=name%3Dtestapp # {"_embedded":{"items":[{"uuid":"app-uuid-123","name":"testapp"}]}}'
469+
470+
# Environment ID
471+
"[TASK] Retrieving prod environment ID."
472+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications/app-uuid-123/environments?filter=name%3Dprod # {"_embedded":{"items":[{"id":"env-id-456","name":"prod"}]}}'
473+
474+
# Create backup
475+
"[TASK] Creating new database backup for testdb."
476+
'@curl -s -L -X POST -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/environments/env-id-456/databases/testdb/backups # {"_links":{"notification":{"href":"https://cloud.acquia.com/api/notifications/notification-uuid-123"}}}'
477+
478+
# Wait for backup - mock status checks
479+
"[TASK] Waiting for backup to complete."
480+
'@sleep 10 # 0'
481+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/notifications/notification-uuid-123 # {"status":"in-progress"}'
482+
" Backup in progress (10s elapsed)..."
483+
'@sleep 10 # 0'
484+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/notifications/notification-uuid-123 # {"status":"completed"}'
485+
"[ OK ] Backup completed successfully."
486+
" Fresh backup will be downloaded."
487+
488+
# Continue with normal download flow
489+
"[TASK] Discovering latest backup ID for DB testdb."
490+
'@curl --progress-bar -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/environments/env-id-456/databases/testdb/backups?sort=created # {"_embedded":{"items":[{"id":"backup-id-new-123","completed":"2024-01-02T00:00:00+00:00"}]}}'
491+
492+
# Rest of download steps...
493+
"[TASK] Discovering backup URL."
494+
'@curl --progress-bar -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/environments/env-id-456/databases/testdb/backups/backup-id-new-123/actions/download # {"url":"https://backup.example.com/db-fresh.sql.gz"}'
495+
496+
"[TASK] Downloading DB dump into file .data/testdb_backup_backup-id-new-123.sql.gz."
497+
'@curl --progress-bar -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://backup.example.com/db-fresh.sql.gz -o .data/testdb_backup_backup-id-new-123.sql.gz # 0 # # echo "CREATE TABLE fresh (id INT);" | gzip > .data/testdb_backup_backup-id-new-123.sql.gz'
498+
499+
"[TASK] Expanding DB file .data/testdb_backup_backup-id-new-123.sql.gz into .data/testdb_backup_backup-id-new-123.sql."
500+
"@gunzip -t .data/testdb_backup_backup-id-new-123.sql.gz # 0"
501+
"@gunzip -c .data/testdb_backup_backup-id-new-123.sql.gz # 0 # CREATE TABLE fresh (id INT);"
502+
503+
'[TASK] Renaming file ".data/testdb_backup_backup-id-new-123.sql" to ".data/db.sql".'
504+
'@mv .data/testdb_backup_backup-id-new-123.sql .data/db.sql # 0 # # echo "CREATE TABLE fresh (id INT);" > .data/db.sql'
505+
506+
"[ OK ] Finished database dump download from Acquia."
507+
)
508+
509+
export VORTEX_ACQUIA_KEY="test-key"
510+
export VORTEX_ACQUIA_SECRET="test-secret"
511+
export VORTEX_ACQUIA_APP_NAME="testapp"
512+
export VORTEX_DB_DOWNLOAD_ENVIRONMENT="prod"
513+
export VORTEX_DB_DOWNLOAD_ACQUIA_DB_NAME="testdb"
514+
export VORTEX_DB_DIR=".data"
515+
export VORTEX_DB_FILE="db.sql"
516+
export VORTEX_DB_DOWNLOAD_FRESH="1"
517+
518+
mocks="$(run_steps "setup")"
519+
run scripts/vortex/download-db-acquia.sh
520+
run_steps "assert" "${mocks}"
521+
522+
assert_success
523+
assert_file_exists ".data/db.sql"
524+
assert_file_contains ".data/db.sql" "CREATE TABLE fresh"
525+
526+
rm -rf .data
527+
popd >/dev/null
528+
}
529+
530+
@test "download-db-acquia: Fresh backup creation fails with API error" {
531+
pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1
532+
533+
declare -a STEPS=(
534+
"[INFO] Started database dump download from Acquia."
535+
536+
# Authentication
537+
"[TASK] Retrieving authentication token."
538+
'@curl -s -L https://accounts.acquia.com/api/auth/oauth/token --data-urlencode client_id=test-key --data-urlencode client_secret=test-secret --data-urlencode grant_type=client_credentials # {"access_token":"test-token","expires_in":3600}'
539+
540+
# Application UUID
541+
"[TASK] Retrieving testapp application UUID."
542+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications?filter=name%3Dtestapp # {"_embedded":{"items":[{"uuid":"app-uuid-123","name":"testapp"}]}}'
543+
544+
# Environment ID
545+
"[TASK] Retrieving prod environment ID."
546+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications/app-uuid-123/environments?filter=name%3Dprod # {"_embedded":{"items":[{"id":"env-id-456","name":"prod"}]}}'
547+
548+
# Create backup fails with error
549+
"[TASK] Creating new database backup for testdb."
550+
'@curl -s -L -X POST -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/environments/env-id-456/databases/testdb/backups # {"error":"insufficient_permissions","message":"Insufficient permissions to create backup"}'
551+
552+
# Assert failure message
553+
"[FAIL] Failed to create backup for database 'testdb'."
554+
)
555+
556+
export VORTEX_ACQUIA_KEY="test-key"
557+
export VORTEX_ACQUIA_SECRET="test-secret"
558+
export VORTEX_ACQUIA_APP_NAME="testapp"
559+
export VORTEX_DB_DOWNLOAD_ENVIRONMENT="prod"
560+
export VORTEX_DB_DOWNLOAD_ACQUIA_DB_NAME="testdb"
561+
export VORTEX_DB_DIR=".data"
562+
export VORTEX_DB_FILE="db.sql"
563+
export VORTEX_DB_DOWNLOAD_FRESH="1"
564+
565+
mocks="$(run_steps "setup")"
566+
run scripts/vortex/download-db-acquia.sh
567+
run_steps "assert" "${mocks}"
568+
569+
assert_failure
570+
571+
popd >/dev/null
572+
}
573+
574+
@test "download-db-acquia: Fresh backup creation fails - missing notification URL" {
575+
pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1
576+
577+
declare -a STEPS=(
578+
"[INFO] Started database dump download from Acquia."
579+
580+
# Authentication
581+
"[TASK] Retrieving authentication token."
582+
'@curl -s -L https://accounts.acquia.com/api/auth/oauth/token --data-urlencode client_id=test-key --data-urlencode client_secret=test-secret --data-urlencode grant_type=client_credentials # {"access_token":"test-token","expires_in":3600}'
583+
584+
# Application UUID
585+
"[TASK] Retrieving testapp application UUID."
586+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications?filter=name%3Dtestapp # {"_embedded":{"items":[{"uuid":"app-uuid-123","name":"testapp"}]}}'
587+
588+
# Environment ID
589+
"[TASK] Retrieving prod environment ID."
590+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications/app-uuid-123/environments?filter=name%3Dprod # {"_embedded":{"items":[{"id":"env-id-456","name":"prod"}]}}'
591+
592+
# Create backup succeeds but notification URL is empty
593+
"[TASK] Creating new database backup for testdb."
594+
'@curl -s -L -X POST -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/environments/env-id-456/databases/testdb/backups # {"_links":{"notification":{"href":""}}}'
595+
596+
# Assert failure message
597+
"[FAIL] Unable to get notification URL for backup creation."
598+
)
599+
600+
export VORTEX_ACQUIA_KEY="test-key"
601+
export VORTEX_ACQUIA_SECRET="test-secret"
602+
export VORTEX_ACQUIA_APP_NAME="testapp"
603+
export VORTEX_DB_DOWNLOAD_ENVIRONMENT="prod"
604+
export VORTEX_DB_DOWNLOAD_ACQUIA_DB_NAME="testdb"
605+
export VORTEX_DB_DIR=".data"
606+
export VORTEX_DB_FILE="db.sql"
607+
export VORTEX_DB_DOWNLOAD_FRESH="1"
608+
609+
mocks="$(run_steps "setup")"
610+
run scripts/vortex/download-db-acquia.sh
611+
run_steps "assert" "${mocks}"
612+
613+
assert_failure
614+
615+
popd >/dev/null
616+
}
617+
618+
@test "download-db-acquia: Fresh backup fails during creation" {
619+
pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1
620+
621+
declare -a STEPS=(
622+
"[INFO] Started database dump download from Acquia."
623+
624+
# Authentication
625+
"[TASK] Retrieving authentication token."
626+
'@curl -s -L https://accounts.acquia.com/api/auth/oauth/token --data-urlencode client_id=test-key --data-urlencode client_secret=test-secret --data-urlencode grant_type=client_credentials # {"access_token":"test-token","expires_in":3600}'
627+
628+
# Application UUID
629+
"[TASK] Retrieving testapp application UUID."
630+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications?filter=name%3Dtestapp # {"_embedded":{"items":[{"uuid":"app-uuid-123","name":"testapp"}]}}'
631+
632+
# Environment ID
633+
"[TASK] Retrieving prod environment ID."
634+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications/app-uuid-123/environments?filter=name%3Dprod # {"_embedded":{"items":[{"id":"env-id-456","name":"prod"}]}}'
635+
636+
# Create backup
637+
"[TASK] Creating new database backup for testdb."
638+
'@curl -s -L -X POST -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/environments/env-id-456/databases/testdb/backups # {"_links":{"notification":{"href":"https://cloud.acquia.com/api/notifications/notification-uuid-123"}}}'
639+
640+
# Wait for backup - status check returns failed
641+
"[TASK] Waiting for backup to complete."
642+
'@sleep 10 # 0'
643+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/notifications/notification-uuid-123 # {"status":"failed","message":"Database backup failed due to insufficient disk space"}'
644+
645+
# Assert failure message
646+
"[FAIL] Backup creation failed."
647+
)
648+
649+
export VORTEX_ACQUIA_KEY="test-key"
650+
export VORTEX_ACQUIA_SECRET="test-secret"
651+
export VORTEX_ACQUIA_APP_NAME="testapp"
652+
export VORTEX_DB_DOWNLOAD_ENVIRONMENT="prod"
653+
export VORTEX_DB_DOWNLOAD_ACQUIA_DB_NAME="testdb"
654+
export VORTEX_DB_DIR=".data"
655+
export VORTEX_DB_FILE="db.sql"
656+
export VORTEX_DB_DOWNLOAD_FRESH="1"
657+
658+
mocks="$(run_steps "setup")"
659+
run scripts/vortex/download-db-acquia.sh
660+
run_steps "assert" "${mocks}"
661+
662+
assert_failure
663+
664+
popd >/dev/null
665+
}
666+
667+
@test "download-db-acquia: Fresh backup times out" {
668+
pushd "${LOCAL_REPO_DIR}" >/dev/null || exit 1
669+
670+
declare -a STEPS=(
671+
"[INFO] Started database dump download from Acquia."
672+
673+
# Authentication
674+
"[TASK] Retrieving authentication token."
675+
'@curl -s -L https://accounts.acquia.com/api/auth/oauth/token --data-urlencode client_id=test-key --data-urlencode client_secret=test-secret --data-urlencode grant_type=client_credentials # {"access_token":"test-token","expires_in":3600}'
676+
677+
# Application UUID
678+
"[TASK] Retrieving testapp application UUID."
679+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications?filter=name%3Dtestapp # {"_embedded":{"items":[{"uuid":"app-uuid-123","name":"testapp"}]}}'
680+
681+
# Environment ID
682+
"[TASK] Retrieving prod environment ID."
683+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/applications/app-uuid-123/environments?filter=name%3Dprod # {"_embedded":{"items":[{"id":"env-id-456","name":"prod"}]}}'
684+
685+
# Create backup
686+
"[TASK] Creating new database backup for testdb."
687+
'@curl -s -L -X POST -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/environments/env-id-456/databases/testdb/backups # {"_links":{"notification":{"href":"https://cloud.acquia.com/api/notifications/notification-uuid-123"}}}'
688+
689+
# Wait for backup - keep returning in-progress until timeout
690+
"[TASK] Waiting for backup to complete."
691+
'@sleep 5 # 0'
692+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/notifications/notification-uuid-123 # {"status":"in-progress"}'
693+
" Backup in progress (5s elapsed)..."
694+
'@sleep 5 # 0'
695+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/notifications/notification-uuid-123 # {"status":"in-progress"}'
696+
" Backup in progress (10s elapsed)..."
697+
'@sleep 5 # 0'
698+
'@curl -s -L -H Accept: application/json, version=2 -H Authorization: Bearer test-token https://cloud.acquia.com/api/notifications/notification-uuid-123 # {"status":"in-progress"}'
699+
" Backup in progress (15s elapsed)..."
700+
701+
# Assert timeout failure message
702+
"[FAIL] Backup creation timed out after 15 seconds."
703+
)
704+
705+
export VORTEX_ACQUIA_KEY="test-key"
706+
export VORTEX_ACQUIA_SECRET="test-secret"
707+
export VORTEX_ACQUIA_APP_NAME="testapp"
708+
export VORTEX_DB_DOWNLOAD_ENVIRONMENT="prod"
709+
export VORTEX_DB_DOWNLOAD_ACQUIA_DB_NAME="testdb"
710+
export VORTEX_DB_DIR=".data"
711+
export VORTEX_DB_FILE="db.sql"
712+
export VORTEX_DB_DOWNLOAD_FRESH="1"
713+
export VORTEX_DB_DOWNLOAD_ACQUIA_BACKUP_MAX_WAIT="15"
714+
export VORTEX_DB_DOWNLOAD_ACQUIA_BACKUP_WAIT_INTERVAL="5"
715+
716+
mocks="$(run_steps "setup")"
717+
run scripts/vortex/download-db-acquia.sh
718+
run_steps "assert" "${mocks}"
719+
720+
assert_failure
721+
722+
popd >/dev/null
723+
}

‎.vortex/tests/bats/unit/download-db-lagoon.bats‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ load ../_helper.bash
140140

141141
export LAGOON_PROJECT="testproject"
142142
export VORTEX_DB_DOWNLOAD_ENVIRONMENT="main"
143-
export VORTEX_DB_DOWNLOAD_NO_CACHE="1"
143+
export VORTEX_DB_DOWNLOAD_FRESH="1"
144144
export VORTEX_DB_DIR=".data"
145145
export VORTEX_DB_FILE="db.sql"
146146

0 commit comments

Comments
 (0)