Skip to content

Commit 4cf4ef4

Browse files
herzog0herzog0
authored
Feedback PR for boostorg#2287: Story 2269: Enable Conditional S3 Storage for development (boostorg#2291)
1 parent dde67fc commit 4cf4ef4

4 files changed

Lines changed: 201 additions & 204 deletions

File tree

‎README.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,51 @@ To work with mailinglist data locally, the django application expects to be
174174
able to query a copy of the hyperkitty database from HYPERKITTY_DATABASE_NAME.
175175
Then, `just manage sync_mailinglist_stats` management command can be run.
176176

177+
## Syncing Large Static Images
178+
179+
Large static images and other assets are stored in S3 buckets rather than in the repository. Use the `scripts/sync-large-static-images.sh` script to manage these files.
180+
181+
### Prerequisites
182+
183+
- `awscli` must be installed.
184+
- Configure a set of credentials in your `~/.aws/credentials` file with the profile name `upload-images`:
185+
186+
```ini
187+
[sync-boost-images]
188+
aws_access_key_id = <your_key_id>
189+
aws_secret_access_key = <your_secret_key>
190+
```
191+
192+
### Usage
193+
194+
The script supports both uploading and downloading files.
195+
196+
#### Uploading
197+
198+
To upload files from your local directory (`static/static-large/`) to the default S3 bucket:
199+
200+
```shell
201+
$ just up_sync_images
202+
```
203+
204+
To upload to all S3 buckets:
205+
206+
```shell
207+
$ just up_sync_images_all_buckets
208+
```
209+
210+
*Note: The script will prompt you for the S3 destination path, defaulting to `/static/img/v3`.*
211+
212+
#### Downloading
213+
214+
To download missing or outdated static items from the staging bucket to your local directory:
215+
216+
```shell
217+
$ just down_sync_images
218+
```
219+
220+
---
221+
177222
## Deploying
178223

179224
TDB

‎justfile‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -184,18 +184,11 @@ alias shell := console
184184
docker compose run --rm web python manage.py {{ args }}
185185

186186
# Static File Management
187-
@down_sync_images BUCKET='stage.boost.org.v2': ## syncs all items from specified bucket to static/static-large for local development. See scripts/upload-images.sh for required vars.
188-
if ! command -v aws &>/dev/null; then \
189-
echo "awscli is required. Please install it from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"; \
190-
exit 1; \
191-
fi
187+
@down_sync_images:
188+
scripts/sync-large-static-images.sh --down-sync;
192189

193-
if echo {{VALID_BUCKETS}} | grep -q -w '{{BUCKET}}'; then \
194-
aws s3 sync s3://{{BUCKET}}/static/ static/static-large/ --profile 'upload-images' --delete; \
195-
echo "All missing or outdated static items synced."; \
196-
else \
197-
echo "Bucket name invalid."; \
198-
fi
190+
@up_sync_images:
191+
scripts/sync-large-static-images.sh --up-sync;
199192

200-
@upload_images:
201-
scripts/upload-images.sh;
193+
@up_sync_images_all_buckets:
194+
scripts/sync-large-static-images.sh --up-sync --all-buckets;

‎scripts/sync-large-static-images.sh‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
#!/usr/bin/env bash
2+
# sync-large-static-images.sh — Upload local images to S3 buckets
3+
4+
set -euo pipefail
5+
6+
DEFAULT_BUCKET="stage.boost.org.v2"
7+
S3_BUCKETS="boost.org.v2 boost.org-cppal-dev-v2 ${DEFAULT_BUCKET}"
8+
AWS_PROFILE='sync-boost-images'
9+
DEFAULT_DEST="/static/"
10+
DEST_PATH=${DEFAULT_DEST}
11+
12+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
13+
SOURCE_DIR="$(dirname "$SCRIPT_DIR")/static/static-large/"
14+
15+
# ─────────────────────────────────────────────
16+
# PARSE COMMAND-LINE OPTIONS
17+
# ─────────────────────────────────────────────
18+
usage() {
19+
cat <<EOF
20+
Usage: $0 [OPTION]
21+
22+
Upload or download image files and other static assets to/from the website S3 buckets.
23+
24+
Options:
25+
--up-sync Upload files from the default source dir (${SOURCE_DIR}) to S3 buckets.
26+
--down-sync Download files from S3 stage bucket to local static directory (${SOURCE_DIR}).
27+
--all-buckets When used with --up-sync, upload to all buckets instead of the default bucket.
28+
--help Display this help and exit.
29+
30+
Configuration:
31+
The default destination for upload is ${DEFAULT_DEST}.
32+
In your .aws/credentials file, add a set of credentials:
33+
34+
[${AWS_PROFILE}]
35+
aws_access_key_id = <your_key_id>
36+
aws_secret_access_key = <your_secret_key>
37+
EOF
38+
}
39+
40+
validate_dependencies() {
41+
if ! command -v aws &>/dev/null; then
42+
echo "awscli is required. Please install it from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html"
43+
exit 1
44+
fi
45+
}
46+
47+
upload_images() {
48+
local all_buckets="${1:-false}"
49+
# ─────────────────────────────────────────────
50+
# CHECK FOR AWS CLI
51+
# ─────────────────────────────────────────────
52+
validate_dependencies
53+
54+
echo ""
55+
echo "Source files found in $SOURCE_DIR:"
56+
ls -1 "$SOURCE_DIR"
57+
58+
echo "Destination path: $DEST_PATH"
59+
60+
# ─────────────────────────────────────────────
61+
# UPLOAD TO BUCKETS
62+
# ─────────────────────────────────────────────
63+
UPLOAD_FAILED=0
64+
65+
BUCKETS_TO_UPLOAD="$DEFAULT_BUCKET"
66+
if [[ "$all_buckets" == "true" ]]; then
67+
BUCKETS_TO_UPLOAD="$S3_BUCKETS"
68+
fi
69+
70+
for bucket in $BUCKETS_TO_UPLOAD; do
71+
S3_DEST="s3://${bucket}${DEST_PATH}"
72+
echo ""
73+
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
74+
echo "Uploading to: $S3_DEST"
75+
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
76+
77+
# Capture exit code; show stdout+stderr live via tee to /dev/null trick
78+
set +e
79+
aws s3 sync --profile "${AWS_PROFILE}" "$SOURCE_DIR" "$S3_DEST" 2>&1
80+
EXIT_CODE=$?
81+
set -e
82+
83+
if [[ $EXIT_CODE -ne 0 ]]; then
84+
echo ""
85+
echo "The upload failed for bucket: $bucket (exit code $EXIT_CODE)"
86+
echo " Source: $SOURCE_DIR"
87+
echo " Destination: $S3_DEST"
88+
UPLOAD_FAILED=1
89+
else
90+
echo ""
91+
echo "✓ Upload succeeded for bucket: $bucket"
92+
fi
93+
done
94+
95+
# ─────────────────────────────────────────────
96+
# POST-UPLOAD SUMMARY
97+
# ─────────────────────────────────────────────
98+
echo ""
99+
if [[ $UPLOAD_FAILED -ne 0 ]]; then
100+
echo "One or more uploads failed. Please review the output above for details."
101+
echo " Source folder: $SOURCE_DIR"
102+
echo " Destination: $DEST_PATH"
103+
echo " Buckets: $S3_BUCKETS"
104+
exit 1
105+
fi
106+
107+
echo ""
108+
echo "Done."
109+
}
110+
111+
download_images() {
112+
validate_dependencies
113+
114+
if echo "${S3_BUCKETS}" | grep -q -w "${DEFAULT_BUCKET}"; then
115+
aws s3 sync "s3://${DEFAULT_BUCKET}/static/" "${SOURCE_DIR}" --profile "${AWS_PROFILE}";
116+
echo "All missing or outdated static items synced.";
117+
else
118+
echo "Bucket name invalid: ${DEFAULT_BUCKET}";
119+
exit 1;
120+
fi
121+
}
122+
123+
124+
125+
ALL_BUCKETS=false
126+
while [[ $# -gt 0 ]]; do
127+
case "$1" in
128+
--help)
129+
usage
130+
exit 0
131+
;;
132+
--all-buckets)
133+
ALL_BUCKETS=true
134+
;;
135+
--up-sync)
136+
upload_images "$ALL_BUCKETS"
137+
exit $?
138+
;;
139+
--down-sync)
140+
download_images
141+
exit $?
142+
;;
143+
*)
144+
echo "Unknown option: $1"
145+
usage
146+
exit 1
147+
;;
148+
esac
149+
shift
150+
done

0 commit comments

Comments
 (0)