Skip to content

Commit 8a1d792

Browse files
[minor] Bundle application releases and harden images
1 parent 914f500 commit 8a1d792

150 files changed

Lines changed: 3239 additions & 924 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/build.yml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ on:
3434

3535
jobs:
3636
build:
37+
env:
38+
# renovate: datasource=docker depName=moby/buildkit
39+
BUILDKIT_IMAGE: moby/buildkit:v0.31.0@sha256:a095b3d11ce1a9a05b6064ef515dfca0291ec5bcf2ea8178da8f6461924294e1
3740
strategy:
3841
matrix:
3942
runner: [ubuntu-24.04, ubuntu-24.04-arm]
@@ -138,14 +141,14 @@ jobs:
138141
done
139142
}
140143
141-
retry_with_backoff docker pull moby/buildkit:v0.31.0
144+
retry_with_backoff docker pull "${BUILDKIT_IMAGE}"
142145
143146
- name: Set up Docker Buildx
144147
id: buildx
145148
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
146149
with:
147150
driver-opts: |
148-
image=moby/buildkit:v0.31.0
151+
image=${{ env.BUILDKIT_IMAGE }}
149152
network=host
150153
151154
- id: build

.github/workflows/push.yml

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
name: Build and Push Docker Images
22
on:
33
push:
4-
branches: ["*"]
4+
branches: ["**"]
55
paths:
6+
- ".github/workflows/**"
7+
- "ci/**"
8+
- "cmd/**"
9+
- "go.mod"
10+
- "go.sum"
611
- "images/**"
12+
- "internal/**"
13+
- "docker-bake.hcl"
14+
- "Makefile"
715
workflow_dispatch:
816
inputs:
917
image:
@@ -35,7 +43,9 @@ jobs:
3543
- uses: "actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10" # v6
3644

3745
- name: lint
38-
run: shellcheck **/*.sh
46+
run: |
47+
mapfile -d '' scripts < <(git grep -Ilz -E '^#!.*(ba)?sh')
48+
shellcheck "${scripts[@]}"
3949
4050
plan:
4151
runs-on: ubuntu-24.04
@@ -73,6 +83,9 @@ jobs:
7383
with:
7484
go-version-file: go.mod
7585

86+
- name: Test planner and repository contracts
87+
run: go test -count=1 ./...
88+
7689
- id: plan
7790
name: Plan
7891
env:
@@ -144,21 +157,47 @@ jobs:
144157
)"
145158
}
146159
160+
coordinated_template_args() {
161+
local repository="$1"
162+
local branch="${GITHUB_REF_NAME}"
163+
164+
if [ "${branch}" = "main" ]; then
165+
return
166+
fi
167+
if [[ ! "${branch}" =~ ^[A-Za-z0-9._/-]+$ ]]; then
168+
echo "Skipping coordinated template branch with unsafe ref name: ${branch}" >&2
169+
return
170+
fi
171+
if git ls-remote --exit-code --heads "${repository}" "refs/heads/${branch}" >/dev/null 2>&1; then
172+
printf '%s' "--template-repo ${repository} --template-branch ${branch}"
173+
fi
174+
}
175+
147176
sitectl_wp_image_tags="$(planned_tag_assignment wp wp-php84)"
148177
sitectl_omeka_s_image_tags="$(planned_tag_assignment omeka-s omeka-s-php84)"
149178
sitectl_omeka_classic_image_tags="$(planned_tag_assignment omeka-classic omeka-classic-php84)"
150179
sitectl_ojs_image_tags="$(planned_tag_assignment ojs ojs-php84)"
151180
sitectl_drupal_image_tags="$(planned_tag_assignment drupal drupal-php84)"
152181
sitectl_isle_libops_image_tags="$(planned_tag_assignment drupal islandora-php84)"
153182
sitectl_isle_images=""
183+
sitectl_wp_create_args="$(coordinated_template_args https://github.com/libops/wp)"
184+
sitectl_omeka_s_create_args="$(coordinated_template_args https://github.com/libops/omeka-s)"
185+
sitectl_omeka_classic_create_args="$(coordinated_template_args https://github.com/libops/omeka-classic)"
186+
sitectl_ojs_create_args="$(coordinated_template_args https://github.com/libops/ojs)"
187+
sitectl_drupal_create_args="$(coordinated_template_args https://github.com/libops/drupal)"
188+
sitectl_isle_template_args="$(coordinated_template_args https://github.com/libops/isle)"
189+
if [ -z "${sitectl_isle_template_args}" ]; then
190+
sitectl_isle_template_args="--template-repo https://github.com/libops/isle"
191+
fi
192+
sitectl_isle_create_args="${sitectl_isle_template_args} --fcrepo off --blazegraph off --isle-file-system-uri private --iiif triplet --iiif-topology disabled --bot-mitigation off"
154193
155194
smoke_matrix='[]'
156-
add_smoke_row "wp" "sitectl sitectl-wp" "" "$sitectl_wp_image_tags" ""
157-
add_smoke_row "omeka-s" "sitectl sitectl-omeka-s" "" "$sitectl_omeka_s_image_tags" ""
158-
add_smoke_row "omeka-classic" "sitectl sitectl-omeka-classic" "" "$sitectl_omeka_classic_image_tags" ""
159-
add_smoke_row "ojs" "sitectl sitectl-ojs" "" "$sitectl_ojs_image_tags" ""
160-
add_smoke_row "drupal" "sitectl sitectl-drupal" "" "$sitectl_drupal_image_tags" ""
161-
add_smoke_row "isle" "sitectl sitectl-isle sitectl-drupal" "--template-repo https://github.com/libops/isle --fcrepo off --blazegraph off --isle-file-system-uri private --iiif triplet --iiif-topology disabled --bot-mitigation off" "$sitectl_isle_libops_image_tags" "$sitectl_isle_images"
195+
add_smoke_row "wp" "sitectl sitectl-wp" "$sitectl_wp_create_args" "$sitectl_wp_image_tags" ""
196+
add_smoke_row "omeka-s" "sitectl sitectl-omeka-s" "$sitectl_omeka_s_create_args" "$sitectl_omeka_s_image_tags" ""
197+
add_smoke_row "omeka-classic" "sitectl sitectl-omeka-classic" "$sitectl_omeka_classic_create_args" "$sitectl_omeka_classic_image_tags" ""
198+
add_smoke_row "ojs" "sitectl sitectl-ojs" "$sitectl_ojs_create_args" "$sitectl_ojs_image_tags" ""
199+
add_smoke_row "drupal" "sitectl sitectl-drupal" "$sitectl_drupal_create_args" "$sitectl_drupal_image_tags" ""
200+
add_smoke_row "isle" "sitectl sitectl-isle sitectl-drupal" "$sitectl_isle_create_args" "$sitectl_isle_libops_image_tags" "$sitectl_isle_images"
162201
163202
{
164203
echo "sitectl_wp_image_tags=${sitectl_wp_image_tags}"
@@ -375,13 +414,13 @@ jobs:
375414

376415
sitectl-create-smoke-test:
377416
if: ${{ always() && needs.plan.outputs.sitectl_create_smoke_matrix != '[]' && !cancelled() && !failure() }}
378-
needs: [plan, test-level-4]
417+
needs: [plan, test-level-0, test-level-1, test-level-2, test-level-3, test-level-4]
379418
strategy:
380419
fail-fast: false
381420
matrix:
382421
include: ${{ fromJson(needs.plan.outputs.sitectl_create_smoke_matrix) }}
383422

384-
uses: libops/.github/.github/workflows/sitectl-create-smoke-test.yaml@main
423+
uses: libops/.github/.github/workflows/sitectl-create-smoke-test.yaml@8f2eb98981e477cb8ac65d24c6fe06f45e31159c # main
385424
with:
386425
plugin: ${{ matrix.plugin }}
387426
create-definition: default

Makefile

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,12 @@ endef
3131
# Used to include host-platform specific docker compose files.
3232
OS := $(shell uname -s | tr A-Z a-z)
3333

34-
# Used to determine set TAGS when no explicit value provided,
35-
# as well as to fetch branch specific remote caches when building.
36-
BRANCH = $(shell git rev-parse --abbrev-ref HEAD)
34+
# Used to set TAGS when no explicit value is provided and to fetch branch-specific
35+
# remote caches. Keep it aligned with the metadata helper's Docker-tag fallback
36+
# normalization so names such as feature/foo produce valid cache references.
37+
# Pipe the ref directly into the normalizer so branch text is never re-evaluated
38+
# as part of a generated shell command.
39+
BRANCH = $(shell git -c safe.directory='$(CURDIR)' rev-parse --abbrev-ref HEAD | sed -E 's/[^A-Za-z0-9_.-]+/-/g; s/^-+//; s/-+$$//')
3740

3841
# The buildkit builder to use.
3942
BUILDER ?= default
@@ -121,7 +124,7 @@ docker-buildx: | docker
121124
.SILENT: build/bake.json
122125
build/bake.json: | docker-buildx jq build folder-permissions executable-permissons
123126
set -x; \
124-
BRANCH=$(BRANCH) \
127+
BRANCH="$(BRANCH)" \
125128
CACHE_FROM_REPOSITORY=$(CACHE_FROM_REPOSITORY) \
126129
CACHE_TO_REPOSITORY=$(CACHE_TO_REPOSITORY) \
127130
REPOSITORY=$(REPOSITORY) \

README.md

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,26 @@ Changes in this fork:
3232
- Dropped fcrepo file persistence
3333
- Added ArchivesSpace, OJS, Omeka, and Wordpress images.
3434
- Moved Islandora buildkit's `drupal` image to `islandora` and made a drupal-only image
35-
- All of these images are intended to be infra scaffolding and app-specific codebase dropped into the container in local builds
35+
- ArchivesSpace, OJS, Omeka S, and Omeka Classic include checksum-verified upstream application releases.
36+
- Drupal and Wordpress remain infrastructure scaffolding for Composer-managed application code in downstream builds.
3637
- Tags are based on software versions
3738
- Multiple versions of the same software/image can coexist (e.g. java, solr, fcrepo, php, etc.)
3839
- `base` environment variables (e.g. `DB_NAME`) are overriden instead of `IMAGE_NAME_*` env vars
3940
- Removed Islandora multisite support
4041
- Added optional Vault-backed secret bootstrap
42+
43+
## OJS dataset initialization
44+
45+
Fresh OJS installations defer the install-time ROR registry and IP geolocation
46+
downloads so that container initialization is bounded and does not require
47+
outbound network access. OJS's built-in scheduler remains enabled and updates
48+
the ROR registry monthly on the first day of the month and the IP database
49+
monthly on the tenth.
50+
51+
To populate either dataset immediately after installation, run the corresponding
52+
OJS scheduler task from `/var/www/ojs` as the application user:
53+
54+
```sh
55+
php lib/pkp/tools/scheduler.php test --name='PKP\task\UpdateRorRegistryDataset'
56+
php lib/pkp/tools/scheduler.php test --name='PKP\task\UpdateIPGeoDB'
57+
```

ci/tests/update-sha-readme.sh

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
6+
work_dir=$(mktemp -d)
7+
trap 'rm -rf "${work_dir}"' EXIT
8+
9+
# shellcheck disable=SC1090
10+
UPDATE_SHA_LIBRARY_ONLY=true source "${root_dir}/ci/update-sha.sh" test 1.2.3 2.0.0 ignored
11+
12+
readme="${work_dir}/README.md"
13+
printf '%s\n' 'Docker image version 1.2.3.' >"${readme}"
14+
update_readme "${readme}" 1.2.3 2.0.0
15+
grep -qx 'Docker image version 2.0.0.' "${readme}"
16+
17+
printf '%s\n' 'Docker image version 9.9.9.' >"${readme}"
18+
if update_readme "${readme}" 1.2.3 2.0.0; then
19+
echo "Missing README source version unexpectedly succeeded" >&2
20+
exit 1
21+
fi
22+
23+
printf '%s\n' 'First 1.2.3.' 'Second 1.2.3.' >"${readme}"
24+
if update_readme "${readme}" 1.2.3 2.0.0; then
25+
echo "Ambiguous README source version unexpectedly succeeded" >&2
26+
exit 1
27+
fi

ci/update-sha.sh

Lines changed: 114 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -19,37 +19,124 @@ update_dockerfile_sha() {
1919
local ARG="$2"
2020
local DOCKERFILES=("${@:3}")
2121
local SHA
22-
curl -fLs "$URL" -o curl.resp || echo "Request failed with exit code $?"
23-
SHA=$(shasum -a 256 curl.resp | awk '{print $1}')
22+
local DOWNLOAD
23+
local dockerfile
24+
DOWNLOAD=$(mktemp)
25+
26+
if ! curl \
27+
--fail \
28+
--location \
29+
--retry 3 \
30+
--retry-all-errors \
31+
--connect-timeout 20 \
32+
--max-time 600 \
33+
--silent \
34+
--show-error \
35+
--output "$DOWNLOAD" \
36+
"$URL"; then
37+
rm -f "$DOWNLOAD"
38+
echo "Failed to download release artifact: $URL" >&2
39+
return 1
40+
fi
2441

25-
if [[ "$OSTYPE" == "darwin"* ]]; then
26-
sed -i '' 's|^ARG '"$ARG"'=.*|ARG '"$ARG"'="'"$SHA"'"|g' "${DOCKERFILES[@]}"
27-
else
28-
sed -i 's|^ARG '"$ARG"'=.*|ARG '"$ARG"'="'"$SHA"'"|g' "${DOCKERFILES[@]}"
42+
SHA=$(shasum -a 256 "$DOWNLOAD" | awk '{print $1}')
43+
if [[ ! "$SHA" =~ ^[0-9a-f]{64}$ ]]; then
44+
rm -f "$DOWNLOAD"
45+
echo "Failed to calculate SHA256 for release artifact: $URL" >&2
46+
return 1
2947
fi
30-
rm curl.resp
48+
49+
for dockerfile in "${DOCKERFILES[@]}"; do
50+
if [[ $(grep -Ec "^[[:space:]]*(ARG[[:space:]]+)?${ARG}=\"?[[:xdigit:]]{64}\"?" "${dockerfile}") -ne 1 ]]; then
51+
rm -f "$DOWNLOAD"
52+
echo "Expected exactly one existing ${ARG} checksum in ${dockerfile}" >&2
53+
return 1
54+
fi
55+
56+
if [[ "$OSTYPE" == "darwin"* ]]; then
57+
sed -E -i '' "s|^([[:space:]]*(ARG[[:space:]]+)?${ARG}=)\"?[[:xdigit:]]{64}\"?(.*)$|\\1\"${SHA}\"\\3|" "${dockerfile}"
58+
else
59+
sed -E -i "s|^([[:space:]]*(ARG[[:space:]]+)?${ARG}=)\"?[[:xdigit:]]{64}\"?(.*)$|\\1\"${SHA}\"\\3|" "${dockerfile}"
60+
fi
61+
62+
if ! grep -Eq "^[[:space:]]*(ARG[[:space:]]+)?${ARG}=\"${SHA}\"" "${dockerfile}"; then
63+
rm -f "$DOWNLOAD"
64+
echo "Failed to update ${ARG} in ${dockerfile}" >&2
65+
return 1
66+
fi
67+
done
68+
rm -f "$DOWNLOAD"
3169
}
3270

3371
update_readme() {
3472
local README="$1"
3573
local OLD_VERSION="$2"
3674
local NEW_VERSION="$3"
37-
# update the README to specify the new version
38-
if [ "$README" != "" ]; then
39-
if [[ "$OSTYPE" == "darwin"* ]]; then
40-
sed -i '' "s/${OLD_VERSION}\.$/${NEW_VERSION}\./" "$README"
41-
else
42-
sed -i "s/${OLD_VERSION}\.$/${NEW_VERSION}\./" "$README"
43-
fi
75+
local match_count
76+
local new_match_count
77+
local replacement
78+
79+
if [ -z "$README" ]; then
80+
return 0
81+
fi
82+
if [ ! -f "$README" ]; then
83+
echo "Configured README does not exist: $README" >&2
84+
return 1
85+
fi
86+
87+
match_count=$(awk -v suffix="${OLD_VERSION}." '
88+
length($0) >= length(suffix) && substr($0, length($0) - length(suffix) + 1) == suffix { count++ }
89+
END { print count + 0 }
90+
' "$README")
91+
if [ "$match_count" -ne 1 ]; then
92+
echo "Expected exactly one README line ending in ${OLD_VERSION}. in ${README}; found ${match_count}" >&2
93+
return 1
94+
fi
95+
96+
replacement=$(mktemp)
97+
if ! awk -v old_suffix="${OLD_VERSION}." -v new_suffix="${NEW_VERSION}." '
98+
length($0) >= length(old_suffix) && substr($0, length($0) - length(old_suffix) + 1) == old_suffix {
99+
print substr($0, 1, length($0) - length(old_suffix)) new_suffix
100+
next
101+
}
102+
{ print }
103+
' "$README" >"$replacement"; then
104+
rm -f "$replacement"
105+
echo "Failed to prepare README update for ${README}" >&2
106+
return 1
107+
fi
108+
if cmp -s "$README" "$replacement"; then
109+
rm -f "$replacement"
110+
echo "README replacement did not change ${README}" >&2
111+
return 1
112+
fi
113+
command cat "$replacement" >"$README"
114+
rm -f "$replacement"
115+
116+
new_match_count=$(awk -v suffix="${NEW_VERSION}." '
117+
length($0) >= length(suffix) && substr($0, length($0) - length(suffix) + 1) == suffix { count++ }
118+
END { print count + 0 }
119+
' "$README")
120+
if [ "$new_match_count" -ne 1 ]; then
121+
echo "Failed to verify README version ${NEW_VERSION} in ${README}" >&2
122+
return 1
44123
fi
45124
}
46125

126+
if [ "${UPDATE_SHA_LIBRARY_ONLY:-false}" = "true" ]; then
127+
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
128+
exit 0
129+
fi
130+
return 0
131+
fi
132+
47133
echo "Updating SHA for $DEP@$NEW_VERSION"
48134

49135
if [ "$DEP" = "alpaca" ] ; then
50136
URL="https://github.com/islandora/alpaca/archive/refs/tags/${NEW_VERSION}.tar.gz"
51137
ARG=ALPACA_SHA256
52138
DOCKERFILES=("images/alpaca/Dockerfile")
139+
README="images/alpaca/README.md"
53140

54141
elif [ "$DEP" = "apache-tomcat9" ]; then
55142
URL="https://downloads.apache.org/tomcat/tomcat-9/v$NEW_VERSION/bin/apache-tomcat-$NEW_VERSION.tar.gz"
@@ -109,10 +196,19 @@ elif [ "$DEP" = "fcrepo7" ]; then
109196
DOCKERFILES=("images/fcrepo7/Dockerfile")
110197
README="images/fcrepo7/README.md"
111198

112-
elif [ "$DEP" = "islandora-syn" ]; then
199+
elif [ "$DEP" = "islandora-syn-fcrepo6" ]; then
113200
URL="https://github.com/Islandora/Syn/releases/download/v${NEW_VERSION}/islandora-syn-${NEW_VERSION}-all.jar"
114201
ARG="SYN_SHA256"
115-
DOCKERFILES=("images/fcrepo6/Dockerfile" "images/fcrepo7/Dockerfile")
202+
DOCKERFILES=("images/fcrepo6/Dockerfile")
203+
204+
elif [ "$DEP" = "islandora-syn-fcrepo7" ]; then
205+
URL="https://github.com/Islandora/Syn/releases/download/v${NEW_VERSION}/islandora-syn-${NEW_VERSION}-all.jar"
206+
ARG="SYN_SHA256"
207+
DOCKERFILES=("images/fcrepo7/Dockerfile")
208+
209+
elif [ "$DEP" = "islandora-syn" ]; then
210+
echo "Use islandora-syn-fcrepo6 or islandora-syn-fcrepo7 so Syn major lines remain independent" >&2
211+
exit 1
116212

117213
elif [ "$DEP" = "fcrepo-import-export" ]; then
118214
URL="https://github.com/fcrepo-exts/fcrepo-import-export/releases/download/fcrepo-import-export-${NEW_VERSION}/fcrepo-import-export-${NEW_VERSION}.jar"
@@ -214,8 +310,8 @@ elif [ "$DEP" = "s6-overlay" ]; then
214310

215311
exit 0
216312
else
217-
echo "DEP not found"
218-
exit 0
313+
echo "Unsupported dependency: $DEP" >&2
314+
exit 1
219315
fi
220316

221317
update_dockerfile_sha "$URL" "$ARG" "${DOCKERFILES[@]}"

0 commit comments

Comments
 (0)