From 8ba59955a9bcad42b399528da48d8e6a6cff1892 Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Fri, 29 May 2026 12:31:43 -0700 Subject: [PATCH 1/5] ci: gate staging build/deploy on extension test suite Re-adds the CKAN extension tests that died with the Jenkins pipeline, now as a 'test' job in build-deploy.yml that build/deploy depend on. Brings up the docker-compose dev stack (the same flow the old ci_setup.sh/ci_test.sh drove via adx) on pinned submodule commits and runs the 5 Fjelltopp suites: unaids, validation, scheming, dhis2harvester, emailasusername Gate semantics: - push: tests must pass before build and deploy run - redeploy (workflow_dispatch + image_tag): tests skipped, image already tested when built - test failure blocks both build and deploy Tests run against the dev image + bind-mounted submodules, matching local 'adx test'. Not yet verified green under CKAN 2.11/Py3.10. --- .github/workflows/build-deploy.yml | 75 ++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 4c8ae61..928be94 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -21,9 +21,72 @@ env: URL: https://dev.adr.fjelltopp.org jobs: - build: + test: + name: Extension tests + # Skip on a pure redeploy (workflow_dispatch with an existing image_tag); + # that image was already tested when it was built. if: github.event_name != 'workflow_dispatch' || inputs.image_tag == '' runs-on: ubuntu-latest + timeout-minutes: 45 + env: + # Fresh runner DB — no need to restart the db container during testsetup. + SKIP_DB_RESTART: "True" + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + submodules: recursive + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Prepare .env + run: cp dev.env .env + + - name: Build images and start the dev stack + run: | + ./adx build + ./adx up + + - name: Wait for CKAN bootstrap + run: | + for i in $(seq 1 90); do + if docker logs ckan 2>&1 | grep -q 'CKAN bootstrapping finished, environment ready'; then + echo "CKAN ready after ${i} checks" + exit 0 + fi + echo "Waiting for CKAN bootstrap (${i}/90)…" + sleep 10 + done + echo "::error::CKAN did not finish bootstrapping in time" + docker logs ckan + exit 1 + + - name: Create test databases + run: ./adx testsetup + + - name: Run extension tests + run: | + set -e + for ext in unaids validation scheming dhis2harvester emailasusername; do + echo "::group::ckanext-${ext}" + ./adx test "${ext}" --no-interaction + echo "::endgroup::" + done + + - name: Dump CKAN logs on failure + if: failure() + run: docker logs ckan || true + + build: + needs: test + # Run when tests pass (success) or were skipped (redeploy), and we're + # actually building (not a redeploy of an existing tag). + if: >- + always() + && needs.test.result != 'failure' + && needs.test.result != 'cancelled' + && (github.event_name != 'workflow_dispatch' || inputs.image_tag == '') + runs-on: ubuntu-latest outputs: image_tag: ${{ steps.meta.outputs.version }} steps: @@ -61,8 +124,14 @@ jobs: labels: ${{ steps.meta.outputs.labels }} deploy: - needs: build - if: always() && (needs.build.result == 'success' || needs.build.result == 'skipped') + needs: [test, build] + # Never deploy if tests failed/cancelled. Otherwise deploy when the image + # was built (success) or already exists (build skipped on redeploy). + if: >- + always() + && needs.test.result != 'failure' + && needs.test.result != 'cancelled' + && (needs.build.result == 'success' || needs.build.result == 'skipped') runs-on: ubuntu-latest environment: name: staging From 7805c499cc300675fc1f1797e2aee892101a9e33 Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Fri, 29 May 2026 12:39:34 -0700 Subject: [PATCH 2/5] ci: run tests on PRs + cache pipenv venv and React node_modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add pull_request trigger so the extension-test job runs as a visible status check; build/deploy stay gated off pull_request events. - Cache .adxvenv on Pipfile.lock — the dominant cost is bootstrap's 'pipenv sync --dev' (CKAN + ~19 extensions), not the image build. - Cache the unaids React node_modules on its yarn.lock. Both caches are safe on miss, so the first run is a clean cold signal. --- .github/workflows/build-deploy.yml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 928be94..19244fd 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -3,6 +3,10 @@ name: Build and Deploy CKAN (staging) on: push: branches: [master, ckan211-prod-deploy-pr] + pull_request: + # Runs the test job only (build/deploy are gated off pull_request below), + # so PRs get a visible test status check without deploying. + branches: [master, ckan211-prod-deploy-pr] workflow_dispatch: inputs: image_tag: @@ -36,8 +40,27 @@ jobs: with: submodules: recursive - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + # The dominant cost is `pipenv sync --dev` at bootstrap (CKAN + ~19 + # extensions), which lands in the bind-mounted .adxvenv. Caching it on + # Pipfile.lock makes warm runs a near no-op. .adxvenv is gitignored and + # written by root in-container, so there are no ownership issues here. + - name: Cache Python venv + uses: actions/cache@v4 + with: + path: .adxvenv + key: adxvenv-${{ runner.os }}-${{ hashFiles('Pipfile.lock') }} + restore-keys: | + adxvenv-${{ runner.os }}- + + # Second cost: the entrypoint runs `yarn install` + build for the unaids + # React app on every boot. Cache its node_modules on the React yarn.lock. + - name: Cache unaids React node_modules + uses: actions/cache@v4 + with: + path: submodules/ckanext-unaids/ckanext/unaids/react/node_modules + key: react-nm-${{ runner.os }}-${{ hashFiles('submodules/ckanext-unaids/ckanext/unaids/react/yarn.lock') }} + restore-keys: | + react-nm-${{ runner.os }}- - name: Prepare .env run: cp dev.env .env @@ -83,6 +106,7 @@ jobs: # actually building (not a redeploy of an existing tag). if: >- always() + && github.event_name != 'pull_request' && needs.test.result != 'failure' && needs.test.result != 'cancelled' && (github.event_name != 'workflow_dispatch' || inputs.image_tag == '') @@ -129,6 +153,7 @@ jobs: # was built (success) or already exists (build skipped on redeploy). if: >- always() + && github.event_name != 'pull_request' && needs.test.result != 'failure' && needs.test.result != 'cancelled' && (needs.build.result == 'success' || needs.build.result == 'skipped') From e25c726d96c289778a446b649b44b40bb872e2be Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Fri, 29 May 2026 12:50:00 -0700 Subject: [PATCH 3/5] ci: run all suites (no fail-fast) and save caches on failure - Run all 5 extension suites and aggregate, so one run reports the full picture instead of stopping at the first failing suite. - Split cache restore/save so .adxvenv and node_modules are saved even when tests fail, making triage runs warm instead of cold. --- .github/workflows/build-deploy.yml | 44 ++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index 19244fd..17ee9b8 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -44,8 +44,12 @@ jobs: # extensions), which lands in the bind-mounted .adxvenv. Caching it on # Pipfile.lock makes warm runs a near no-op. .adxvenv is gitignored and # written by root in-container, so there are no ownership issues here. - - name: Cache Python venv - uses: actions/cache@v4 + # Restore + save are split (rather than the combined cache action) so the + # venv is saved even when the test step fails — otherwise the expensive + # `pipenv sync` is repeated cold on every triage run. Saved at job end. + - name: Restore Python venv + id: venv-cache + uses: actions/cache/restore@v4 with: path: .adxvenv key: adxvenv-${{ runner.os }}-${{ hashFiles('Pipfile.lock') }} @@ -54,8 +58,9 @@ jobs: # Second cost: the entrypoint runs `yarn install` + build for the unaids # React app on every boot. Cache its node_modules on the React yarn.lock. - - name: Cache unaids React node_modules - uses: actions/cache@v4 + - name: Restore unaids React node_modules + id: react-cache + uses: actions/cache/restore@v4 with: path: submodules/ckanext-unaids/ckanext/unaids/react/node_modules key: react-nm-${{ runner.os }}-${{ hashFiles('submodules/ckanext-unaids/ckanext/unaids/react/yarn.lock') }} @@ -89,17 +94,44 @@ jobs: - name: Run extension tests run: | - set -e + # Run every suite (don't fail-fast) so one run shows the full picture, + # then fail at the end if any suite failed. + failed=() for ext in unaids validation scheming dhis2harvester emailasusername; do echo "::group::ckanext-${ext}" - ./adx test "${ext}" --no-interaction + if ./adx test "${ext}" --no-interaction; then + echo "ckanext-${ext}: PASS" + else + echo "ckanext-${ext}: FAIL" + failed+=("${ext}") + fi echo "::endgroup::" done + echo "--- summary ---" + if [ ${#failed[@]} -ne 0 ]; then + echo "::error::Failing suites: ${failed[*]}" + exit 1 + fi + echo "All extension suites passed" - name: Dump CKAN logs on failure if: failure() run: docker logs ckan || true + - name: Save Python venv + if: always() && steps.venv-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .adxvenv + key: ${{ steps.venv-cache.outputs.cache-primary-key }} + + - name: Save unaids React node_modules + if: always() && steps.react-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: submodules/ckanext-unaids/ckanext/unaids/react/node_modules + key: ${{ steps.react-cache.outputs.cache-primary-key }} + build: needs: test # Run when tests pass (success) or were skipped (redeploy), and we're From bfe10552d970178b9749600274d98df144850aa6 Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Fri, 29 May 2026 13:10:22 -0700 Subject: [PATCH 4/5] test: fix extension test suite under CKAN 2.11/Py3.10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add 'mock' and 'pyfakefs' to dev-packages: ckanext-validation, -scheming and -emailasusername import the standalone 'mock' package (and pyfakefs in validation), which weren't installed, causing pytest collection errors. Dev-only — prod uses 'pipenv sync' without --dev, so these don't ship. - run_tests: blank CKAN_SMTP_SERVER for test runs so suites never reach the dev stack's smtp4dev. Fixes ckanext-unaids' test_send_dataset_transfer_emails_errors, which asserts mail sending fails when no server is configured. --- Pipfile | 2 ++ Pipfile.lock | 68 ++++++++++++++++++++++++++++++------------------ util/__init__.py | 5 ++++ 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/Pipfile b/Pipfile index 2dbfe82..dca3e1f 100644 --- a/Pipfile +++ b/Pipfile @@ -120,6 +120,8 @@ pytest-rerunfailures = "==15.0" pytest-split = "==0.10.0" pytest-retry = "==1.7.0" pytest-mock = "*" +mock = "*" +pyfakefs = "*" coverage = "==7.7.1" junitparser = "==3.2.0" junit2html = "==31.0.2" diff --git a/Pipfile.lock b/Pipfile.lock index d60b022..aad1579 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a3d944c8b8a4464935d9dc09ee7fe579104fa5f1de1d7373feed712f0a51ece7" + "sha256": "9c65150bc602efed14e2cef6b078391e5656865a1d20a10f3a37025609c9b29a" }, "pipfile-spec": 6, "requires": { @@ -126,19 +126,19 @@ }, "boto3": { "hashes": [ - "sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4", - "sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d" + "sha256:33138883e984eb1937d1553da699182c8ad2099138091e885b65c9accbccea16", + "sha256:7b62ce5c0a51428d692aa4f2adc9dc2a4a4c2989bf65a0a12834eeffa99b0b84" ], "markers": "python_version >= '3.10'", - "version": "==1.43.14" + "version": "==1.43.18" }, "botocore": { "hashes": [ - "sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658", - "sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516" + "sha256:dc8c105351b49688c667065cd5a45fc5b9db982657cefc9e3fbfb9417a55c7df", + "sha256:e2610fce16df9f89deab5f3c163430a814e6804034eb95bef8957c8db60b7dbc" ], "markers": "python_version >= '3.10'", - "version": "==1.43.14" + "version": "==1.43.18" }, "cached-property": { "hashes": [ @@ -696,11 +696,11 @@ }, "idna": { "hashes": [ - "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", - "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d" + "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", + "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f" ], "markers": "python_version >= '3.9'", - "version": "==3.16" + "version": "==3.17" }, "ijson": { "hashes": [ @@ -1130,11 +1130,11 @@ }, "marko": { "hashes": [ - "sha256:6940308e655f63733ca518c47a68ec9510279dbb916c83616e4c4b5829f052e8", - "sha256:f064ae8c10416285ad1d96048dc11e98ef04e662d3342ae416f662b70aa7959e" + "sha256:8e1d7a0387281e59dfbc52a381b58c570156970e36b2bbe047f8a3a2f368cacc", + "sha256:e31ec2875383bc62f9093d16babed5a2c2cde601c00d834ea935a2222120ec19" ], "markers": "python_version >= '3.9'", - "version": "==2.2.2" + "version": "==2.2.3" }, "markupsafe": { "hashes": [ @@ -1846,11 +1846,11 @@ }, "s3transfer": { "hashes": [ - "sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a", - "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20" + "sha256:239c13b09e65ad0346e1be7348b8a202dcad44ac7ea7c6eb858fc881dce739b6", + "sha256:3760b8b7ec1315da54048b2d626276732bee4300d054d492d4e1d43e20d4ecbd" ], "markers": "python_version >= '3.10'", - "version": "==0.17.0" + "version": "==0.18.0" }, "setuptools": { "hashes": [ @@ -2157,11 +2157,11 @@ "all" ], "hashes": [ - "sha256:537d27ae686d82967f6383382a952cb32ba4768898541effccb69ca75bbd5d23", - "sha256:933e4f0083521f3c57d6a5aedf3b073271b2f95a19761b171b494dd6fdb21ff6" + "sha256:3e2b9352f535e5303ef27806dadc2c8647687bdca5c902f03fec3fb88f46a46a", + "sha256:e70549ec5a403ca8a0bf0802ddd9f3c6ff7a14ccbb859b01b697baa943636f33" ], "markers": "python_version >= '3.10'", - "version": "==0.26.1" + "version": "==0.26.3" }, "typing-extensions": { "hashes": [ @@ -2718,11 +2718,11 @@ }, "idna": { "hashes": [ - "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", - "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d" + "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c", + "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f" ], "markers": "python_version >= '3.9'", - "version": "==3.16" + "version": "==3.17" }, "imagesize": { "hashes": [ @@ -2904,6 +2904,15 @@ "markers": "python_version >= '3.7'", "version": "==0.1.2" }, + "mock": { + "hashes": [ + "sha256:4e460e818629b4b173f32d08bf30d3af8123afbb8e04bb5707a1fd4799e503f0", + "sha256:7ba87f72ca0e915175596069dbbcc7c75af7b5e9b9bc107ad6349ede0819982f" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==5.2.0" + }, "packaging": { "hashes": [ "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002", @@ -3063,6 +3072,15 @@ ], "version": "==0.2.3" }, + "pyfakefs": { + "hashes": [ + "sha256:0968a49db692694ffed420e54a9f1cbae4636637b880e8ab09c8ccc0f11bd7ae", + "sha256:e59a36db447bf509ce9c97ab3d1510c08cc51895c5311325a560a5e5b5dc1940" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==6.2.0" + }, "pygments": { "hashes": [ "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", @@ -3466,11 +3484,11 @@ "all" ], "hashes": [ - "sha256:537d27ae686d82967f6383382a952cb32ba4768898541effccb69ca75bbd5d23", - "sha256:933e4f0083521f3c57d6a5aedf3b073271b2f95a19761b171b494dd6fdb21ff6" + "sha256:3e2b9352f535e5303ef27806dadc2c8647687bdca5c902f03fec3fb88f46a46a", + "sha256:e70549ec5a403ca8a0bf0802ddd9f3c6ff7a14ccbb859b01b697baa943636f33" ], "markers": "python_version >= '3.10'", - "version": "==0.26.1" + "version": "==0.26.3" }, "typing-extensions": { "hashes": [ diff --git a/util/__init__.py b/util/__init__.py index 5ed385f..80a6409 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -162,7 +162,12 @@ def run_tests(args, extra): # for cases like ckanext-ytp-requests repository which uses ytp_requests test directory internally extension_sub_path = extension_sub_path.replace("-", "_") retcode = call_command([ + # Blank out the SMTP server for test runs so tests never reach the + # dev stack's smtp4dev and send real mail. Some suites (e.g. + # ckanext-unaids' send_dataset_transfer_emails error path) assert that + # sending fails when no server is configured. f'docker exec {args.interaction} -e CKAN_SQLALCHEMY_URL={CKAN_TEST_SQLALCHEMY_URL} ' + f'-e CKAN_SMTP_SERVER= ' f'ckan /usr/local/bin/ckan-pytest --capture=no --disable-warnings ' f'--ckan-ini={extension_path}/test.ini ' f'{extension_path}/{extension_sub_path}/tests ' From 5f6c227ba3fc3f3cf8c74adc9c4292059b6b2450 Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Fri, 29 May 2026 13:31:05 -0700 Subject: [PATCH 5/5] deps: pin frictionless==5.13.1 and pyfakefs==4.6.* to match extensions The validation/unaids extensions pin frictionless[ckan]==5.13.1 and pyfakefs==4.6.* in their own requirements and pass their own CI against those. Our merged Pipfile had frictionless>=5.0.0,<6.0.0 (drifted to a newer 5.x that dropped Resource.__create__) and pyfakefs=* (6.x dropped the CreateFile API), so ckanext-validation's test suite failed in our stack only. Aligning the pins fixes it without touching the submodule; frictionless is prod-facing but this matches the version the extensions are built and tested against. --- Pipfile | 9 +++++++-- Pipfile.lock | 17 ++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Pipfile b/Pipfile index dca3e1f..f6ca46a 100644 --- a/Pipfile +++ b/Pipfile @@ -92,7 +92,10 @@ python-jose = "==3.3.0" setuptools = ">=65.0.0,<80.0.0" raven = "==6.10.0" tableschema = "==1.20.2" -frictionless = ">=5.0.0,<6.0.0" +# Pinned to match ckanext-validation/-unaids requirements.txt (the version +# those extensions are built and tested against). Newer 5.x dropped +# frictionless.Resource.__create__, which their test suites still patch. +frictionless = "==5.13.1" markupsafe = "==2.1.5" [dev-packages] @@ -121,7 +124,9 @@ pytest-split = "==0.10.0" pytest-retry = "==1.7.0" pytest-mock = "*" mock = "*" -pyfakefs = "*" +# Pinned to match ckanext-validation dev-requirements.txt; pyfakefs 5.0+ +# removed the CamelCase API (CreateFile) its TestFiles suite uses. +pyfakefs = "==4.6.*" coverage = "==7.7.1" junitparser = "==3.2.0" junit2html = "==31.0.2" diff --git a/Pipfile.lock b/Pipfile.lock index aad1579..63f0547 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9c65150bc602efed14e2cef6b078391e5656865a1d20a10f3a37025609c9b29a" + "sha256": "e0480c0ef39380e1673551bf3abbffc6ac41189380ea5c91507976d9e05324fb" }, "pipfile-spec": 6, "requires": { @@ -586,12 +586,11 @@ }, "frictionless": { "hashes": [ - "sha256:6d22dc39126fb65cf7853b6db8d409637b056c77bc709abb344bbd50ca1bdddf", - "sha256:ab4bfe55f5d831c15a16760685036fdd85c9026d2d7c633b6672b1b04f8c48c6" + "sha256:78ee7c4ea784e85ef7117f17165baf4f2728790ba9406693df8370e812726c4a", + "sha256:8e5afc7c08364d4563797944920e044a8d37970c767c0c6920ac804a05c5f1a9" ], "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==5.16.1" + "version": "==5.13.1" }, "giftless-client": { "hashes": [ @@ -3074,12 +3073,12 @@ }, "pyfakefs": { "hashes": [ - "sha256:0968a49db692694ffed420e54a9f1cbae4636637b880e8ab09c8ccc0f11bd7ae", - "sha256:e59a36db447bf509ce9c97ab3d1510c08cc51895c5311325a560a5e5b5dc1940" + "sha256:6df12a7cf657637a1b036bc20059727c642f92990e90fee2fb003daa3cda6ca1", + "sha256:8959fe7058ba7efa65694b7979e123e27921a58f557a88628be93f0a936e6757" ], "index": "pypi", - "markers": "python_version >= '3.10'", - "version": "==6.2.0" + "markers": "python_version >= '3.6'", + "version": "==4.6.3" }, "pygments": { "hashes": [