From 3ce51caf524606feee09cec0a9e31510617c1283 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 13 Jun 2025 10:39:19 +0100 Subject: [PATCH 01/11] Split workflows to test from pip vs. conda install. --- .github/workflows/code_test_and_deploy.yml | 8 +---- .github/workflows/conda_install_check.yml | 37 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/conda_install_check.yml diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index 49054c6b1..c8b573ed3 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -45,14 +45,8 @@ jobs: auto-update-conda: true channels: conda-forge activate-environment: "datashuttle-test" - - name: Install datashuttle with conda + - name: Install datashuttle run: | - conda activate datashuttle-test - conda install -c conda-forge datashuttle - - name: Install local datashuttle for testing - run: | - # --force will only uninstall datashuttle not dependencies - conda uninstall datashuttle --force python -m pip install --upgrade pip pip install .[dev] - name: Test diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml new file mode 100644 index 000000000..7c0101692 --- /dev/null +++ b/.github/workflows/conda_install_check.yml @@ -0,0 +1,37 @@ +name: conda install check + +on: + schedule: + # Weekly cron job at 12:00 AM UTC on Mondays + - cron: '0 0 * * 1' + +jobs: + conda_install_check: + name: Conda install check (${{ matrix.os }} py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-13, macos-14] + python-version: ["3.9", "3.12"] + + defaults: + run: + shell: bash -l {0} # Required for conda activation + + steps: + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ matrix.python-version }} + auto-update-conda: true + channels: conda-forge + activate-environment: "datashuttle-test" + - name: Install datashuttle from conda + run: | + conda activate datashuttle-test + conda install -c conda-forge datashuttle + - name: Run datashuttle --version + run: | + conda activate datashuttle-test + datashuttle --version From 2df59e487f339d3c679d67820c9cd9309b361239 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 13 Jun 2025 10:50:13 +0100 Subject: [PATCH 02/11] Install rclone. --- .github/workflows/code_test_and_deploy.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index c8b573ed3..1955e7b77 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -45,8 +45,18 @@ jobs: auto-update-conda: true channels: conda-forge activate-environment: "datashuttle-test" - - name: Install datashuttle + + # Here we test the repo dependencies so that dependency updates + # in the pyproject.toml are installed in the environment. + # Conda installation is tested in a separate workflow and conda release pipeline. + - name: Install rclone + run: | + conda activate datashuttle-test + conda install -c conda-forge rclone + - name: Install datashuttle from repo run: | + # --force will only uninstall datashuttle not dependencies + conda uninstall datashuttle --force python -m pip install --upgrade pip pip install .[dev] - name: Test From f77ae97ed3049e6e2fec62e2107a1ffc500aaba8 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 13 Jun 2025 11:35:13 +0100 Subject: [PATCH 03/11] Fix install. --- .github/workflows/code_test_and_deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index 1955e7b77..65fac7bf7 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -56,7 +56,6 @@ jobs: - name: Install datashuttle from repo run: | # --force will only uninstall datashuttle not dependencies - conda uninstall datashuttle --force python -m pip install --upgrade pip pip install .[dev] - name: Test From 418e023f9c6a03c946af95188334e96e937cf34c Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 13 Jun 2025 12:44:19 +0100 Subject: [PATCH 04/11] Use --help not --version. --- .github/workflows/code_test_and_deploy.yml | 5 ++--- .github/workflows/conda_install_check.yml | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index 65fac7bf7..fb009da89 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -46,9 +46,8 @@ jobs: channels: conda-forge activate-environment: "datashuttle-test" - # Here we test the repo dependencies so that dependency updates - # in the pyproject.toml are installed in the environment. - # Conda installation is tested in a separate workflow and conda release pipeline. + # The recommended installation is via conda, but we need to test + # pip dependencies so any dependency changes in a PR are reflected - name: Install rclone run: | conda activate datashuttle-test diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml index 7c0101692..ceec684a5 100644 --- a/.github/workflows/conda_install_check.yml +++ b/.github/workflows/conda_install_check.yml @@ -1,8 +1,9 @@ +# As conda is the recommended installation method, run weekly checks +# to ensure name: conda install check on: schedule: - # Weekly cron job at 12:00 AM UTC on Mondays - cron: '0 0 * * 1' jobs: @@ -31,7 +32,7 @@ jobs: run: | conda activate datashuttle-test conda install -c conda-forge datashuttle - - name: Run datashuttle --version + - name: Run datashuttle --help run: | conda activate datashuttle-test datashuttle --version From 6807951c633d726a75aa7dfa73f0a5310fe78c3f Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 13 Jun 2025 12:48:17 +0100 Subject: [PATCH 05/11] Change conda running to test in PR. --- .github/workflows/conda_install_check.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml index ceec684a5..c019906b2 100644 --- a/.github/workflows/conda_install_check.yml +++ b/.github/workflows/conda_install_check.yml @@ -3,7 +3,15 @@ name: conda install check on: + push: + branches: + - main + tags: + - 'v*' + pull_request: schedule: + # Weekly cron job at 12:00 AM UTC on Mondays. + # This will only run on main by default. - cron: '0 0 * * 1' jobs: From 1caaa1de76612ee7a8b7a7df0d28b018107ad958 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 13 Jun 2025 12:51:00 +0100 Subject: [PATCH 06/11] Fix running --help. --- .github/workflows/conda_install_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml index c019906b2..ad8f88ec6 100644 --- a/.github/workflows/conda_install_check.yml +++ b/.github/workflows/conda_install_check.yml @@ -43,4 +43,4 @@ jobs: - name: Run datashuttle --help run: | conda activate datashuttle-test - datashuttle --version + datashuttle --help From c3d65a431148ad1fef873efd61866cf753834762 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Fri, 13 Jun 2025 12:52:04 +0100 Subject: [PATCH 07/11] Extend python versions for conda workflow. --- .github/workflows/conda_install_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml index ad8f88ec6..07c69c5e4 100644 --- a/.github/workflows/conda_install_check.yml +++ b/.github/workflows/conda_install_check.yml @@ -22,7 +22,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest, macos-13, macos-14] - python-version: ["3.9", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] defaults: run: From 166280ec6cd38893aad4c6547ed9e505a65d2a50 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 16 Jun 2025 13:19:42 +0100 Subject: [PATCH 08/11] Remove run conda install on PRs now that it has been checked. --- .github/workflows/conda_install_check.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml index 07c69c5e4..3f7b36fe0 100644 --- a/.github/workflows/conda_install_check.yml +++ b/.github/workflows/conda_install_check.yml @@ -3,15 +3,8 @@ name: conda install check on: - push: - branches: - - main - tags: - - 'v*' - pull_request: schedule: # Weekly cron job at 12:00 AM UTC on Mondays. - # This will only run on main by default. - cron: '0 0 * * 1' jobs: From dcb745d8a5fe2da7c6157ce25a8b0265d8538827 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 16 Jun 2025 13:21:36 +0100 Subject: [PATCH 09/11] Fix comment. --- .github/workflows/code_test_and_deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index fb009da89..ce6bd321a 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -47,7 +47,8 @@ jobs: activate-environment: "datashuttle-test" # The recommended installation is via conda, but we need to test - # pip dependencies so any dependency changes in a PR are reflected + # against dependencies from the pyproject.toml on the branch + # to ensure dependency changes in a PR are reflected. - name: Install rclone run: | conda activate datashuttle-test From 5676e11e8250c00811884e3a18658ec9f5a4ae3e Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 16 Jun 2025 16:25:47 +0100 Subject: [PATCH 10/11] Apply review feedback. --- .github/workflows/code_test_and_deploy.yml | 1 - .github/workflows/conda_install_check.yml | 12 +++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code_test_and_deploy.yml b/.github/workflows/code_test_and_deploy.yml index ce6bd321a..52975585f 100644 --- a/.github/workflows/code_test_and_deploy.yml +++ b/.github/workflows/code_test_and_deploy.yml @@ -55,7 +55,6 @@ jobs: conda install -c conda-forge rclone - name: Install datashuttle from repo run: | - # --force will only uninstall datashuttle not dependencies python -m pip install --upgrade pip pip install .[dev] - name: Test diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml index 3f7b36fe0..ca40f0cb4 100644 --- a/.github/workflows/conda_install_check.yml +++ b/.github/workflows/conda_install_check.yml @@ -3,6 +3,12 @@ name: conda install check on: + push: + branches: + - main + tags: + - 'v*' + pull_request: schedule: # Weekly cron job at 12:00 AM UTC on Mondays. - cron: '0 0 * * 1' @@ -29,11 +35,7 @@ jobs: auto-update-conda: true channels: conda-forge activate-environment: "datashuttle-test" - - name: Install datashuttle from conda + - name: Check conda installation run: | - conda activate datashuttle-test conda install -c conda-forge datashuttle - - name: Run datashuttle --help - run: | - conda activate datashuttle-test datashuttle --help From c17d93dc5fcdbc5d57736c3e38394eb1d6b2f546 Mon Sep 17 00:00:00 2001 From: JoeZiminski Date: Mon, 16 Jun 2025 16:36:12 +0100 Subject: [PATCH 11/11] Remove run on PR schedule (for testing) in conda workflow. --- .github/workflows/conda_install_check.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/conda_install_check.yml b/.github/workflows/conda_install_check.yml index ca40f0cb4..5d411615e 100644 --- a/.github/workflows/conda_install_check.yml +++ b/.github/workflows/conda_install_check.yml @@ -3,12 +3,6 @@ name: conda install check on: - push: - branches: - - main - tags: - - 'v*' - pull_request: schedule: # Weekly cron job at 12:00 AM UTC on Mondays. - cron: '0 0 * * 1'