Add workflow to run gdrive and aws tests on forks with a label.

Author: JoeZiminski

.github/workflows/code_test_and_deploy.yml

@@ -7,7 +7,7 @@ on:
     tags:
       - 'v*'
   pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+
   schedule:
     # Cron runs on the 1st and 15th of every month.
     # This will only run on main by default.
@@ -20,6 +20,7 @@ jobs:
       - uses: neuroinformatics-unit/actions/lint@v2
 
   test:
+    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
     needs: [linting]
     name: ${{ matrix.os }} py${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
@@ -74,22 +75,29 @@ jobs:
           sudo service mysql stop  # free up port 3306 for ssh tests
           pytest tests/tests_transfers/ssh
 
-      - name: Test Google Drive
+      - name: Test Google Drive and AWS
+        # AWS and GDrive tests require secrets, which are empty string by default on forked repositories.
+        # The below flag ensures that these tests do not run on forked repositories. To run, the
+        # ok-to-run-gdrive-aws label must be manually added, after checking no malicious code
+        # that extracts the secrets is found on the PR.
+        if: >
+          github.event_name != 'pull_request' ||
+          github.event.pull_request.head.repo.full_name == github.repository ||
+          contains(
+            join(fromJSON('["' + join(github.event.pull_request.labels.*.name, '","') + '"]')),
+            'ok-to-run-gdrive-aws'
+          )
         env:
           GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }}
           GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }}
           GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }}
           GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }}
-        run: |
-          pytest tests/tests_transfers/gdrive
-
-      - name: Test AWS
-        env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: ${{ secrets.AWS_REGION }}
           AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
         run: |
+          pytest tests/tests_transfers/gdrive
           pytest tests/tests_transfers/aws
 
       - name: All Other Tests

.github/workflows/run_secure_tests_on_forked_pr.yml

@@ -0,0 +1,54 @@
+# Secure Tests Workflow
+#
+# Runs the Google Drive and AWS transfer tests that need secrets.
+# GitHub doesn’t pass secrets to pull requests from forks, so this
+# workflow exists to let maintainers explicitly approve those runs.
+
+name: Secure tests (label gated)
+
+on:
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  run-secure-tests:
+    if: github.event.label.name == 'ok-to-run-gdrive-aws'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout PR merge ref
+        uses: actions/checkout@v4
+        with:
+          ref: refs/pull/${{ github.event.pull_request.number }}/merge
+
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: 3.12
+          channels: conda-forge
+          activate-environment: datashuttle-test
+
+      - name: Install deps
+        run: |
+          conda activate datashuttle-test
+          conda install -c conda-forge -y rclone
+          python -m pip install --upgrade pip
+          pip install .[dev]
+
+      - name: Google Drive tests
+        env:
+          GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }}
+          GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }}
+          GDRIVE_ROOT_FOLDER_ID: ${{ secrets.GDRIVE_ROOT_FOLDER_ID }}
+          GDRIVE_CONFIG_TOKEN: ${{ secrets.GDRIVE_CONFIG_TOKEN }}
+        run: pytest tests/tests_transfers/gdrive
+
+      - name: AWS tests
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME }}
+        run: pytest tests/tests_transfers/aws

tests/tests_transfers/aws/aws_test_utils.py

@@ -53,4 +53,8 @@ def has_aws_environment_variables():
     ]:
         if key not in os.environ:
             return False
+
+        if os.environ[key].strip() == "":
+            return False
+
     return True

tests/tests_transfers/gdrive/gdrive_test_utils.py

@@ -67,4 +67,8 @@ def has_gdrive_environment_variables():
     ]:
         if key not in os.environ:
             return False
+
+        if os.environ[key].strip() == "":
+            return False
+
     return True