Add GHA to run warehouses tests

Author: martyngigg

.github/actions/run-pytest-with-uv/action.yml

@@ -5,9 +5,9 @@ inputs:
   compose-file-path:
     description: If supplied, brings up the services defined in the given docker compose file.
     default: ""
-  pyproject-directory:
-    description: Path to the directory containing pyproject.toml
-    required: true
+  install-args-extra:
+    description: Extra arguments to the uv sync command
+    default: ""
   pytest-file-or-dir:
     description: Path to a file or directory for pytest execution
     required: true
@@ -19,6 +19,9 @@ inputs:
   uv-version:
     description: The version of uv to install e.g., `0.5.0` Defaults to the version in pyproject.toml or 'latest'.
     default: "latest"
+  working-directory:
+    description: Working directory while running pytest
+    required: true
 
 runs:
   using: composite
@@ -41,13 +44,13 @@ runs:
 
     - name: Install the project
       shell: bash -l {0}
-      run: uv sync --locked --all-extras --dev
-      working-directory: ${{ inputs.pyproject-directory }}
+      run: uv sync --locked --all-extras --dev ${{ inputs.install-args-extra }}
+      working-directory: ${{ inputs.working-directory }}
 
     - name: Run tests
       shell: bash -l {0}
       run: uv run pytest --durations-min=0.5 --exitfirst "${{ inputs.pytest-file-or-dir }}" --cache-clear
-      working-directory: ${{ inputs.pyproject-directory }}
+      working-directory: ${{ inputs.working-directory }}
 
     - name: Dump Docker Compose logs on failure
       if: failure() && inputs.compose-file-path != ''

.github/workflows/elt-common_e2e_tests.yml

@@ -41,9 +41,9 @@ jobs:
         uses: ./.github/actions/run-pytest-with-uv
         with:
           compose-file-path: infra/local/docker-compose.yml
-          pyproject-directory: elt-common
           pytest-file-or-dir: tests/e2e_tests
           uv-cache-dependency-glob: elt-common/pyproject.toml
+          working-directory: elt-common
 
       - name: Remove adp-router from /etc/hosts
         shell: bash -l {0}

.github/workflows/elt-common_unit_tests.yml

@@ -28,6 +28,6 @@ jobs:
       - name: Run unit tests
         uses: ./.github/actions/run-pytest-with-uv
         with:
-          pyproject-directory: elt-common
           pytest-file-or-dir: tests/unit_tests
           uv-cache-dependency-glob: elt-common/pyproject.toml
+          working-directory: elt-common

.github/workflows/warehouses_e2e_tests.yml

@@ -0,0 +1,47 @@
+name: End-to-end tests for extract and load scripts in the warehouse
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - ".github/workflows/warehouses_e2e_tests.yml"
+      - "elt-common/**"
+      - "warehouses/accelerator/extract_load/opralogweb/**"
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - ".github/workflows/warehouses_e2e_tests.yml"
+      - "elt-common/**"
+      - "warehouses/accelerator/extract_load/opralogweb/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: warehouses end-to-end tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Add adp-router to /etc/hosts
+        shell: bash -l {0}
+        run: |
+          echo "127.0.0.1    adp-router" | sudo tee -a /etc/hosts
+
+      - name: Run end-to-end tests
+        uses: ./.github/actions/run-pytest-with-uv
+        with:
+          compose-file-path: infra/local/docker-compose.yml
+          install-args-extra: --script extract_and_load.py.lock
+          pytest-file-or-dir: tests
+          uv-cache-dependency-glob: elt-common/pyproject.toml
+          working-directory: warehouses/accelerator/extract_load/opralogweb
+
+      - name: Remove adp-router from /etc/hosts
+        shell: bash -l {0}
+        run: |
+          sudo sed -i -e '/adp-router/d' /etc/hosts