feat(infra/local): Enable AUTHZ_BACKEND in local infrastructure configuration (#174)

## Summary by CodeRabbit

## Release Notes

* **New Features**
  * Added fine-grained access control infrastructure to the platform, excluding Superset
  * Enabled OAuth-based single sign-on authentication for Superset.
  * Automated local certificate generation for development environments. Only used for Trino access.

* **Documentation**
* Updated local development setup instructions with required /etc/hosts
configuration.
  * Expanded service endpoints and access documentation.
  * Clarified HTTPS requirement for Trino connections.
  * Reorganised testing guidance to emphasise unit tests.

* **Infrastructure**
  * Configured file-based access control for Trino.
  * Enhanced Traefik routing with TLS support.
  * Streamlined authentication realm configuration.

Author: martyngigg

.github/actions/run-pytest-with-uv/action.yml

@@ -24,8 +24,9 @@ runs:
   using: composite
   steps:
     - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
       with:
+        activate-environment: false
         cache-dependency-glob: ${{ inputs.uv-cache-dependency-glob }}
         version: ${{ inputs.uv-version }}
         python-version: ${{ inputs.python-version }}
@@ -43,23 +44,11 @@ runs:
       run: uv sync --locked --all-extras --dev
       working-directory: ${{ inputs.pyproject-directory }}
 
-    - name: Add minio to /etc/hosts
-      if: inputs.compose-file-path != ''
-      shell: bash -l {0}
-      run: |
-        echo "127.0.0.1 minio" | sudo tee -a /etc/hosts
-
     - name: Run tests
       shell: bash -l {0}
       run: uv run pytest --durations-min=0.5 --exitfirst "${{ inputs.pytest-file-or-dir }}" --cache-clear
       working-directory: ${{ inputs.pyproject-directory }}
 
-    - name: Remove minio from /etc/hosts
-      if: inputs.compose-file-path != ''
-      shell: bash -l {0}
-      run: |
-        sudo sed -i -e '/minio/d' /etc/hosts
-
     - name: Dump Docker Compose logs on failure
       if: failure() && inputs.compose-file-path != ''
       shell: bash -l {0}

.github/workflows/ci-static.yml

@@ -12,8 +12,8 @@ jobs:
     name: static checks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
       - name: "Install prek"
         run: >
           python -m pip install prek

.github/workflows/docker-superset.yml

@@ -33,7 +33,7 @@ jobs:
       id-token: write
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
       - name: Compute fully qualified image name
         run: echo "FQ_IMAGE_NAME=${{ env.REGISTRY }}/${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }}" >> $GITHUB_ENV
       - name: Log in to the Container registry

.github/workflows/elt-common_e2e_tests.yml

@@ -20,14 +20,22 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
   cancel-in-progress: true
 
+env:
+  TRINO_CATALOG_STORE: memory
+
 jobs:
   test:
     name: elt-common end-to-end tests
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+
+      - name: Add adp-router to /etc/hosts
+        shell: bash -l {0}
+        run: |
+          echo "127.0.0.1    adp-router" | sudo tee -a /etc/hosts
 
       - name: Run end-to-end tests
         uses: ./.github/actions/run-pytest-with-uv
@@ -36,3 +44,8 @@ jobs:
           pyproject-directory: elt-common
           pytest-file-or-dir: tests/e2e_tests
           uv-cache-dependency-glob: elt-common/pyproject.toml
+
+      - name: Remove adp-router from /etc/hosts
+        shell: bash -l {0}
+        run: |
+          sudo sed -i -e '/adp-router/d' /etc/hosts

.github/workflows/elt-common_unit_tests.yml

@@ -23,7 +23,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Run unit tests
         uses: ./.github/actions/run-pytest-with-uv

AGENTS.md

@@ -57,6 +57,7 @@ uv run pytest unit_tests
 
 ```bash
 pushd infra/local; docker compose up -d; popd    # start the services
+echo "127.0.0.1    adp-router" | sudo tee -a /etc/hosts   # edit /etc/hosts
 cd elt-common/
 uv run pytest e2e_tests
 pushd infra/local; docker compose down -v; popd  # stop the services

elt-common/README.md

@@ -13,55 +13,33 @@ Development requires the following tools:
 ### Setting up a Python virtual environment
 
 Once `uv` is installed, create an environment and install the `elt-common`
-package in editable mode using the following command:
+package in editable mode, along with the development dependencies:
 
 ```bash
+> uv venv
+> source .venv/bin/activate
 > uv pip install --editable . --group dev
 ```
 
-## Running end-to-end tests
-
-The end-to-end (e2e) tests for the `pyiceberg` destination require a running Iceberg
-rest catalog to test complete functionality.
-A (`docker-compose`)[./tests/docker-compose.yml] file is provided to both run the
-required services and provide a `python-uv` service for executing test commands.
-Please ensure you have docker and docker compose available on your command line
-before continuing.
-
-To run the end-to-end tests, from this directory execute
-
-```bash
-> docker compose -f tests/docker-compose.yml run python-uv uv run pytest tests/e2e_tests
-```
+## Running unit tests
 
-When you have finished running the tests run
+Run the unit tests using `pytest`:
 
 ```bash
-> docker compose -f tests/docker-compose.yml down
+> pytest tests/unit_tests
 ```
 
-to bring down the dependent services.
-
-### Debugging
-
-Using a debugger to debug the end-to-end tests is more complicated as it requires
-the dependent services to be accessible using their service names from within
-the compose file.
-
-To workaround this the `/etc/hosts` file can be edited to map the service names
-to localhost (127.0.0.1). Open `/etc/hosts` and add
+## Running end-to-end tests
 
-```text
-# docker compose services
-127.0.0.1 minio
-127.0.0.1 keycloak
-127.0.0.1 lakekeeper
-```
+The end-to-end (e2e) tests for the `pyiceberg` destination require a running Iceberg
+rest catalog to test complete functionality.
+The local, docker-compose-based configuration provided by
+[infra/local/docker-compose.yml](../infra/local/docker-compose.yml) is the easiest way to
+spin up a set of services compatible with running the tests.
+_Note the requirement to edit `/etc/hosts` described in [here](../infra/local/README.md)._
 
-Now bring up the services:
+Once the compose services are running, execute the e2e tests using `pytest`:
 
 ```bash
-> docker compose -f tests/docker-compose.yml up -d
+> pytest tests/e2e_tests
 ```
-
-and start your debugger as normal.

elt-common/src/elt_common/dlt_destinations/pyiceberg/configuration.py

@@ -15,6 +15,7 @@
 @configspec(init=False)
 class PyIcebergRestCatalogCredentials(CredentialsConfiguration):
     uri: str = None  # type: ignore
+    project_id: Optional[str] = None
     warehouse: Optional[str] = None
     access_delegation: TPyIcebergAccessDelegation = "vended-credentials"
     oauth2_server_uri: Optional[str] = None  # This is the endpoint to use to retrieve a token
@@ -29,8 +30,9 @@ def as_dict(self) -> Dict[str, str]:
         properties = {"credential": self.client_credential()} if self.client_id else {}
 
         field_aliases: Dict[str, str] = {
-            "access_delegation": f"{CATALOG_HEADER_PREFIX}X-Iceberg-Access-Delegation",
+            "access_delegation": f"{CATALOG_HEADER_PREFIX}x-iceberg-access-delegation",
             "oauth2_server_uri": "oauth2-server-uri",
+            "project_id": f"{CATALOG_HEADER_PREFIX}x-project-id",
         }
         skip_fields = ("client_id", "client_secret")
         properties.update(

elt-common/src/elt_common/iceberg/trino.py

@@ -109,5 +109,6 @@ def _create_engine(self, credentials: TrinoCredentials) -> Engine:
             connect_args={
                 "auth": BasicAuthentication(credentials.user, credentials.password),
                 "http_scheme": credentials.http_scheme,
+                "verify": False,
             },
         )

elt-common/tests/e2e_tests/conftest.py

@@ -1,6 +1,7 @@
 from contextlib import contextmanager
 import dataclasses
 from pathlib import Path
+import time
 from typing import Any, Callable, Dict, Generator, List
 import urllib.parse
 import uuid
@@ -28,7 +29,7 @@
 
 _RETRY_ARGS = {
     "wait": tenacity.wait_exponential(max=10),
-    "stop": tenacity.stop_after_attempt(10),
+    "stop": tenacity.stop_after_attempt(5),
     "reraise": True,
 }
 
@@ -76,33 +77,36 @@ class Settings(BaseSettings):
     # The default values assume the docker-compose.yml in the infra/local has been used.
     # These are provided for the convenience of easily running a debugger without having
     # to set up remote debugging
-    host_netloc: str = "localhost:58080"
-    docker_netloc: str = "traefik"
-    s3_access_key: str = "adpuser"
+    host_netloc: str = "localhost:50080"
+    docker_netloc: str = "adp-router:50080"
+    s3_access_key: str = "adpsuperuser"
     s3_secret_key: str = "adppassword"
-    s3_bucket: str = "e2e-tests-warehouse"
-    s3_endpoint: str = "http://minio:59000"
+    s3_bucket: str = "e2e-tests"
+    s3_endpoint: str = "http://adp-router:59000"
     s3_region: str = "local-01"
     s3_path_style_access: bool = True
-    openid_client_id: str = "localinfra"
+    openid_client_id: str = "machine-infra"
     openid_client_secret: str = "s3cr3t"
     openid_scope: str = "lakekeeper"
+    project_id: str = "c4fcd44f-7ce7-4446-9f7c-dcc7ba76dd22"
     warehouse_name: str = "e2e_tests"
 
     # trino
-    trino_http_scheme: str = "http"
+    trino_http_scheme: str = "https"
     trino_host: str = "localhost"
-    trino_port: str = "59088"
-    trino_user: str = "trino"
-    trino_password: str = ""
+    trino_port: str = "58443"
+    trino_user: str = "machine-infra"
+    trino_password: str = "s3cr3t"
 
     @property
     def lakekeeper_url(self) -> Endpoint:
         return Endpoint(f"http://{self.host_netloc}/iceberg", self.docker_netloc)
 
     @property
     def openid_provider_uri(self) -> Endpoint:
-        return Endpoint(f"http://{self.host_netloc}/auth/realms/iceberg", self.docker_netloc)
+        return Endpoint(
+            f"http://{self.host_netloc}/auth/realms/analytics-data-platform", self.docker_netloc
+        )
 
     def storage_config(self) -> Dict[str, Any]:
         return {
@@ -129,26 +133,12 @@ def storage_config(self) -> Dict[str, Any]:
 
 
 class Server:
+    """Wraps a Lakekeeper instance. It is assumed that the instance is bootstrapped."""
+
     def __init__(self, access_token: str, settings: Settings):
         self.access_token = access_token
         self.settings = settings
 
-        # Bootstrap server once
-        management_endpoint_v1 = self.management_endpoint(version=1)
-        server_info = self._request_with_auth(
-            requests.get,
-            url=management_endpoint_v1 + "/info",
-        )
-        server_info.raise_for_status()
-        server_info = server_info.json()
-        if not server_info["bootstrapped"]:
-            response = self._request_with_auth(
-                requests.post,
-                management_endpoint_v1 + "/bootstrap",
-                json={"accept-terms-of-use": True},
-            )
-            response.raise_for_status()
-
     @property
     def token_endpoint(self) -> Endpoint:
         return self.settings.openid_provider_uri + "/protocol/openid-connect/token"
@@ -169,13 +159,11 @@ def management_endpoint(self, *, version: int | None = None) -> Endpoint:
     def warehouse_endpoint(self, *, version: int = 1) -> Endpoint:
         return self.management_endpoint(version=version) + "/warehouse"
 
-    def create_warehouse(
-        self, name: str, project_id: uuid.UUID, storage_config: dict
-    ) -> "Warehouse":
+    def create_warehouse(self, name: str, project_id: str, storage_config: dict) -> "Warehouse":
         """Create a warehouse in this server"""
 
         payload = {
-            "project-id": str(project_id),
+            "project-id": project_id,
             **storage_config,
         }
 
@@ -233,6 +221,7 @@ def connect(self) -> PyIcebergCatalog:
         """Connect to the warehouse in the catalog"""
         creds = PyIcebergCatalogCredentials()
         creds.uri = str(self.server.catalog_endpoint())
+        creds.project_id = self.server.settings.project_id
         creds.warehouse = self.name
         creds.oauth2_server_uri = str(self.server.token_endpoint)
         creds.client_id = self.server.settings.openid_client_id
@@ -326,12 +315,7 @@ def server(access_token: str) -> Server:
 
 
 @pytest.fixture(scope="session")
-def project() -> uuid.UUID:
-    return uuid.UUID("{00000000-0000-0000-0000-000000000000}")
-
-
-@pytest.fixture(scope="session")
-def warehouse(server: Server, project: uuid.UUID) -> Generator:
+def warehouse(server: Server) -> Generator:
     if not settings.warehouse_name:
         raise ValueError("Empty 'warehouse_name' is not allowed.")
 
@@ -349,7 +333,9 @@ def warehouse(server: Server, project: uuid.UUID) -> Generator:
         minio_client.make_bucket(bucket_name=bucket_name)
         print(f"Bucket {bucket_name} created.")
 
-    warehouse = server.create_warehouse(settings.warehouse_name, project, storage_config)
+    warehouse = server.create_warehouse(
+        settings.warehouse_name, server.settings.project_id, storage_config
+    )
     print(f"Warehouse {warehouse.project_id} created.")
     try:
         yield warehouse
@@ -360,6 +346,8 @@ def _remove_bucket(bucket_name):
             minio_client.remove_bucket(bucket_name=bucket_name)
 
         try:
+            # Allow a brief pause for the test operations to complete
+            time.sleep(1)
             server.purge_warehouse(warehouse)
             server.delete_warehouse(warehouse)
             _remove_bucket(bucket_name)