Multiple warehouses (#134)

### Summary

Big overhaul of Ansible code to support defining multiple Lakekeeper
warehouses along with multiple Superset instances.

The Lakekeeper catalog warehouses are defined by a dict in
`infra/ansible-docker/group_vars/all/lakekeeper.yml`. The list of
Superset instances is defined in
`infra/ansible-docker/group_vars/all/superset.yml` with each specific
instance configured by a
`infra/ansible-docker/playbooks/superset/group_vars`.

The definitions here create a single node to house multiple Superset
instances to reduce the resource requirements at these early stages of
usage. Each Superset instance uses a separate schema in the same backend
database hosted by DB services. The DB schema currently needs to be
created manually.

We now also have a single `create_vm` playbook that can be parametrized instead
of a separate create playbook for each playbook.

Author: martyngigg

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,18 @@
-name: End-to-end tests for pipelines-common package
+name: End-to-end tests for elt-common package
 on:
   push:
     branches:
       - "main"
     paths:
-      - ".github/workflows/pipelines-common_e2e_tests.yml"
+      - ".github/workflows/elt-common_e2e_tests.yml"
       - "infra/local/**"
-      - "pipelines/pipelines-common/**"
+      - "elt-common/**"
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
-      - ".github/workflows/pipelines-common_e2e_tests.yml"
+      - ".github/workflows/elt-common_e2e_tests.yml"
       - "infra/local/**"
-      - "pipelines/pipelines-common/**"
+      - "elt-common/**"
   workflow_dispatch:
     inputs:
       ref:
@@ -43,7 +43,7 @@ jobs:
       - name: Bring up Lakekeeper & dependent services
         uses: hoverkraft-tech/compose-action@v2.0.1
         with:
-          compose-file: "pipelines/pipelines-common/tests/docker-compose.yml"
+          compose-file: "elt-common/tests/docker-compose.yml"
 
       - name: Run pipeline-common end-to-end tests
         run: >
@@ -53,4 +53,4 @@ jobs:
           --env CI=1
           python-uv
           uv run pytest --durations-min=0.5 --exitfirst tests/e2e_tests --cache-clear
-        working-directory: ./pipelines/pipelines-common
+        working-directory: ./elt-common

…workflows/pipelines-common_e2e_tests.yml …ithub/workflows/elt-common_e2e_tests.yml.github/workflows/pipelines-common_e2e_tests.yml renamed to .github/workflows/elt-common_e2e_tests.yml .github/workflows/pipelines-common_e2e_tests.yml renamed to .github/workflows/elt-common_e2e_tests.yml

@@ -1,16 +1,16 @@
-name: Unit tests for pipelines-common package
+name: Unit tests for elt-common package
 on:
   push:
     branches:
       - "main"
     paths:
-      - ".github/workflows/pipelines-common_unit_tests.yml"
-      - "pipelines/pipelines-common/**"
+      - ".github/workflows/elt-common_unit_tests.yml"
+      - "pipelines/elt-common/**"
   pull_request:
     types: [opened, synchronize, reopened]
     paths:
-      - ".github/workflows/pipelines-common_unit_tests.yml"
-      - "pipelines/pipelines-common/**"
+      - ".github/workflows/elt-common_unit_tests.yml"
+      - "pipelines/elt-common/**"
   workflow_dispatch:
     inputs:
       ref:
@@ -43,12 +43,12 @@ jobs:
         with:
           # Install a specific version of uv.
           version: "0.7.3"
-          cache-dependency-glob: "pipelines/pipelines-common/pyproject.toml"
+          cache-dependency-glob: "elt-common/pyproject.toml"
 
       - name: Install the project
         run: uv sync --locked --all-extras --dev
-        working-directory: pipelines/pipelines-common
+        working-directory: elt-common
 
       - name: Run tests
         run: uv run pytest --durations-min=0.5 --exitfirst tests/unit_tests --cache-clear
-        working-directory: pipelines/pipelines-common
+        working-directory: elt-common

…orkflows/pipelines-common_unit_tests.yml …thub/workflows/elt-common_unit_tests.yml.github/workflows/pipelines-common_unit_tests.yml renamed to .github/workflows/elt-common_unit_tests.yml .github/workflows/pipelines-common_unit_tests.yml renamed to .github/workflows/elt-common_unit_tests.yml

@@ -12,7 +12,7 @@ repos:
     hooks:
       # Run the linter.
       - id: ruff-check
-        files: pipelines/pipelines-common
+        files: elt-common
       # Run the formatter.
       - id: ruff-format
-        files: pipelines/pipelines-common
+        files: elt-common

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.0.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "pipelines-common"
+name = "elt-common"
 version = "0.1.0"
 description = "A set of common utility routines for data pipelines."
 readme = "README.md"

pipelines/pipelines-common/.gitignore elt-common/.gitignorepipelines/pipelines-common/.gitignore renamed to elt-common/.gitignore

@@ -93,7 +93,7 @@ def cli_main(
     ).parse_args()
     # TODO: Make the log filtering more configurable
     logging_utils.configure_logging(
-        args.log_level, keep_records_from=["dlt", "pipelines_common", "__main__"]
+        args.log_level, keep_records_from=["dlt", "elt_common", "__main__"]
     )
 
     pipeline = dlt.pipeline(