From 3fc52c6166062e3d1e3c823df6cea0362ced4aeb Mon Sep 17 00:00:00 2001 From: rederik76 Date: Sat, 23 May 2026 15:20:19 +1000 Subject: [PATCH 1/9] feat(package): restructure src/ into pip-installable lakeflow_framework package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Introduce src/lakeflow_framework/ as a proper Python package with pyproject.toml (hatchling build); config/ and schemas/ bundled as package data - Implement Strategy B (Workspace Files-first) resolver in config_resolver: load_framework_default_json resolves via Workspace Files → importlib.resources → local/config/ overlay; load_framework_schema returns an importlib.resources traversable for bundled JSON schemas - Reduce src/*.py shims to thin re-exports from lakeflow_framework for backward compat - Add contrib/ extension point with README and __init__ stub - Add tests: test_package.py (import surface), test_strategy_b_resolver.py (resolver + schema) - Update all internal imports across dataflow/, dataflow_spec_builder/, and support modules - Add docs: ADR-0007 (package layout), ADR-0008 (Workspace Files-first resolver), deploy_wheel.rst, deploy_framework_overview.rst, contributor_contrib.rst; update all existing docs/ pages to reference lakeflow_framework imports --- .../0007-lakeflow-framework-package.md | 145 ++++ .../0008-strategy-b-disk-first-resolver.md | 120 ++++ docs/source/concepts.rst | 11 +- docs/source/contributor.rst | 1 + docs/source/contributor_contrib.rst | 179 +++++ docs/source/contributor_dev_env.rst | 17 + docs/source/deploy_framework.rst | 2 + docs/source/deploy_framework_overview.rst | 38 + docs/source/deploy_wheel.rst | 105 +++ docs/source/feature_auto_complete.rst | 16 +- .../feature_builder_parallelization.rst | 2 +- .../feature_framework_configuration.rst | 60 +- .../feature_mandatory_table_properties.rst | 2 +- docs/source/feature_spark_configuration.rst | 2 +- docs/source/feature_spec_format.rst | 8 +- docs/source/feature_substitutions.rst | 4 +- docs/source/feature_table_migration.rst | 2 +- docs/source/feature_validation.rst | 2 +- .../feature_versioning_dataflow_spec.rst | 4 +- docs/source/feature_versioning_framework.rst | 10 + docs/source/introduction.rst | 10 +- pyproject.toml | 30 + requirements-dev.txt | 1 + scripts/validate_dataflows.py | 4 +- src/__init__.py | 61 +- src/bundle_loader.py | 175 +---- src/config_resolver.py | 158 +---- src/constants.py | 211 +----- src/dataflow/cdc_snapshot.py | 91 ++- src/dlt_pipeline.ipynb | 8 +- src/dlt_pipeline_builder.py | 467 +------------ src/lakeflow_framework/__init__.py | 96 +++ src/lakeflow_framework/bundle_loader.py | 168 +++++ .../0.1.0/dataflow_spec_mapping.json | 0 .../0.2.0/dataflow_spec_mapping.json | 0 .../config/default/global.json | 0 .../default/operational_metadata_bronze.json | 0 .../default/operational_metadata_gold.json | 0 .../default/operational_metadata_silver.json | 0 src/lakeflow_framework/config_resolver.py | 269 ++++++++ src/lakeflow_framework/constants.py | 200 ++++++ src/lakeflow_framework/contrib/README.rst | 35 + src/lakeflow_framework/contrib/__init__.py | 1 + .../dataflow/__init__.py | 0 src/{ => lakeflow_framework}/dataflow/cdc.py | 2 +- .../dataflow/cdc_snapshot.py | 651 ++++++++++++++++++ .../dataflow/dataflow.py | 4 +- .../dataflow/dataflow_config.py | 0 .../dataflow/dataflow_spec.py | 22 +- .../dataflow/enums.py | 0 .../dataflow/expectations.py | 2 +- .../dataflow/features.py | 0 .../dataflow/flow_group.py | 0 .../dataflow/flows/__init__.py | 0 .../dataflow/flows/append_sql.py | 0 .../dataflow/flows/append_view.py | 2 +- .../dataflow/flows/base.py | 2 +- .../dataflow/flows/factory.py | 0 .../dataflow/flows/materialized_view.py | 0 .../dataflow/flows/merge.py | 0 .../dataflow/operational_metadata.py | 2 +- .../dataflow/quarantine.py | 6 +- .../dataflow/schema.py | 4 +- .../dataflow/sources/__init__.py | 0 .../dataflow/sources/base.py | 6 +- .../dataflow/sources/batch_files.py | 0 .../dataflow/sources/cloud_files.py | 0 .../dataflow/sources/delta.py | 4 +- .../dataflow/sources/delta_join.py | 0 .../dataflow/sources/factory.py | 0 .../dataflow/sources/kafka.py | 0 .../dataflow/sources/python.py | 6 +- .../dataflow/sources/sql.py | 0 src/{ => lakeflow_framework}/dataflow/sql.py | 0 .../dataflow/table_import.py | 6 +- .../dataflow/table_migration.py | 4 +- .../dataflow/targets/__init__.py | 0 .../dataflow/targets/base.py | 4 +- .../targets/delta_materialized_view.py | 0 .../dataflow/targets/delta_streaming_table.py | 0 .../dataflow/targets/factory.py | 0 .../dataflow/targets/sink_custom_python.py | 0 .../dataflow/targets/sink_delta.py | 0 .../dataflow/targets/sink_foreach_batch.py | 2 +- .../dataflow/targets/sink_kafka.py | 0 .../dataflow/targets/staging_table.py | 0 src/{ => lakeflow_framework}/dataflow/view.py | 2 +- .../dataflow_spec_builder/__init__.py | 0 .../dataflow_spec_builder.py | 10 +- .../expectations_builder.py | 6 +- .../dataflow_spec_builder/spec_mapper.py | 8 +- .../template_processor.py | 6 +- .../transformer/__init__.py | 0 .../dataflow_spec_builder/transformer/base.py | 2 +- .../transformer/factory.py | 0 .../dataflow_spec_builder/transformer/flow.py | 0 .../transformer/materialized_views.py | 2 +- .../transformer/standard.py | 2 +- .../dlt_pipeline_builder.py | 464 +++++++++++++ src/lakeflow_framework/logger.py | 319 +++++++++ src/lakeflow_framework/pipeline_config.py | 139 ++++ src/lakeflow_framework/pipeline_details.py | 14 + .../schemas/definitions_flows.json | 0 .../schemas/definitions_main.json | 0 .../schemas/definitions_sources.json | 0 .../schemas/definitions_targets.json | 0 .../schemas/expectations.json | 0 .../schemas/flow_group.json | 0 .../schemas/main.json | 0 .../schemas/secrets.json | 0 .../schemas/spec_flows.json | 0 .../schemas/spec_mapping.json | 0 .../schemas/spec_materialized_views.json | 0 .../schemas/spec_standard.json | 0 .../schemas/spec_template.json | 0 .../schemas/spec_template_definition.json | 0 src/lakeflow_framework/secrets_manager.py | 196 ++++++ .../substitution_manager.py | 186 +++++ src/lakeflow_framework/utility.py | 535 ++++++++++++++ src/logger.py | 329 +-------- src/pipeline_config.py | 143 +--- src/pipeline_details.py | 17 +- src/secrets_manager.py | 203 +----- src/substitution_manager.py | 189 +---- src/utility.py | 539 +-------------- tests/test_bundle_loader.py | 2 +- tests/test_package.py | 57 ++ tests/test_strategy_b_resolver.py | 166 +++++ tests/test_validate_dataflows.py | 6 +- 129 files changed, 4451 insertions(+), 2535 deletions(-) create mode 100644 docs/decisions/0007-lakeflow-framework-package.md create mode 100644 docs/decisions/0008-strategy-b-disk-first-resolver.md create mode 100644 docs/source/contributor_contrib.rst create mode 100644 docs/source/deploy_framework_overview.rst create mode 100644 docs/source/deploy_wheel.rst create mode 100644 pyproject.toml create mode 100644 src/lakeflow_framework/__init__.py create mode 100644 src/lakeflow_framework/bundle_loader.py rename src/{ => lakeflow_framework}/config/default/dataflow_spec_mapping/0.1.0/dataflow_spec_mapping.json (100%) rename src/{ => lakeflow_framework}/config/default/dataflow_spec_mapping/0.2.0/dataflow_spec_mapping.json (100%) rename src/{ => lakeflow_framework}/config/default/global.json (100%) rename src/{ => lakeflow_framework}/config/default/operational_metadata_bronze.json (100%) rename src/{ => lakeflow_framework}/config/default/operational_metadata_gold.json (100%) rename src/{ => lakeflow_framework}/config/default/operational_metadata_silver.json (100%) create mode 100644 src/lakeflow_framework/config_resolver.py create mode 100644 src/lakeflow_framework/constants.py create mode 100644 src/lakeflow_framework/contrib/README.rst create mode 100644 src/lakeflow_framework/contrib/__init__.py rename src/{ => lakeflow_framework}/dataflow/__init__.py (100%) rename src/{ => lakeflow_framework}/dataflow/cdc.py (98%) create mode 100644 src/lakeflow_framework/dataflow/cdc_snapshot.py rename src/{ => lakeflow_framework}/dataflow/dataflow.py (99%) rename src/{ => lakeflow_framework}/dataflow/dataflow_config.py (100%) rename src/{ => lakeflow_framework}/dataflow/dataflow_spec.py (90%) rename src/{ => lakeflow_framework}/dataflow/enums.py (100%) rename src/{ => lakeflow_framework}/dataflow/expectations.py (98%) rename src/{ => lakeflow_framework}/dataflow/features.py (100%) rename src/{ => lakeflow_framework}/dataflow/flow_group.py (100%) rename src/{ => lakeflow_framework}/dataflow/flows/__init__.py (100%) rename src/{ => lakeflow_framework}/dataflow/flows/append_sql.py (100%) rename src/{ => lakeflow_framework}/dataflow/flows/append_view.py (97%) rename src/{ => lakeflow_framework}/dataflow/flows/base.py (98%) rename src/{ => lakeflow_framework}/dataflow/flows/factory.py (100%) rename src/{ => lakeflow_framework}/dataflow/flows/materialized_view.py (100%) rename src/{ => lakeflow_framework}/dataflow/flows/merge.py (100%) rename src/{ => lakeflow_framework}/dataflow/operational_metadata.py (98%) rename src/{ => lakeflow_framework}/dataflow/quarantine.py (98%) rename src/{ => lakeflow_framework}/dataflow/schema.py (97%) rename src/{ => lakeflow_framework}/dataflow/sources/__init__.py (100%) rename src/{ => lakeflow_framework}/dataflow/sources/base.py (98%) rename src/{ => lakeflow_framework}/dataflow/sources/batch_files.py (100%) rename src/{ => lakeflow_framework}/dataflow/sources/cloud_files.py (100%) rename src/{ => lakeflow_framework}/dataflow/sources/delta.py (97%) rename src/{ => lakeflow_framework}/dataflow/sources/delta_join.py (100%) rename src/{ => lakeflow_framework}/dataflow/sources/factory.py (100%) rename src/{ => lakeflow_framework}/dataflow/sources/kafka.py (100%) rename src/{ => lakeflow_framework}/dataflow/sources/python.py (96%) rename src/{ => lakeflow_framework}/dataflow/sources/sql.py (100%) rename src/{ => lakeflow_framework}/dataflow/sql.py (100%) rename src/{ => lakeflow_framework}/dataflow/table_import.py (96%) rename src/{ => lakeflow_framework}/dataflow/table_migration.py (99%) rename src/{ => lakeflow_framework}/dataflow/targets/__init__.py (100%) rename src/{ => lakeflow_framework}/dataflow/targets/base.py (99%) rename src/{ => lakeflow_framework}/dataflow/targets/delta_materialized_view.py (100%) rename src/{ => lakeflow_framework}/dataflow/targets/delta_streaming_table.py (100%) rename src/{ => lakeflow_framework}/dataflow/targets/factory.py (100%) rename src/{ => lakeflow_framework}/dataflow/targets/sink_custom_python.py (100%) rename src/{ => lakeflow_framework}/dataflow/targets/sink_delta.py (100%) rename src/{ => lakeflow_framework}/dataflow/targets/sink_foreach_batch.py (99%) rename src/{ => lakeflow_framework}/dataflow/targets/sink_kafka.py (100%) rename src/{ => lakeflow_framework}/dataflow/targets/staging_table.py (100%) rename src/{ => lakeflow_framework}/dataflow/view.py (98%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/__init__.py (100%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/dataflow_spec_builder.py (99%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/expectations_builder.py (96%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/spec_mapper.py (98%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/template_processor.py (98%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/transformer/__init__.py (100%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/transformer/base.py (96%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/transformer/factory.py (100%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/transformer/flow.py (100%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/transformer/materialized_views.py (98%) rename src/{ => lakeflow_framework}/dataflow_spec_builder/transformer/standard.py (98%) create mode 100644 src/lakeflow_framework/dlt_pipeline_builder.py create mode 100644 src/lakeflow_framework/logger.py create mode 100644 src/lakeflow_framework/pipeline_config.py create mode 100644 src/lakeflow_framework/pipeline_details.py rename src/{ => lakeflow_framework}/schemas/definitions_flows.json (100%) rename src/{ => lakeflow_framework}/schemas/definitions_main.json (100%) rename src/{ => lakeflow_framework}/schemas/definitions_sources.json (100%) rename src/{ => lakeflow_framework}/schemas/definitions_targets.json (100%) rename src/{ => lakeflow_framework}/schemas/expectations.json (100%) rename src/{ => lakeflow_framework}/schemas/flow_group.json (100%) rename src/{ => lakeflow_framework}/schemas/main.json (100%) rename src/{ => lakeflow_framework}/schemas/secrets.json (100%) rename src/{ => lakeflow_framework}/schemas/spec_flows.json (100%) rename src/{ => lakeflow_framework}/schemas/spec_mapping.json (100%) rename src/{ => lakeflow_framework}/schemas/spec_materialized_views.json (100%) rename src/{ => lakeflow_framework}/schemas/spec_standard.json (100%) rename src/{ => lakeflow_framework}/schemas/spec_template.json (100%) rename src/{ => lakeflow_framework}/schemas/spec_template_definition.json (100%) create mode 100644 src/lakeflow_framework/secrets_manager.py create mode 100644 src/lakeflow_framework/substitution_manager.py create mode 100644 src/lakeflow_framework/utility.py create mode 100644 tests/test_package.py create mode 100644 tests/test_strategy_b_resolver.py diff --git a/docs/decisions/0007-lakeflow-framework-package.md b/docs/decisions/0007-lakeflow-framework-package.md new file mode 100644 index 0000000..a61bf4a --- /dev/null +++ b/docs/decisions/0007-lakeflow-framework-package.md @@ -0,0 +1,145 @@ +# ADR-0007: `lakeflow_framework` pip-installable package + +**Date:** 2026-05-23 +**Status:** Accepted +**PR:** refactor/lakeflow-framework-package (v0.16.0) + +--- + +## Context + +The framework shipped as a flat collection of `.py` files directly under `src/`. +Consumers cloned the repository and added `src/` to `sys.path` via +`framework.sourcePath` — effectively a "copy the source" distribution model. + +This had three structural problems: + +1. **No stable import namespace.** All modules were importable as bare names + (`from constants import ...`, `from bundle_loader import ...`). Any module + name in a customer's pipeline bundle could shadow a framework module. + +2. **No pip distribution path.** Teams that manage Python dependencies via PyPI, + a UC Volume, or Artifactory had no clean way to consume the framework. Every + consumer was forced to clone the repository and re-deploy it as a DAB bundle. + +3. **Data files (config, schemas) lived outside any package.** `src/config/default/` + and `src/schemas/` were loaded via hard-coded `os.path` joins relative to + `framework.sourcePath`. There was no mechanism to bundle these files with a + wheel and access them portably. + +The project's `pyproject.toml` did not exist; `setup.py` or any standard packaging +entry point was absent. + +## Decision + +Introduce a proper Python package under `src/lakeflow_framework/` and a +`pyproject.toml` at the repository root. + +### Package layout + +``` +src/ + lakeflow_framework/ + __init__.py ← public API + __version__ + constants.py ← FrameworkPaths, enums + config_resolver.py + bundle_loader.py + logger.py + dlt_pipeline_builder.py + ... + config/ + default/ ← bundled default config (package data) + schemas/ ← bundled JSON schemas (package data) + dataflow/ + flows/ + dataflow_spec_builder/ + contrib/ + __init__.py ← scaffold; empty until first module lands + README.rst +``` + +All imports inside the package use **absolute `lakeflow_framework.*` names**. +No bare imports remain inside `src/lakeflow_framework/`. + +### Compat shims + +The old flat `src/*.py` locations (e.g. `src/constants.py`, +`src/bundle_loader.py`) are replaced by thin re-export shims: + +```python +# src/constants.py — compat shim, remove at v1.0.0 +from lakeflow_framework.constants import * # noqa: F401,F403 +``` + +Existing pipeline notebooks and bundles that import bare names continue to work +without change until `v1.0.0`. + +### `pyproject.toml` + +```toml +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "lakeflow-framework" +dynamic = ["version"] +... + +[tool.setuptools.dynamic] +version = { file = "VERSION" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +lakeflow_framework = [ + "config/default/**/*", + "schemas/**/*", +] +``` + +`VERSION` is the single source of truth for the release version; both +`importlib.metadata` (wheel installs) and a direct file read (editable / flat +deploy) converge on it. + +### `contrib` subpackage + +`src/lakeflow_framework/contrib/` is introduced as a scaffold with an empty +`__init__.py` and a support-policy `README.rst`. No modules land in this PR. +See `docs/decisions/0007-*` (this file) for the publish model and +`docs/source/contributor_contrib.rst` for the contributor guide. + +### Optional extras + +``` +pip install lakeflow-framework # core only +pip install "lakeflow-framework[contrib]" # + future contrib extras +pip install "lakeflow-framework[all]" # everything +``` + +The `contrib` extra is currently empty; installing it is a no-op until a contrib +module is added. + +### Deprecation timeline + +| Version | Action | +|---------|--------| +| v0.16.0 | `src/lakeflow_framework/` package introduced; bare `src/` imports still work via shims | +| v1.0.0 | Compat shims at old flat `src/*.py` paths removed | + +## Consequences + +- Teams can now `pip install lakeflow-framework` from PyPI, a UC Volume, or + Artifactory. The wheel bundles default config and schemas; no separate file + deployment is required. +- The `lakeflow_framework.*` import namespace is globally unique; no shadow-import + risk with customer code. +- Existing flat-deploy customers are unaffected: the `src/` shims preserve + backward compatibility, and `framework.sourcePath` + disk-first resolution + (ADR-0008) means behaviour is identical to pre-v0.16.0. +- Editable installs (`pip install -e ".[contrib]"`) are supported for local + development; the `src/` layout ensures the installed package and the source + tree are the same directory. +- `docs/conf.py` reads `release` from the `VERSION` file at build time — the + version shown in HTML headers is always current without manual edits. diff --git a/docs/decisions/0008-strategy-b-disk-first-resolver.md b/docs/decisions/0008-strategy-b-disk-first-resolver.md new file mode 100644 index 0000000..ba9aa46 --- /dev/null +++ b/docs/decisions/0008-strategy-b-disk-first-resolver.md @@ -0,0 +1,120 @@ +# ADR-0008: Strategy B — Workspace Files-first config and schema resolution + +**Date:** 2026-05-23 +**Status:** Accepted +**PR:** refactor/lakeflow-framework-package (v0.16.0) + +--- + +## Context + +With the introduction of the `lakeflow_framework` Python package (ADR-0007), +default config files (`config/default/`) and JSON schemas (`schemas/`) are +bundled inside the wheel via `importlib.resources`. Previously they lived only +in Workspace Files at `{framework_path}/lakeflow_framework/config/default/`. + +Two deployments now co-exist: + +| Deploy mode | Where defaults live | +|-------------|---------------------| +| Flat DAB deploy | On workspace files under `framework.sourcePath` | +| Wheel install | Bundled inside the wheel (`importlib.resources`) | +| Wheel + local overlay | Wheel bundled, plus `src/local/config/` fragments | + +The resolver (`load_framework_default_json`) must decide which source to read +first when both a Workspace Files path and the wheel package data are potentially available. + +### Options considered + +**Option A — Package-first (wheel-first)** +- Always read from `importlib.resources`; treat Workspace Files as an optional + override layer. +- `+` Simple: the wheel is always the canonical source of defaults. +- `−` **Breaking for existing flat-deploy customers.** Their defaults live in + Workspace Files; if the wheel shadow-contains a newer (or different) version of a default + file, behaviour silently changes on upgrade. +- `−` Contradicts the "explicit wins" principle: a customer who set + `framework.sourcePath` explicitly expects Workspace Files to be authoritative. + +**Option B — Workspace Files-first (chosen)** +- When `framework.sourcePath` is set and the file exists in Workspace Files, use it. +- Fall back to `importlib.resources` only when no Workspace Files path is configured or + the file is absent from Workspace Files. +- Always deep-merge `src/local/config/` overlay on top (unchanged from + ADR-0006). + +**Option C — Explicit-only** +- Require callers to pass either a Workspace Files path or `importlib.resources` traversable; + no automatic fallback. +- `−` Forces every call site to be aware of the deploy mode; adds boilerplate + at all 15+ call sites. + +## Decision + +Adopt **Strategy B (Workspace Files-first)** via `load_framework_default_json`: + +```python +def load_framework_default_json( + name: str, + framework_path: Optional[str] = None, +) -> Dict: + """ + Resolution order: + 1. Workspace Files — {framework_path}/lakeflow_framework/config/default/{name} + if framework_path is set and the file exists. + 2. Package data — importlib.resources (wheel or src/lakeflow_framework/ on sys.path). + 3. src/local/config/{name} overlay — deep-merged on top of (1) or (2) + when framework_path is set and the fragment exists. + """ +``` + +### Resolution sequence in detail + +1. **Workspace Files (explicit):** if `framework_path` is provided and + `{framework_path}/lakeflow_framework/config/default/{name}` exists, load it. +2. **Package data (fallback):** otherwise use + `importlib.resources.files("lakeflow_framework") / "config" / "default" / name`. + This works for both an installed wheel and an editable / flat-deploy install + (the `src/lakeflow_framework/` directory is on `sys.path` in both cases). +3. **Local overlay:** if `framework_path` is set and + `{framework_path}/src/local/config/{name}` exists, deep-merge it on top of + whichever base was loaded (dict keys merged recursively; overlay wins on + conflict; non-dict values replaced wholesale). This preserves the ADR-0006 + sparse-overlay guarantee regardless of deploy mode. + +The same logic applies to schema resolution via +`load_framework_schema(name)` in `config_resolver`, which returns an +`importlib.resources` traversable suitable for `jsonschema.RefResolver` and +similar validators. + +### Rationale for Workspace Files-first + +- **Zero-change upgrade for flat-deploy customers.** Their `framework.sourcePath` + points to the deployed `src/` directory in Workspace Files; those files are + found at step 1 and the wheel is never consulted. Behaviour is identical to + pre-v0.16.0. +- **Explicit wins.** Setting `framework.sourcePath` is a deliberate act. + Workspace Files-first honours that intent; package-first would silently override it. +- **Wheel-install customers get clean defaults from the package.** They do not + set `framework.sourcePath` (or set it only for local overlays), so step 1 + is skipped and `importlib.resources` is used — exactly as expected. +- **Testability.** Workspace Files-first is trivially testable: point + `framework_path` at a temp directory and place a fixture file there. + Package-data fallback is also testable by omitting `framework_path`. + +## Consequences + +- **`load_framework_default_json`** is the single resolver for all default + config and schema reads; call sites that previously used + `os.path.join(framework_path, FrameworkPaths.CONFIG_PATH, name)` are + migrated to it. +- Flat-deploy customers see **no behaviour change** on upgrade to v0.16.0. +- Wheel-install customers get defaults from the wheel with no `framework.sourcePath` + required. +- The `src/local/config/` sparse overlay (ADR-0006) continues to work in all + three deploy modes — it is applied at step 3 independent of which base was + used in steps 1–2. +- If a future release ships an intentionally different default (e.g. a renamed + key), flat-deploy customers picking up a new wheel without re-deploying their + bundle will continue to use their Workspace Files defaults until they re-deploy. + This is safe and predictable: Workspace Files-first ensures the Workspace Files always win. diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index 6295f08..00cc06d 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -52,11 +52,12 @@ The Framework Bundle contains: * - Component - Description * - **Framework Source Code** - - The core framework source code under the ``src`` folder. + - The core framework source code under the ``src/lakeflow_framework/`` package. * - **Global Framework Configuration** - - The global framework configuration under the ``src/config`` folder. - * - **Data Flow Spec Schema Definition** - - The Data Flow Spec schema definition and validations under the ``src/schemas`` folder. + - The framework default configuration under ``src/lakeflow_framework/config/default/``. Customer + overrides go in ``src/local/config/`` as sparse files — only the keys to change are needed. + * - **Data Flow Spec Schema Definitions** + - The Data Flow Spec schema definitions and validations under ``src/lakeflow_framework/schemas/``. * - **Deployment YAML file** - The ``databricks.yml`` file which defines the bundle deployment settings. @@ -68,7 +69,7 @@ The Framework Bundle is deployed to a given workspace files location from where Framework Configuration ~~~~~~~~~~~~~~~~~~~~~~~ -The Framework and most of its features will have configuration settings that can be set in the Framework Bundle ``src/config`` folder. The configuration settings are explained in the section: :doc:`features` +The Framework and most of its features will have configuration settings that can be set in the Framework Bundle ``src/lakeflow_framework/config/default/`` folder. Default values ship with the framework and are resolved automatically; individual keys can be overridden by placing sparse files in ``src/local/config/``. The configuration settings are explained in the section: :doc:`features` .. admonition:: Setting Precedence :class: note diff --git a/docs/source/contributor.rst b/docs/source/contributor.rst index c2c906a..885c173 100644 --- a/docs/source/contributor.rst +++ b/docs/source/contributor.rst @@ -9,3 +9,4 @@ Framework Development & Contributors contributor_dev_git contributor_dev_steps contributor_dev_docs + contributor_contrib diff --git a/docs/source/contributor_contrib.rst b/docs/source/contributor_contrib.rst new file mode 100644 index 0000000..7461628 --- /dev/null +++ b/docs/source/contributor_contrib.rst @@ -0,0 +1,179 @@ +Contributing to ``lakeflow_framework.contrib`` +################################################## + +The ``lakeflow_framework.contrib`` subpackage is the home for optional, +community-maintained integrations. It is deliberately separate from the core +framework so that: + +* The core ``pip install lakeflow-framework`` stays lightweight with no + extra dependencies. +* Community integrations can evolve at their own pace without affecting core + stability guarantees. +* Individual integrations can be gated behind their own extras (e.g. + ``pip install "lakeflow-framework[contrib.myintegration]"``). + +.. _contrib-deploy-context: + +Deployment context: wheel vs. flat DAB deploy +---------------------------------------------- + +Contrib modules live inside the ``lakeflow_framework`` package tree and +therefore behave differently depending on how the framework is deployed. +Understanding both modes is important when writing or using a contrib module. + +.. list-table:: + :header-rows: 1 + :widths: 20 40 40 + + * - Mode + - How the framework is available + - How contrib modules are available + * - **Wheel install** + - ``pip install lakeflow-framework`` via PyPI, UC Volume, or Artifactory. + The entire ``lakeflow_framework`` package — including ``contrib/`` — is + installed into the cluster Python environment. + - ``pip install "lakeflow-framework[contrib]"`` (or the specific + sub-extra). No further path setup needed. + * - **Flat DAB deploy** + - The framework repository is cloned and deployed via + ``databricks bundle deploy``. ``framework.sourcePath`` points to the + ``src/`` directory on workspace files, and Databricks adds it to + ``sys.path`` at pipeline start. + - ``src/lakeflow_framework/contrib/`` is on ``sys.path`` automatically + because ``src/`` is. However, any *external* dependencies declared by + the contrib module still need to be installed separately — they are not + bundled with the flat deploy. Install them as cluster libraries or + via ``src/local/libraries/`` (see :doc:`/source/feature_python_extensions`). + * - **Wheel + local overlay** + - Wheel installed, plus ``framework.sourcePath`` set for + ``src/local/config/`` sparse overrides. + - Same as wheel install — contrib is available from the wheel. + +.. admonition:: Contrib vs. pipeline-bundle custom code + :class: note + + ``contrib`` is for **reusable framework-level integrations** that benefit + many teams (e.g. a connector, a utility class, an alternative logger + backend). Pipeline-specific Python — transforms, domain logic, helper + functions used by a single pipeline — belongs in the pipeline bundle under + ``src/local/python/`` or ``src/python/``, not in ``contrib``. + +Support policy +-------------- + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Property + - Policy + * - **Install gate** + - ``pip install "lakeflow-framework[contrib]"`` (or the specific sub-extra + for the integration). + * - **Stability label** + - All contrib modules carry the **experimental** label until explicitly + graduated. Breaking changes between minor releases are allowed for + experimental modules. + * - **Stability graduation** + - A module graduates to *stable* via a PR that removes the experimental + label and passes a core-team review. Once stable, normal semver + deprecation rules apply. + * - **External dependencies** + - Must be declared as optional extras in ``pyproject.toml``. A plain + ``pip install lakeflow-framework`` must not pull them in. For flat + DAB deploy, contributors must document how to install them separately + (cluster libraries or ``src/local/libraries/``). + * - **Review scope** + - Maintainers review for safety and conformance with framework conventions. + Feature completeness is the contributor's responsibility. + +Adding a new contrib module +--------------------------- + +1. **Create the module directory** + + .. code-block:: text + + src/lakeflow_framework/contrib// + ├── __init__.py + └── README.rst ← required: purpose, status, external deps + +2. **Declare the extra in ``pyproject.toml``** + + Add a new entry under ``[project.optional-dependencies]``: + + .. code-block:: toml + + [project.optional-dependencies] + contrib = [] + contrib.myintegration = ["some-third-party>=1.0"] + all = ["lakeflow-framework[contrib]", "lakeflow-framework[contrib.myintegration]"] + +3. **Write a ``README.rst``** + + The README must state: + + * Purpose of the integration. + * Stability label (``experimental`` until graduated). + * Required external dependencies and minimum versions. + * **Install instructions for both deploy modes:** + + * Wheel: ``pip install "lakeflow-framework[contrib.myintegration]"`` + * Flat DAB deploy: how to install external deps as cluster libraries or + via ``src/local/libraries/``. + * Basic usage example. + +4. **Add tests** + + Tests for contrib modules must be gated behind the relevant extra. The + easiest pattern is a ``pytest.importorskip`` at the top of the test module: + + .. code-block:: python + + some_dep = pytest.importorskip("some_third_party") + + This ensures the test is skipped (not failed) when the extra is not + installed. + +5. **Open a pull request** + + Follow the standard PR process in :doc:`contributor_dev_steps`. In the PR + description, note: + + * The extra name being added. + * A brief rationale for why this belongs in ``contrib`` rather than the + core. + * Any known limitations or areas not yet covered. + +Module conventions +------------------ + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Convention + - Details + * - **``__init__.py``** + - Import only what is needed. Avoid heavy imports at module-load time + — use lazy imports if the external dependency is optional even within + the contrib module. + * - **Naming** + - Use ``snake_case`` for module directories. Avoid names that conflict + with top-level PyPI packages. + * - **Logging** + - Use the framework logger (``from lakeflow_framework.logger import + get_logger``) rather than ``print`` or a bare ``logging.getLogger``. + * - **No private framework internals** + - Contrib modules must import only from the public ``lakeflow_framework`` + API (``lakeflow_framework.*``). Importing from + ``lakeflow_framework._private.*`` is not allowed and will break without + notice. + +See also +-------- + +- ``src/lakeflow_framework/contrib/README.rst`` in the repository — the + in-source support policy document. +- :doc:`contributor_dev_steps` — general contribution workflow. +- :doc:`contributor_dev_env` — development environment setup. diff --git a/docs/source/contributor_dev_env.rst b/docs/source/contributor_dev_env.rst index 74e45e5..7fc3c00 100644 --- a/docs/source/contributor_dev_env.rst +++ b/docs/source/contributor_dev_env.rst @@ -28,6 +28,23 @@ Once you have cloned the Lakeflow Framework repository, you'll need to follow th too, so you do not need a separate install step for building the documentation. + Alternatively, for a quick editable install that resolves imports without + locking all transitive hashes (useful for IDE auto-complete and + ``pytest`` runs): + + .. code-block:: bash + + pip install -e ".[contrib]" + + This installs the framework in editable mode from ``pyproject.toml``, + including the ``[contrib]`` extra. Use ``pip install -e ".[all]"`` to also + pull in any future contrib sub-extras. You can then build a distribution + wheel at any time with: + + .. code-block:: bash + + python -m build + If you change a dependency in any of the ``requirements*.txt`` files, regenerate all three lockfiles with the helper script from the repo root: diff --git a/docs/source/deploy_framework.rst b/docs/source/deploy_framework.rst index 741a061..14c2a11 100644 --- a/docs/source/deploy_framework.rst +++ b/docs/source/deploy_framework.rst @@ -4,6 +4,8 @@ Deploy the Framework .. toctree:: :maxdepth: 1 + deploy_framework_overview deploy_framework_bundle deploy_ci_cd + deploy_wheel feature_versioning_framework diff --git a/docs/source/deploy_framework_overview.rst b/docs/source/deploy_framework_overview.rst new file mode 100644 index 0000000..92a8047 --- /dev/null +++ b/docs/source/deploy_framework_overview.rst @@ -0,0 +1,38 @@ +Deployment Options +################## + +The Lakeflow Framework can be deployed to a Databricks workspace in three +ways. Choose the mode that matches how your team manages dependencies and +configuration. + +.. list-table:: + :header-rows: 1 + :widths: 20 42 38 + + * - Mode + - How it works + - When to use + * - **Flat DAB deploy** *(default)* + - The framework repository is cloned and deployed with + ``databricks bundle deploy``. ``framework.sourcePath`` points to the + deployed ``src/`` directory on workspace files; the cluster reads + Python modules and default config directly from there. + - Default for all existing customers. No change to existing pipelines + required. + * - **Wheel install** + - ``pip install lakeflow-framework`` installs the package into the + cluster Python environment. Default configs and schemas are bundled + inside the wheel via ``importlib.resources``. + - Teams that manage Python dependencies via PyPI, a UC Volume, or an + internal Artifactory feed. + * - **Wheel + local overlay** + - Wheel installed, plus ``framework.sourcePath`` still set so that + ``src/local/config/`` sparse overrides are deep-merged on top of the + bundled defaults. + - Teams that want pip-managed installs but need per-deployment config + customisation. + +.. note:: + For wheel based deployments it is your responsibility to provide + an entry-point notebook and a location for local config overrides and + libraries/init scripts when moving away from the default flat deploy. \ No newline at end of file diff --git a/docs/source/deploy_wheel.rst b/docs/source/deploy_wheel.rst new file mode 100644 index 0000000..fc1d163 --- /dev/null +++ b/docs/source/deploy_wheel.rst @@ -0,0 +1,105 @@ +Installing the Framework as a Wheel +###################################### + +.. list-table:: + :header-rows: 0 + + * - **Available From:** + - v0.16.0 + +From v0.16.0 the Lakeflow Framework ships as a proper Python package +(``lakeflow-framework``) with a ``pyproject.toml``. For a comparison of all +deployment modes, see :doc:`deploy_framework_bundle`. + +Installing +---------- + +Core install (no optional extensions): + +.. code-block:: bash + + pip install lakeflow-framework + +With community extensions (``contrib``): + +.. code-block:: bash + + pip install "lakeflow-framework[contrib]" + +Everything: + +.. code-block:: bash + + pip install "lakeflow-framework[all]" + +.. note:: + ``[contrib]`` is currently empty — the ``contrib`` subpackage is a + scaffold for future community-maintained integrations. Installing it does + not add dependencies until a contrib module lands. + +Checking the installed version +------------------------------- + +From Python: + +.. code-block:: python + + import lakeflow_framework + print(lakeflow_framework.__version__) + +From the command line: + +.. code-block:: bash + + python -c "import lakeflow_framework; print(lakeflow_framework.__version__)" + +Config and schema resolution +----------------------------- + +When the framework is installed as a wheel, default configuration files and +JSON schemas are bundled inside the package and accessed via +``importlib.resources``. The resolution order is: + +1. **Workspace Files** — ``{framework_path}/lakeflow_framework/config/default/`` + when ``framework.sourcePath`` is configured and the file exists in workspace files. +2. **Package data** — bundled defaults from the wheel (or from + ``src/lakeflow_framework/`` on ``sys.path`` for flat deploy). +3. **``src/local/config/`` custom override** — deep-merged on top when + ``framework.sourcePath`` is set and the sparse fragment exists. + +See :ref:`config-resolution-order` in :doc:`feature_framework_configuration` +for the full table. + +Using ``src/local/config/`` custom override`` with a wheel +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you install the framework as a wheel but still want per-deployment config +overrides, set ``framework.sourcePath`` in your pipeline resource YAML to point +to a directory that contains ``local/config/``: + +.. code-block:: yaml + + spark_conf: + framework.sourcePath: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target}/files/src + +Then place sparse override files under that ``src/local/config/``. The +framework will load defaults from the wheel and deep-merge your overrides on +top. + +Backward compatibility +----------------------- + +- **Existing flat-deploy customers** are unaffected. The workspace files first + resolution means that if ``framework.sourcePath`` is set and default files + are in workspace files, they take priority over the wheel — behaviour is identical to + earlier releases. +- **Compat shims** at the old flat ``src/`` import paths (e.g. + ``from constants import FrameworkPaths``) remain until v1.0.0. Prefer + ``from lakeflow_framework.constants import FrameworkPaths`` in new code. + +See also +-------- + +- :doc:`deploy_framework_bundle` — deployment modes overview and flat DAB deploy +- :doc:`feature_framework_configuration` — full config resolution reference +- :doc:`feature_versioning_framework` — version pinning and rollback diff --git a/docs/source/feature_auto_complete.rst b/docs/source/feature_auto_complete.rst index a116155..1114ddc 100644 --- a/docs/source/feature_auto_complete.rst +++ b/docs/source/feature_auto_complete.rst @@ -64,25 +64,25 @@ Add the following code into your ``settings.json`` replacing ``/src/schemas/flow_group.json" + "url": "/src/lakeflow_framework/schemas/flow_group.json" }, { "fileMatch": [ "*_main.json" ], - "url": "/src/schemas/main.json" + "url": "/src/lakeflow_framework/schemas/main.json" }, { "fileMatch": [ "*_dqe.json" ], - "url": "/src/schemas/expectations.json" + "url": "/src/lakeflow_framework/schemas/expectations.json" }, { "fileMatch": [ "*_secrets.json" ], - "url": "/src/schemas/secrets.json" + "url": "/src/lakeflow_framework/schemas/secrets.json" } ] } @@ -100,25 +100,25 @@ Example ``settings.json`` file: "fileMatch": [ "*flow.json" ], - "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/schemas/flow_group.json" + "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/lakeflow_framework/schemas/flow_group.json" }, { "fileMatch": [ "*main.json" ], - "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/schemas/main.json" + "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/lakeflow_framework/schemas/main.json" }, { "fileMatch": [ "*_dqe.json" ], - "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/schemas/expectations.json" + "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/lakeflow_framework/schemas/expectations.json" }, { "fileMatch": [ "*_secrets.json" ], - "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/schemas/secrets.json" + "url": "/Users/erik.seefeld/Documents/dev_work/dlt_framework/src/lakeflow_framework/schemas/secrets.json" } ] } diff --git a/docs/source/feature_builder_parallelization.rst b/docs/source/feature_builder_parallelization.rst index 67f2abb..ce44a51 100644 --- a/docs/source/feature_builder_parallelization.rst +++ b/docs/source/feature_builder_parallelization.rst @@ -55,7 +55,7 @@ Configuration Global Configuration ~~~~~~~~~~~~~~~~~~~~~ -Configure these parameters globally for all pipelines in your ``src/config/default/global.json|yaml`` file: +Configure these parameters globally for all pipelines in your ``src/lakeflow_framework/config/default/global.json|yaml`` file: .. tabs:: diff --git a/docs/source/feature_framework_configuration.rst b/docs/source/feature_framework_configuration.rst index 9d98eb6..9d8c383 100644 --- a/docs/source/feature_framework_configuration.rst +++ b/docs/source/feature_framework_configuration.rst @@ -12,18 +12,62 @@ Framework configuration - NA Framework-level settings (global JSON/YAML, substitutions, secrets, spec -mappings, operational metadata) live under ``src/config/default/``. Individual -values can be overridden per-deployment using sparse files in -``src/local/config/`` — only the keys you want to change are needed. +mappings, operational metadata) ship inside the ``lakeflow_framework`` package +and are resolved automatically at runtime. Individual values can be overridden +per-deployment using sparse files in ``src/local/config/`` — only the keys you +want to change are needed. + +.. note:: + + **Default config location (v0.16.0+)** + + Framework default files now live inside the installed package at + ``src/lakeflow_framework/config/default/`` rather than at the top-level + ``src/config/default/`` used in earlier releases. If you are referencing + these paths directly (e.g. in CI scripts or custom tooling), update them + accordingly. All framework-internal loading is handled automatically and + requires no code changes. Configuration ------------- | **Scope: Global (framework bundle)** -| **Default:** ``src/config/default/`` — authoritative, always read. +| **Defaults:** ``src/lakeflow_framework/config/default/`` — bundled with the package, always available. | **Override:** ``src/local/config/`` — sparse override; deep-merged on top of defaults. -Under ``src/config/default/`` you normally have: +.. _config-resolution-order: + +Resolution order (Strategy B) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The framework resolves every default config file through a three-step process: + +.. list-table:: + :header-rows: 1 + :widths: 5 30 65 + + * - Step + - Source + - Details + * - 1 + - **Workspace Files** (explicit) + - ``{framework_path}/lakeflow_framework/config/default/`` + when ``framework.sourcePath`` is set and the file exists in workspace files. + Workspace Files are checked first so that the workspace files copy always wins — explicit + beats implicit. + * - 2 + - **Package data** (fallback) + - ``importlib.resources`` — reads from the installed + ``lakeflow_framework`` wheel or the ``src/lakeflow_framework/`` tree + when ``src/`` is on ``sys.path`` (flat deploy). Used when the file is + not found via Step 1, or when ``framework.sourcePath`` is not set. + * - 3 + - **``src/local/config/`` custom override** + - Deep-merged on top of the result from Step 1 or 2, when + ``framework.sourcePath`` is set and the sparse fragment exists. + Only the keys present in the custom override are changed. + +Under ``src/lakeflow_framework/config/default/`` you normally have: * exactly one global file: ``global.json``, ``global.yaml``, or ``global.yml`` * a ``dataflow_spec_mapping/`` directory (see :doc:`feature_versioning_dataflow_spec`) @@ -66,12 +110,12 @@ Local override (``src/local/config/``) Place sparse JSON/YAML files in ``src/local/config/`` to override individual keys without copying the entire default file. The framework **deep-merges** the -overlay on top of the defaults at runtime: +custom override on top of the defaults at runtime: -* Dict values are merged recursively — only the keys present in the overlay +* Dict values are merged recursively — only the keys present in the custom override are changed. * Non-dict values and lists are replaced wholesale. -* Keys not present in the overlay retain their default values. +* Keys not present in the custom override retain their default values. Example — change one global setting without touching the rest of ``global.json``: diff --git a/docs/source/feature_mandatory_table_properties.rst b/docs/source/feature_mandatory_table_properties.rst index d6ec046..a7256f2 100644 --- a/docs/source/feature_mandatory_table_properties.rst +++ b/docs/source/feature_mandatory_table_properties.rst @@ -17,7 +17,7 @@ Configuration ------------- | **Scope: Global** -| Mandatory table properties are defined in the global configuration file located at ``src/config/default/global.json|yaml`` under the ``mandatory_table_properties`` section. +| Mandatory table properties are defined in the global configuration file located at ``src/lakeflow_framework/config/default/global.json|yaml`` under the ``mandatory_table_properties`` section. Configuration Schema ------------------ diff --git a/docs/source/feature_spark_configuration.rst b/docs/source/feature_spark_configuration.rst index b57ed57..52276c9 100644 --- a/docs/source/feature_spark_configuration.rst +++ b/docs/source/feature_spark_configuration.rst @@ -17,7 +17,7 @@ Configuration ------------- | **Scope: Global** -| In the Framework bundle, Spark configurations are defined in the global configuration file located at: ``src/config/default/global.json|yaml`` under the ``spark_config`` section. +| In the Framework bundle, Spark configurations are defined in the global configuration file located at: ``src/lakeflow_framework/config/default/global.json|yaml`` under the ``spark_config`` section. | **Scope: Bundle** | In a Pipeline bundle, Spark configurations are defined in the global configuration file located at: ``src/pipeline_configs/global.json|yaml`` under the ``spark_config`` section. diff --git a/docs/source/feature_spec_format.rst b/docs/source/feature_spec_format.rst index 831bcb3..c9f96c3 100644 --- a/docs/source/feature_spec_format.rst +++ b/docs/source/feature_spec_format.rst @@ -40,7 +40,7 @@ Framework-Level Configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | **Scope: Framework** -| The global specification format is defined in the Framework's global configuration file: ``src/config/default/global.json|yaml`` +| The global specification format is defined in the Framework's global configuration file: ``src/lakeflow_framework/config/default/global.json|yaml`` .. tabs:: @@ -235,7 +235,7 @@ Configuration Examples Example 1: Framework Enforces JSON Format ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Framework Configuration** (``src/config/default/global.json|yaml``): +**Framework Configuration** (``src/lakeflow_framework/config/default/global.json|yaml``): .. tabs:: @@ -263,7 +263,7 @@ Example 1: Framework Enforces JSON Format Example 2: Framework Allows Format Flexibility ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Framework Configuration** (``src/config/default/global.json|yaml``): +**Framework Configuration** (``src/lakeflow_framework/config/default/global.json|yaml``): .. tabs:: @@ -312,7 +312,7 @@ Example 2: Framework Allows Format Flexibility Example 3: Framework Defaults to YAML ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -**Framework Configuration** (``src/config/default/global.json|yaml``): +**Framework Configuration** (``src/lakeflow_framework/config/default/global.json|yaml``): .. tabs:: diff --git a/docs/source/feature_substitutions.rst b/docs/source/feature_substitutions.rst index 9859415..bd10dd4 100644 --- a/docs/source/feature_substitutions.rst +++ b/docs/source/feature_substitutions.rst @@ -35,8 +35,8 @@ Configuration ------------- | **Scope: Global** -| In the Framework bundle, substitutions are defined in the following configuration file: ``src/config/default/_substitutions.json|yaml`` -| e.g. ``src/config/default/dev_substitutions.json|yaml`` +| In the Framework bundle, substitutions are defined in the following configuration file: ``src/lakeflow_framework/config/default/_substitutions.json|yaml`` +| e.g. ``src/lakeflow_framework/config/default/dev_substitutions.json|yaml`` | **Scope: Pipeline** | In a Pipeline bundle, substitutions are defined in the following configuration file: ``src/pipeline_configs/_substitutions.json|yaml`` diff --git a/docs/source/feature_table_migration.rst b/docs/source/feature_table_migration.rst index 5590396..b51c783 100644 --- a/docs/source/feature_table_migration.rst +++ b/docs/source/feature_table_migration.rst @@ -40,7 +40,7 @@ Set as an attribute when creating your Data Flow Spec, refer to the :doc:`datafl **Required Global Configuration** -When table migration is enabled, you must specify the volume path for checkpoint state storage in your ``global.json|yaml`` configuration file at either the framework level (``src/config/default/global.json|yaml``) or pipeline bundle level (``src/pipeline_configs/global.json|yaml``): +When table migration is enabled, you must specify the volume path for checkpoint state storage in your ``global.json|yaml`` configuration file at either the framework level (``src/lakeflow_framework/config/default/global.json|yaml``) or pipeline bundle level (``src/pipeline_configs/global.json|yaml``): .. tabs:: diff --git a/docs/source/feature_validation.rst b/docs/source/feature_validation.rst index 69bdcf7..9a677a2 100644 --- a/docs/source/feature_validation.rst +++ b/docs/source/feature_validation.rst @@ -95,7 +95,7 @@ You can do this in the pipeline resource YAML file or via the Databricks UI in t Validation via CI/CD -------------------- -You can validate Data Flow Specification ``*_main.json`` files in CI without running a Spark Declarative Pipeline. The framework repository includes a ``scripts/validate_dataflows.py`` script: run it from a checkout of Lakeflow Framework repo so it can load JSON Schemas (under ``src/schemas/``) and optional dataflow spec version mappings, then point it at a directory or single ``*_main.json`` in your pipeline bundle or monorepo. +You can validate Data Flow Specification ``*_main.json`` files in CI without running a Spark Declarative Pipeline. The framework repository includes a ``scripts/validate_dataflows.py`` script: run it from a checkout of Lakeflow Framework repo so it can load JSON Schemas (under ``src/lakeflow_framework/schemas/``) and optional dataflow spec version mappings, then point it at a directory or single ``*_main.json`` in your pipeline bundle or monorepo. Validating in GitHub Actions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/feature_versioning_dataflow_spec.rst b/docs/source/feature_versioning_dataflow_spec.rst index 0876948..1e4e85f 100644 --- a/docs/source/feature_versioning_dataflow_spec.rst +++ b/docs/source/feature_versioning_dataflow_spec.rst @@ -27,7 +27,7 @@ The versioning system applies transformation mappings that can rename fields, mo Mapping File Structure ---------------------- DataFlow specification mappings are stored in version-specific directories under: -``src/config/default/dataflow_spec_mapping/[version]/dataflow_spec_mapping.json`` +``src/lakeflow_framework/config/default/dataflow_spec_mapping/[version]/dataflow_spec_mapping.json`` Each mapping file contains transformation rules organized by: @@ -242,7 +242,7 @@ Best Practices Version Management ------------------ 1. Mapping versions should follow semantic versioning (MAJOR.MINOR.PATCH) -2. Each mapping version should be stored in its own directory under ``src/config/default/dataflow_spec_mapping/`` +2. Each mapping version should be stored in its own directory under ``src/lakeflow_framework/config/default/dataflow_spec_mapping/`` 3. Maintain documentation of what each version transforms and why 4. Keep mapping files immutable once deployed to ensure consistency 5. Create new mapping versions rather than modifying existing ones diff --git a/docs/source/feature_versioning_framework.rst b/docs/source/feature_versioning_framework.rst index 1c5c0a4..3266bca 100644 --- a/docs/source/feature_versioning_framework.rst +++ b/docs/source/feature_versioning_framework.rst @@ -74,6 +74,16 @@ Set this path using the ``BUNDLE_VAR_framework_source_path`` environment variabl The ``framework_source_path`` setting applies to all pipelines in the bundle. While individual pipeline versions can be modified directly in the Databricks workspace, this is not recommended for production environments. +.. note:: + **Wheel-deploy mode (v0.16.0+)** + + When using ``pip install lakeflow-framework``, ``framework.sourcePath`` is + optional. Default configs and schemas are bundled inside the wheel and + loaded via ``importlib.resources``. You only need to set + ``framework.sourcePath`` if you want to apply ``src/local/config/`` sparse + overrides on top of the bundled defaults. See :doc:`deploy_wheel` for + details. + Best Practices -------------- 1. Default Version diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index 0e1c17a..b08428e 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -21,17 +21,9 @@ Core Concepts * **Key Design** * DABS native - * No artifacts or wheel files - * Minimized third-party dependencies + * Available as a pip package (``pip install lakeflow-framework``) from v0.16.0 * No control tables * Extensible * Flexible deployment bundles -* **OO & Best Practices** - - * Encapsulation - * Abstraction & Inheritance - * Loosely Coupled - * Separation of Concerns & Single Responsibility - Please refer to the :doc:`concepts` section for an overview of the different components of the framework. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9a0d2d3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "lakeflow-framework" +dynamic = ["version"] +description = "A metadata-driven framework for building Databricks Spark Declarative Pipelines" +readme = "README.md" +license = { file = "LICENSE.md" } +requires-python = ">=3.10" +dependencies = [ + "jsonschema>=4.0", +] + +[project.optional-dependencies] +contrib = [] +all = ["lakeflow-framework[contrib]"] + +[tool.setuptools.dynamic] +version = { file = "VERSION" } + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +lakeflow_framework = [ + "config/default/**/*", + "schemas/**/*", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index a95cf4d..c2ffcff 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,6 +12,7 @@ pytest==9.0.3 ## Dependencies for building wheel files setuptools==82.0.1 wheel==0.47.0 +build==1.2.2 # The following packages are only required for building documentation and are not required at runtime -r requirements-docs.txt diff --git a/scripts/validate_dataflows.py b/scripts/validate_dataflows.py index 5a86130..3cf2b47 100755 --- a/scripts/validate_dataflows.py +++ b/scripts/validate_dataflows.py @@ -59,7 +59,7 @@ def find_project_root() -> Path: # Go up to find the project root while current != current.parent: - if (current / "src" / "schemas" / "main.json").exists(): + if (current / "src" / "lakeflow_framework" / "schemas" / "main.json").exists(): return current current = current.parent @@ -113,7 +113,7 @@ def detect_spec_form(data: Dict) -> str: def get_schema_path(project_root: Path, form: str) -> Path: """Return the schema path for a given spec form.""" - schemas = project_root / "src" / "schemas" + schemas = project_root / "src" / "lakeflow_framework" / "schemas" if form == SPEC_FORM_TEMPLATE: return schemas / "spec_template.json" return schemas / "main.json" diff --git a/src/__init__.py b/src/__init__.py index e62ec8b..3f3b150 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,11 +1,13 @@ """ -Lakeflow Framework - A framework for building Delta Live Tables pipelines. +Compat shim — src/ package init. -This package provides tools and utilities for creating, managing, and orchestrating -Delta Live Tables (DLT) pipelines in Databricks. +Delegates all public symbols to ``lakeflow_framework``. +This file is kept so that any code that does ``import src`` or uses ``src`` +as a package (unusual) continues to resolve correctly while ``src/`` is on +``sys.path`` and the canonical import path is ``lakeflow_framework.*``. """ -from .dataflow import ( +from lakeflow_framework import ( # noqa: F401 DataFlow, FlowConfig, FlowGroup, @@ -15,12 +17,11 @@ TargetType, SinkType, View, - StagingTable -) -from .dataflow.dataflow_spec import DataflowSpec -from .dataflow_spec_builder import DataflowSpecBuilder, DataQualityExpectationBuilder -from .dlt_pipeline_builder import DLTPipelineBuilder -from .pipeline_config import ( + StagingTable, + DataflowSpec, + DataflowSpecBuilder, + DataQualityExpectationBuilder, + DLTPipelineBuilder, get_spark, get_logger, get_operational_metadata_schema, @@ -31,37 +32,11 @@ initialize_operational_metadata_schema, initialize_pipeline_details, initialize_substitution_manager, - initialize_mandatory_table_properties + initialize_mandatory_table_properties, + SecretsManager, + SubstitutionManager, + SystemColumns, + MetaDataColumnDefs, + utility, + __version__, ) -from .secrets_manager import SecretsManager -from .substitution_manager import SubstitutionManager -from .constants import SystemColumns, MetaDataColumnDefs -from . import utility - -__all__ = [ - 'DLTPipelineBuilder', - 'DataflowSpec', - 'DataflowSpecBuilder', - 'DataQualityExpectationBuilder', - - # Main classes - 'DataFlow', 'FlowConfig', 'FlowGroup', 'BaseFlow', 'StagingTable', 'View', - - # Managers - 'SecretsManager', 'SubstitutionManager', 'TableMigrationManager', 'QuarantineManager', - - # Types and Enums - 'TargetType', 'SinkType', - - # Constants - 'SystemColumns', 'MetaDataColumnDefs', - - # Utilities - 'utility', - - # Pipeline Config - 'get_spark', 'get_logger', 'get_operational_metadata_schema', 'get_pipeline_details', 'get_substitution_manager', 'get_mandatory_table_properties', - 'initialize_core', 'initialize_operational_metadata_schema', 'initialize_pipeline_details', 'initialize_substitution_manager', 'initialize_mandatory_table_properties' -] - -__version__ = '0.4.0' diff --git a/src/bundle_loader.py b/src/bundle_loader.py index 23e4ad4..1450656 100644 --- a/src/bundle_loader.py +++ b/src/bundle_loader.py @@ -1,168 +1,7 @@ -""" -Bundle sys.path registration and init script execution. - -Framework bundle (framework.sourcePath → src/): - local/libraries/ — org-wide cluster-install artefacts + loose .py / packages - local/python/ — org-wide Data Flow Spec-referenced Python - extensions/ — DEPRECATED (v0.13.0): flat top-level .py files; removed in v1.0.0. - Migrate importable modules to local/python/. - -Pipeline bundle (bundle.sourcePath → src/): - src/libraries/ — bundle-local cluster-install artefacts + loose .py / packages - src/python/ — bundle Data Flow Spec-referenced Python - extensions/ — DEPRECATED (v0.13.0): flat top-level .py files; removed in v1.0.0. - Migrate importable modules to src/python/. - -Init scripts executed for framework then bundle per phase: - Framework: local/init/pre/ — before SDP table/view declarations inside initialize_pipeline() - local/init/post/ — after all SDP declarations - Bundle: src/init/pre/ - src/init/post/ - -Within each directory, scripts run in sorted filename order; files whose names start -with '_' are skipped. A script that raises an exception fails the pipeline. -""" - -from __future__ import annotations - -import os -import runpy -import sys -import warnings -from typing import Literal - -from constants import FrameworkPaths, PipelineBundlePaths - -Phase = Literal["pre", "post"] - -_FRAMEWORK_PHASE_PATH: dict[str, str] = { - "pre": FrameworkPaths.LOCAL_INIT_PRE_PATH, - "post": FrameworkPaths.LOCAL_INIT_POST_PATH, -} - -_BUNDLE_PHASE_PATH: dict[str, str] = { - "pre": PipelineBundlePaths.INIT_PRE_PATH, - "post": PipelineBundlePaths.INIT_POST_PATH, -} - -_PHASE_LABEL: dict[str, str] = { - "pre": "pre-init", - "post": "post-init", -} - - -def _add_if_dir(path: str, label: str, logger=None) -> None: - """Insert path at the front of sys.path if it is a directory and not already present.""" - if os.path.isdir(path): - if path not in sys.path: - sys.path.insert(0, path) - if logger is not None: - logger.info("Registered %s on sys.path: %s", label, path) - - -def _has_top_level_py(directory: str) -> bool: - """Return True if directory exists and contains at least one top-level .py file.""" - if not os.path.isdir(directory): - return False - return any( - os.path.isfile(os.path.join(directory, name)) and name.endswith(".py") - for name in os.listdir(directory) - ) - - -def _warn_legacy_extensions(ext_path: str, python_path: str, level: str, logger=None) -> None: - """Register legacy extensions/ on sys.path and emit a deprecation warning. - - extensions/ is always registered when it contains top-level .py files, - regardless of whether src/python/ (or local/python/) also exists. - Deprecated means warn-and-still-work until removal in v1.0.0. - """ - if not _has_top_level_py(ext_path): - return - _add_if_dir(ext_path, f"{level} extensions (legacy)", logger) - msg = ( - f"[{level}] Top-level .py files under {ext_path!r} are deprecated. " - "Move importable modules to " - + ("local/python/ (framework) or " if level == "framework" else "") - + "src/python/ (bundle). " - "Flat extensions/ sys.path support will be removed in v1.0.0." - ) - if logger is not None: - logger.warning(msg) - warnings.warn(msg, DeprecationWarning, stacklevel=3) - - -def register_bundle_sys_paths( - framework_path: str, - bundle_path: str, - logger=None, -) -> None: - """ - Register sys.path roots for framework (local/) then pipeline bundle (src/). - - Framework bundle uses local/ paths (local/libraries/, local/python/). - Pipeline bundle uses src/ paths (src/libraries/, src/python/). - Legacy extensions/ is deprecated in both contexts. - - ``logger`` is optional. When omitted, paths are registered silently — useful - for early bootstrap before the real logger is available. - """ - # Framework bundle: custom code lives under local/ only - fw_libraries = os.path.normpath(os.path.join(framework_path, FrameworkPaths.LOCAL_LIBRARIES_PATH)) - fw_python = os.path.normpath(os.path.join(framework_path, FrameworkPaths.LOCAL_PYTHON_PATH)) - fw_legacy = os.path.normpath(os.path.join(framework_path, FrameworkPaths.EXTENSIONS_PATH)) - _add_if_dir(fw_libraries, "framework local/libraries", logger) - _add_if_dir(fw_python, "framework local/python", logger) - _warn_legacy_extensions(fw_legacy, fw_python, "framework", logger) - - # Pipeline bundle: custom code lives under src/ - pipe_libraries = os.path.normpath(os.path.join(bundle_path, PipelineBundlePaths.LIBRARIES_PATH)) - pipe_python = os.path.normpath(os.path.join(bundle_path, PipelineBundlePaths.PYTHON_PATH)) - pipe_legacy = os.path.normpath(os.path.join(bundle_path, PipelineBundlePaths.EXTENSIONS_PATH)) - _add_if_dir(pipe_libraries, "bundle src/libraries", logger) - _add_if_dir(pipe_python, "bundle src/python", logger) - _warn_legacy_extensions(pipe_legacy, pipe_python, "bundle", logger) - - -def run_init_scripts( - framework_path: str, - bundle_path: str, - phase: Phase, - logger, -) -> None: - """ - Execute all .py lifecycle scripts under the init// directory for framework then bundle. - - Framework bundle uses local/init//; pipeline bundle uses src/init//. - Files are sorted by filename; files starting with '_' are skipped. - Each file is run with runpy.run_path(..., run_name='__main__'). - A script that raises an exception fails the pipeline. - - Args: - framework_path: Root of the framework bundle (framework.sourcePath). - bundle_path: Root of the pipeline bundle (bundle.sourcePath). - phase: "pre" (before SDP declarations) or "post" (after). - logger: Framework logger instance. - """ - if phase not in _FRAMEWORK_PHASE_PATH: - raise ValueError(f"Invalid init phase: {phase!r}. Expected one of {list(_FRAMEWORK_PHASE_PATH)}.") - - label = _PHASE_LABEL[phase] - - for base, level, phase_paths in ( - (framework_path, "framework", _FRAMEWORK_PHASE_PATH), - (bundle_path, "bundle", _BUNDLE_PHASE_PATH), - ): - folder = os.path.normpath(os.path.join(base, phase_paths[phase])) - if not os.path.isdir(folder): - continue - scripts = sorted( - name for name in os.listdir(folder) - if name.endswith(".py") and not name.startswith("_") - ) - for filename in scripts: - path = os.path.join(folder, filename) - if not os.path.isfile(path): - continue - logger.info("Running %s %s script: %s", level, label, path) - runpy.run_path(path, run_name="__main__") +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.bundle_loader import ... +from lakeflow_framework.bundle_loader import ( # noqa: F401 + Phase, + register_bundle_sys_paths, + run_init_scripts, +) diff --git a/src/config_resolver.py b/src/config_resolver.py index 5e5ba21..89d3dd4 100644 --- a/src/config_resolver.py +++ b/src/config_resolver.py @@ -1,150 +1,8 @@ -"""Framework configuration resolver. - -Provides the primary API for loading framework config files with -``src/local/config/`` sparse-overlay support, plus the deprecated -``resolve_framework_config_path`` shim kept for backward compatibility -until v1.0.0. -""" -import os -import warnings -from typing import Dict, Sequence, Union - -from constants import FrameworkPaths - - -def load_framework_config( - name: Union[str, Sequence[str]], - framework_path: str, - config_path: str = FrameworkPaths.CONFIG_PATH, - fail_on_not_exists: bool = True, -) -> Dict: - """Load a config file from *config_path* with ``src/local/config/`` deep-merge overlay. - - The caller is responsible for determining the active base path (override or default) - and passing it via ``config_path``. This function contains no override-detection - logic — it only loads from the given path and applies the ``src/local/config/`` - sparse overlay on top. - - When *name* is a sequence (e.g. ``FrameworkPaths.GLOBAL_CONFIG``), the function - resolves to the single matching file within ``config_path``, raising ``ValueError`` - if more than one match is found. - - Args: - name: Config file name (e.g. ``"logger.json"``) **or** a sequence of alternative - filenames (e.g. ``FrameworkPaths.GLOBAL_CONFIG``). - framework_path: Absolute path to the framework bundle's ``src/`` directory. - config_path: Relative path segment for the base config directory, e.g. - ``FrameworkPaths.CONFIG_PATH`` (default) or - ``FrameworkPaths.CONFIG_OVERRIDE_PATH`` when the caller has - determined that the deprecated override directory is active. - fail_on_not_exists: When ``True`` (default) raise ``FileNotFoundError`` if the - base file is absent. When ``False`` return ``{}``. - """ - from utility import load_config_file_auto, deep_merge - - base_dir = os.path.join(framework_path, config_path) - local_dir = os.path.join(framework_path, FrameworkPaths.LOCAL_CONFIG_PATH) - - if not isinstance(name, str): - names = name - matches = [n for n in names if os.path.exists(os.path.join(base_dir, n))] - if len(matches) > 1: - raise ValueError( - f"Multiple config files found in {config_path}. " - f"Only one is allowed: {[os.path.join(base_dir, n) for n in matches]}" - ) - if not matches: - if fail_on_not_exists: - raise FileNotFoundError( - f"Config file not found. Expected one of {list(names)} under {base_dir}" - ) - return {} - name = matches[0] - - base_path = os.path.join(base_dir, name) - local_path = os.path.join(local_dir, name) - - defaults = load_config_file_auto(base_path, fail_on_not_exists=fail_on_not_exists) or {} - - if os.path.exists(local_path): - overlay = load_config_file_auto(local_path, fail_on_not_exists=False) or {} - if overlay: - return deep_merge(defaults, overlay) - - return defaults - - -def resolve_framework_config_dir(subdir: str, framework_path: str) -> str: - """Resolve a config subdirectory path, checking ``src/local/config/`` first. - - If ``src/local/config//`` exists it is returned; otherwise falls - back to ``src/config/default//``. Useful for directory-based config - such as ``dataflow_spec_mapping/``. - - Args: - subdir: Subdirectory name, e.g. ``"dataflow_spec_mapping"``. - framework_path: Absolute path to the framework bundle's ``src/`` directory. - """ - local_path = os.path.join(framework_path, FrameworkPaths.LOCAL_CONFIG_PATH, subdir) - if os.path.isdir(local_path): - return local_path - return os.path.join(framework_path, FrameworkPaths.CONFIG_PATH, subdir) - - -def _has_visible_children(directory: str) -> bool: - """Return True if *directory* exists and contains at least one non-hidden child.""" - if not os.path.isdir(directory): - return False - try: - names = os.listdir(directory) - except OSError: - return False - return any(not n.startswith(".") for n in names) - - -def resolve_framework_config_path(framework_path: str) -> str: - """ - DEPRECATED (v0.13.0): Use ``load_framework_config()`` or ``resolve_framework_config_dir()`` - instead. Removed in v1.0.0. - - Returns FrameworkPaths.CONFIG_OVERRIDE_PATH when the override directory has at least one - non-hidden child and mirrors the required layout; otherwise FrameworkPaths.CONFIG_PATH. - - Raises: - FileNotFoundError: If neither default nor override config roots contain valid files, - or if the override root is active but incomplete. - """ - config_dir = os.path.join(framework_path, FrameworkPaths.CONFIG_PATH) - override_dir = os.path.join(framework_path, FrameworkPaths.CONFIG_OVERRIDE_PATH) - if not _has_visible_children(override_dir): - if not _has_visible_children(config_dir): - raise FileNotFoundError( - f"No valid files found under {FrameworkPaths.CONFIG_PATH} or " - f"{FrameworkPaths.CONFIG_OVERRIDE_PATH} in the framework bundle " - f"({framework_path!s}). Please add framework configuration under " - f"{FrameworkPaths.CONFIG_PATH} (for example a global config file, " - f"the {FrameworkPaths.DATAFLOW_SPEC_MAPPING} directory, and related files)." - ) - return FrameworkPaths.CONFIG_PATH - - warnings.warn( - f"{FrameworkPaths.CONFIG_OVERRIDE_PATH} is deprecated (v0.13.0) and will be removed " - f"in v1.0.0. Migrate your overrides to {FrameworkPaths.LOCAL_CONFIG_PATH} — only the " - "keys you want to change are needed (sparse files are supported). " - "See the framework configuration documentation for migration steps.", - DeprecationWarning, - stacklevel=2, - ) - - mapping_dir = os.path.join(override_dir, FrameworkPaths.DATAFLOW_SPEC_MAPPING) - global_paths = [ - os.path.join(override_dir, name) for name in FrameworkPaths.GLOBAL_CONFIG - ] - if not os.path.isdir(mapping_dir) or not any(os.path.isfile(p) for p in global_paths): - raise FileNotFoundError( - f"Using {FrameworkPaths.CONFIG_OVERRIDE_PATH} requires both a global config file " - f"({' or '.join(FrameworkPaths.GLOBAL_CONFIG)}) and the " - f"{FrameworkPaths.DATAFLOW_SPEC_MAPPING} directory under that path. " - f"Copy the full {FrameworkPaths.CONFIG_PATH} tree into {FrameworkPaths.CONFIG_OVERRIDE_PATH}." - ) - return FrameworkPaths.CONFIG_OVERRIDE_PATH +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.config_resolver import ... +from lakeflow_framework.config_resolver import ( # noqa: F401 + load_framework_default_json, + load_framework_config, + resolve_framework_config_dir, + resolve_framework_config_path, +) diff --git a/src/constants.py b/src/constants.py index f604560..9c20128 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,197 +1,14 @@ -from dataclasses import dataclass -from enum import Enum - -@dataclass(frozen=True) -class FrameworkSettings: - """ - FrameworkSettings is a class that contains constants for the framework settings. - """ - OVERRIDE_MAX_WORKERS_KEY: str = "override_max_workers" - PIPELINE_BUILDER_DISABLE_THREADING_KEY: str = "pipeline_builder_disable_threading" - - -@dataclass(frozen=True) -class DLTPipelineSettingKeys: - """ - DLTPipelineSettingKeys is a class that contains constants for various Pipeline settings keys. - """ - BUNDLE_SOURCE_PATH: str = "bundle.sourcePath" - BUNDLE_TARGET: str = "bundle.target" - FRAMEWORK_SOURCE_PATH: str = "framework.sourcePath" - LOG_LEVEL: str = "logLevel" - LOGICAL_ENV: str = "logicalEnv" - PIPELINE_CATALOG: str = "pipelines.catalog" - PIPELINE_FILE_FILTER: str = "pipeline.fileFilter" - PIPELINE_FILTER_DATA_FLOW_GROUP: str = "pipeline.dataFlowGroupFilter" - PIPELINE_FILTER_DATA_FLOW_ID: str = "pipeline.dataFlowIdFilter" - PIPELINE_FILTER_FLOW_GROUP_ID: str = "pipeline.flowGroupIdFilter" - PIPELINE_FILTER_TARGET_TABLE: str = "pipeline.targetTableFilter" - PIPELINE_ID: str = "pipelines.id" - PIPELINE_IGNORE_VALIDATION_ERRORS: str = "pipeline.ignoreValidationErrors" - PIPELINE_LAYER: str = "pipeline.layer" - PIPELINE_TARGET: str = "pipelines.target" - PIPELINE_SCHEMA: str = "pipelines.schema" - WORKSPACE_HOST: str = "workspace.host" - - -@dataclass(frozen=True) -class FrameworkPaths: - """ - FrameworkPaths is a class that contains constants for various paths and file masks used in the Lakeflow Framework. - - All LOCAL_* paths are relative to ``framework.sourcePath`` (which resolves to the ``src/`` - directory of the framework bundle, e.g. ``.../files/src``). Customer code in the framework - bundle lives exclusively under ``local/`` — there are no top-level ``src/libraries/``, - ``src/python/`` or ``src/init/`` directories in the framework bundle. - - CONFIG_PATH and CONFIG_OVERRIDE_PATH are static path segments (./config/default and - ./config/override). At runtime, which root to use for framework config files should be chosen - using utility.resolve_framework_config_path(framework_path). - - Attributes: - CONFIG_PATH (str): Path to the config/default directory. - CONFIG_OVERRIDE_PATH (str): DEPRECATED (v0.13.0) — use src/local/config/ instead; removed in v1.0.0. - LOCAL_CONFIG_PATH (str): Sparse overlay directory for framework config files. Files here are deep-merged on top of config/default equivalents. - LOCAL_LIBRARIES_PATH (str): Org-wide cluster-install artefacts + loose .py/packages on sys.path. - LOCAL_PYTHON_PATH (str): Org-wide pipeline logic modules called via pythonModule/pythonTransform. - LOCAL_INIT_PRE_PATH (str): Org-wide pre-init lifecycle scripts run before SDP declarations. - LOCAL_INIT_POST_PATH (str): Org-wide post-init lifecycle scripts run after SDP declarations. - EXTENSIONS_PATH (str): DEPRECATED (v0.13.0) — flat extensions/ on sys.path; removed in v1.0.0. Migrate to src/local/python/. - GLOBAL_CONFIG (tuple): Paths to the global configuration files. - GLOBAL_SUBSTITUTIONS (tuple): Paths to the global substitutions files. - GLOBAL_SECRETS (tuple): Paths to the global secrets files. - DATAFLOW_SPEC_MAPPING (str): Directory segment for dataflow spec mapping (under the resolved root). - LOGGER_CONFIG (str): Basename of the pluggable logger configuration file (under the resolved config root). - MAIN_SPEC_SCHEMA_PATH (str): Path to the main specification schema file. - FLOW_GROUP_SPEC_SCHEMA_PATH (str): Path to the flow group specification schema file. - EXPECTATIONS_SPEC_SCHEMA_PATH (str): Path to the expectations specification schema file. - SECRETS_SCHEMA_PATH (str): Path to the secrets specification schema file. - TEMPLATE_DEFINITION_SPEC_SCHEMA_PATH (str): Path to the template definition specification schema file. - TEMPLATE_SPEC_SCHEMA_PATH (str): Path to the template specification schema file. - """ - CONFIG_PATH: str = "./config/default" - CONFIG_OVERRIDE_PATH: str = "./config/override" # DEPRECATED (v0.13.0) — use local/config/; removed v1.0.0 - LOCAL_CONFIG_PATH: str = "./local/config" - LOCAL_LIBRARIES_PATH: str = "./local/libraries" - LOCAL_PYTHON_PATH: str = "./local/python" - LOCAL_INIT_PRE_PATH: str = "./local/init/pre" - LOCAL_INIT_POST_PATH: str = "./local/init/post" - EXTENSIONS_PATH: str = "./extensions" # DEPRECATED (v0.13.0) — migrate to local/python/; removed v1.0.0 - GLOBAL_CONFIG: tuple = ("global.json", "global.yaml", "global.yml") - GLOBAL_SUBSTITUTIONS: tuple = ("_substitutions.json", "_substitutions.yaml", "_substitutions.yml") - GLOBAL_SECRETS: tuple = ("_secrets.json", "_secrets.yaml", "_secrets.yml") - DATAFLOW_SPEC_MAPPING: str = "dataflow_spec_mapping" - LOGGER_CONFIG: str = "logger.json" - REQUIREMENTS_FILE: str = "requirements.txt" - - # Spec schema definitions paths - SPEC_MAPPING_SCHEMA_PATH: str = "./schemas/spec_mapping.json" - MAIN_SPEC_SCHEMA_PATH: str = "./schemas/main.json" - FLOW_GROUP_SPEC_SCHEMA_PATH: str = "./schemas/flow_group.json" - EXPECTATIONS_SPEC_SCHEMA_PATH: str = "./schemas/expectations.json" - SECRETS_SCHEMA_PATH: str = "./schemas/secrets.json" - TEMPLATE_DEFINITION_SPEC_SCHEMA_PATH: str = "./schemas/spec_template_definition.json" - TEMPLATE_SPEC_SCHEMA_PATH: str = "./schemas/spec_template.json" - - -class SupportedSpecFormat(str, Enum): - """Supported specification file formats.""" - JSON = "json" - YAML = "yaml" - - -@dataclass(frozen=True) -class PipelineBundleSuffixesJson: - """ - PipelineBundleSuffixesJson is a class that contains constants for various file suffixes used in the Pipeline Bundles in JSON format. - """ - MAIN_SPEC_FILE_SUFFIX: tuple = ("_main.json") - FLOW_GROUP_FILE_SUFFIX: tuple = ("_flow.json") - EXPECTATIONS_FILE_SUFFIX: tuple = (".json") - SECRETS_FILE_SUFFIX: tuple = ("_secrets.json") - SUBSTITUTIONS_FILE_SUFFIX: tuple = ("_substitutions.json") - - -@dataclass(frozen=True) -class PipelineBundleSuffixesYaml: - """ - PipelineBundleSuffixesYaml is a class that contains constants for various file suffixes used in the Pipeline Bundles in YAML format. - """ - MAIN_SPEC_FILE_SUFFIX: tuple = ("_main.yaml", "_main.yml") - FLOW_GROUP_FILE_SUFFIX: tuple = ("_flow.yaml", "_flow.yml") - EXPECTATIONS_FILE_SUFFIX: tuple = ("_expectations.yaml", "_expectations.yml") - SECRETS_FILE_SUFFIX: tuple = ("_secrets.yaml", "_secrets.yml") - SUBSTITUTIONS_FILE_SUFFIX: tuple = ("_substitutions.yaml", "_substitutions.yml") - - -@dataclass(frozen=True) -class PipelineBundlePaths: - """ - PipelineBundlePaths is a class that contains constants for various paths and file masks - used in the Pipeline Bundles. - - Attributes: - DATAFLOWS_BASE_PATH (str): The base path for dataflows. - DATAFLOW_SPEC_PATH (str): The path for dataflow specifications. - DML_PATH (str): The path for DML (Data Manipulation Language) files. - DQE_PATH (str): The path for data quality expectations. - LIBRARIES_PATH (str): Cluster-install artefacts + loose .py/packages on sys.path (primary). - PYTHON_PATH (str): Data Flow Spec-referenced Python — modules called via pythonModule/pythonTransform. - INIT_PRE_PATH (str): Pre-init lifecycle scripts run before SDP declarations. - INIT_POST_PATH (str): Post-init lifecycle scripts run after SDP declarations. - EXTENSIONS_PATH (str): DEPRECATED (v0.13.0) — flat extensions/ on sys.path; removed in v1.0.0. Migrate to src/python/. - GLOBAL_CONFIG_FILE (tuple): The file names for global configuration files. - PIPELINE_CONFIGS_PATH (str): The path for pipeline configuration files. - LOGGER_CONFIG (str): Basename of the pluggable logger configuration file (under pipeline_configs). - PYTHON_FUNCTION_PATH (str): The path for python functions. - SCHEMA_PATH (str): The path for schema files. - TEMPLATE_PATH (str): Path to the template directory. - """ - DATAFLOWS_BASE_PATH: str = "./dataflows" - DATAFLOW_SPEC_PATH: str = "dataflowspec" - DML_PATH: str = "./dml" - DQE_PATH: str = "./expectations" - LIBRARIES_PATH: str = "./libraries" - PYTHON_PATH: str = "./python" - INIT_PRE_PATH: str = "./init/pre" - INIT_POST_PATH: str = "./init/post" - EXTENSIONS_PATH: str = "./extensions" # DEPRECATED (v0.13.0) — migrate to python/; removed v1.0.0 - GLOBAL_CONFIG_FILE: tuple = ("./global.json", "./global.yaml", "./global.yml") - PIPELINE_CONFIGS_PATH: str = "./pipeline_configs" - LOGGER_CONFIG: str = "logger.json" - PYTHON_FUNCTION_PATH: str = "./python_functions" - SCHEMA_PATH: str = "./schemas" - TEMPLATE_PATH: str = "./templates" - REQUIREMENTS_FILE: str = "requirements.txt" - - -class SystemColumns: - """ - SystemColumns is a container for constants related to SDP and Framework system columns. - - Classes: - CDFColumns (Enum): Contains constants for Change Data Feed (CDF) system columns. - SCD2Columns (Enum): Contains constants for Slowly Changing Dimension Type 2 (SCD2) system columns. - """ - class CDFColumns(Enum): - """Change Data Feed system columns.""" - CDF_CHANGE_TYPE = "_change_type" - CDF_COMMIT_VERSION = "_commit_version" - CDF_COMMIT_TIMESTAMP = "_commit_timestamp" - - class SCD2Columns(Enum): - """SCD2 system columns.""" - SCD2_START_AT = "__START_AT" - SCD2_END_AT = "__END_AT" - -class MetaDataColumnDefs: - """MetaDataColumnDefs is a class that contains constants for all Framework metadata columns.""" - - QUARANTINE_FLAG = { - "name": "is_quarantined", - "type": "boolean", - "nullable": True, - "metadata": {} - } - -FILE_METADATA_COLUMN = "_metadata" \ No newline at end of file +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.constants import ... +from lakeflow_framework.constants import ( # noqa: F401 + FrameworkSettings, + DLTPipelineSettingKeys, + FrameworkPaths, + SupportedSpecFormat, + PipelineBundleSuffixesJson, + PipelineBundleSuffixesYaml, + PipelineBundlePaths, + SystemColumns, + MetaDataColumnDefs, + FILE_METADATA_COLUMN, +) diff --git a/src/dataflow/cdc_snapshot.py b/src/dataflow/cdc_snapshot.py index 6a59234..6e334d9 100644 --- a/src/dataflow/cdc_snapshot.py +++ b/src/dataflow/cdc_snapshot.py @@ -306,7 +306,7 @@ def _next_snapshot_and_version(self, latest_snapshot_version, dataflow_config: D return (df, version_info.raw_value) except Exception as e: - self.logger.error(f"CDC Snapshot: Error processing snapshots: {e}") + self.logger.error(f"CDC Snapshot: Error processing snapshots: {e}", exc_info=True) raise @@ -318,27 +318,92 @@ def _get_available_versions(self, latest_snapshot_version: Optional[Union[int, d return self._get_available_table_versions(latest_snapshot_version) else: raise ValueError(f"Unsupported source type: {self.sourceType}") + + def _list_files(self, path: str, recursive: bool = True) -> List: + """List files in a directory, attempting dbutils.fs.ls() first. - def _list_files(self, path, recursive=True): - """List files in a directory, with optional recursive file lookup. + Falls back to Spark binaryFile if dbutils is unavailable (e.g., blocked + by Serverless Restricted Access / SEG Py4JSecurityException). Args: - path: Directory path to list files from - recursive: If True, list files recursively. If False, list only files in the immediate directory. + path: Directory path to list files from. + recursive: If True, list files recursively. Returns: - List of file objects from dbutils.fs.ls() + List of objects with a .path attribute per discovered file. """ - dbutils = pipeline_config.get_dbutils() - all_files = [] + try: + dbutils = pipeline_config.get_dbutils() + all_files = [] + for f in dbutils.fs.ls(path): + all_files.append(f) + # Recursively list files in subdirectories unless the path ends with .parquet + if recursive and f.isDir() and not f.path.rstrip("/").endswith(".parquet"): + all_files.extend(self._list_files(f.path, recursive=True)) + return all_files + except Exception as e: + self.logger.warning( + f"CDC Snapshot: dbutils.fs.ls() failed at '{path}' " + f"({type(e).__name__}: {e}). " + f"Falling back to Spark binaryFile for file discovery." + ) - for f in dbutils.fs().ls(path): - all_files.append(f) + # Fallback: Spark binaryFile read is SEG-compatible and needs no dbutils. + self.logger.debug("CDC Snapshot: Falling back to Spark binaryFile for file discovery") + + class _FileInfo: + __slots__ = ("path", "name", "size", "modificationTime") - if recursive and f.isDir(): - all_files.extend(self._list_files(f.path, recursive=True)) + # binaryFile returns full URIs (e.g. dbfs:/Volumes/...). Normalise to + # match the caller's path scheme so split-by-segment indices stay aligned. + _prefix = path.startswith("dbfs:") + + def __init__(self, row) -> None: + raw = row.path + if self._prefix: + self.path = raw if raw.startswith("dbfs:") else f"dbfs:{raw}" + else: + self.path = raw[5:] if raw.startswith("dbfs:") else raw + self.name = self.path.rstrip("/").rsplit("/", 1)[-1] + self.size = row.length + self.modificationTime = row.modificationTime + + spark = pipeline_config.get_spark() + rows = ( + spark.read + .format("binaryFile") + .option("recursiveFileLookup", str(recursive).lower()) + .load(path) + .select("path", "modificationTime", "length") + .collect() + ) + + # binaryFile only returns leaf files, not directories. For parquet "files" that + # are actually directories (e.g. customer.parquet/part-00000.parquet), truncate + # the path at the .parquet directory level and deduplicate so each snapshot is + # counted once. + def _truncate_at_parquet_dir(p: str) -> str: + segments = p.split("/") + for i, seg in enumerate(segments[:-1]): # skip the last segment + if seg.endswith(".parquet"): + return "/".join(segments[: i + 1]) + return p + + seen_paths = set() + files: List[_FileInfo] = [] + for row in rows: + fi = _FileInfo(row) + fi.path = _truncate_at_parquet_dir(fi.path) + fi.name = fi.path.rstrip("/").rsplit("/", 1)[-1] + if fi.path not in seen_paths: + seen_paths.add(fi.path) + files.append(fi) + + self.logger.debug( + f"CDC Snapshot: Spark binaryFile fallback listed {len(files)} file(s) under '{path}'" + ) - return all_files + return files def _path_to_regex_pattern(self, path: str) -> str: """Convert path to normalized regex pattern with named groups. diff --git a/src/dlt_pipeline.ipynb b/src/dlt_pipeline.ipynb index 60c16eb..62ff2c6 100644 --- a/src/dlt_pipeline.ipynb +++ b/src/dlt_pipeline.ipynb @@ -22,7 +22,7 @@ "metadata": {}, "outputs": [], "source": [ - "from constants import DLTPipelineSettingKeys\n", + "from lakeflow_framework.constants import DLTPipelineSettingKeys\n", "\n", "def get_required_config(key: str, description: str) -> str:\n", " \"\"\"Get a required config value from spark.conf or raise an error.\"\"\"\n", @@ -51,12 +51,12 @@ "source": [ "import sys\n", "\n", - "# Append the framework path to the system path\n", + "# Append the framework path to the system path (flat-deploy compat)\n", "if framework_source_path not in sys.path:\n", " sys.path.append(framework_source_path)\n", "\n", "# Import and initialize the Spark Declarative Pipeline builder\n", - "from dlt_pipeline_builder import DLTPipelineBuilder\n", + "from lakeflow_framework.dlt_pipeline_builder import DLTPipelineBuilder\n", "DLTPipelineBuilder(spark, dbutils).initialize_pipeline()" ] } @@ -83,4 +83,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file diff --git a/src/dlt_pipeline_builder.py b/src/dlt_pipeline_builder.py index 6334462..dea15d7 100644 --- a/src/dlt_pipeline_builder.py +++ b/src/dlt_pipeline_builder.py @@ -1,464 +1,3 @@ -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timezone -import json -import os - -from pyspark import pipelines as dp -from pyspark.dbutils import DBUtils -import pyspark.sql.types as T -from pyspark.sql import SparkSession -from typing import Dict, Any - -from config_resolver import load_framework_config, resolve_framework_config_path -from constants import ( - FrameworkPaths, FrameworkSettings, PipelineBundlePaths, DLTPipelineSettingKeys, SupportedSpecFormat -) -from dataflow import DataFlow -from dataflow_spec_builder import DataflowSpecBuilder -from bundle_loader import register_bundle_sys_paths, run_init_scripts -from pipeline_details import PipelineDetails -from secrets_manager import SecretsManager -from substitution_manager import SubstitutionManager - -import logger as pipeline_logger -import pipeline_config -import utility - - -class DLTPipelineBuilder: - """ - Initializes a dataflow in a Spark Declarative Pipeline based on the pipeline configuration and dataflow specifications. - - Args: - spark (SparkSession): The Spark session to use for the pipeline. - dbutils (DBUtils): The DBUtils to use for the pipeline. - - Attributes: - spark (SparkSession): The Spark session to use for the pipeline. - dbutils (DBUtils): The DBUtils to use for the pipeline. - - logger (Logger): The logger to use for the pipeline. - context (NotebookContext): The notebook context to use for the pipeline. - token (str): The token to use for the pipeline. - - pipeline_config (Dict[str, Any]): The pipeline configuration to use for the pipeline. - framework_path (str): The path to the framework to use for the pipeline. - bundle_path (str): The path to the bundle to use for the pipeline. - dataflow_path (str): The path to the dataflow to use for the pipeline. - workspace_host (str): The host to use for the pipeline. - - dataflow_specs (List[DataflowSpec]): The dataflow specifications to use for the pipeline. - dataflow_spec_filters (Dict[str, Any]): The filters to use for the dataflow specifications. - - pipeline_details (PipelineDetails): The pipeline details to use for the pipeline. - mandatory_table_properties (Dict[str, Any]): The mandatory table properties to use for the pipeline. - operational_metadata_schema (StructType): The operational metadata schema to use for the pipeline. - substitution_manager (SubstitutionManager): The substitution manager to use for the pipeline. - - Methods: - initialize_pipeline(): Initializes a dataflow in a Spark Declarative Pipeline. - """ - - MANDATORY_CONFIG_PARAMS = [ - DLTPipelineSettingKeys.BUNDLE_SOURCE_PATH, - DLTPipelineSettingKeys.FRAMEWORK_SOURCE_PATH, - DLTPipelineSettingKeys.WORKSPACE_HOST - ] - - def __init__(self, spark: SparkSession, dbutils: DBUtils): - """Initialize the pipeline builder with Spark session and utilities.""" - # Initialize Spark context - self.spark = spark - self.dbutils = dbutils - self.context = self.dbutils.entry_point.getDbutils().notebook().getContext() - self.token = self.context.apiToken().get() - - self.pipeline_bundle_spec_format = "json" # Default to JSON format - self.dataflow_specs = [] - self.dataflow_spec_filters = {} - self.mandatory_table_properties = {} - self.operational_metadata_schema = None - self.pipeline_config = {} - self.secrets_manager = None - self.substitution_manager = None - self.driver_cores = os.cpu_count() - self.default_max_workers = self.driver_cores - 1 if self.driver_cores else 1 - - # Bootstrap mandatory paths and register sys.path - self._load_mandatory_paths() - register_bundle_sys_paths(self.framework_path, self.bundle_path) - - # Resolve the main logger - log_level = self.spark.conf.get(DLTPipelineSettingKeys.LOG_LEVEL, "INFO").upper() - self.logger = pipeline_logger.resolve_pipeline_logger( - spark=self.spark, - dbutils=self.dbutils, - framework_path=self.framework_path, - bundle_path=self.bundle_path, - spark_log_level=log_level, - ) - self.logger.info("Initializing Pipeline...") - self.logger.info("Logical cores (threads): %s", self.driver_cores) - self.logger.info("Max workers: %s", self.default_max_workers) - - # Initialize core singletons - pipeline_config.initialize_core( - spark=self.spark, - dbutils=self.dbutils, - logger=self.logger, - ) - - # Load configurations and initialize components - self._init_configurations() - self._init_pipeline_components() - - def _load_mandatory_paths(self) -> None: - """Load mandatory Spark conf paths required before logger and config init.""" - config_values = { - param: self.spark.conf.get(param, None) - for param in self.MANDATORY_CONFIG_PARAMS - } - - missing_params = [param for param, value in config_values.items() if not value] - if missing_params: - raise ValueError(f"Missing mandatory config parameters: {missing_params}") - - self.bundle_path = config_values[DLTPipelineSettingKeys.BUNDLE_SOURCE_PATH] - self.framework_path = config_values[DLTPipelineSettingKeys.FRAMEWORK_SOURCE_PATH] - self._framework_config_path = resolve_framework_config_path(self.framework_path) - self.workspace_host = config_values[DLTPipelineSettingKeys.WORKSPACE_HOST] - - def _init_configurations(self) -> None: - """Load and validate all necessary configurations.""" - # Load optional parameters - ignore_validation_errors = self.spark.conf.get( - DLTPipelineSettingKeys.PIPELINE_IGNORE_VALIDATION_ERRORS, "false" - ) - self.ignore_validation_errors = (ignore_validation_errors.lower() == "true") - - # Load pipeline details - self.logger.info("Loading Pipeline Details...") - self.pipeline_details = PipelineDetails( - pipeline_id=self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_ID, None), - pipeline_catalog=self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_CATALOG, None), - pipeline_schema=( - self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_SCHEMA, None) - or self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_TARGET, None) - ), - pipeline_layer=self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_LAYER, None), - start_utc_timestamp=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'), - workspace_env=self.spark.conf.get(DLTPipelineSettingKeys.BUNDLE_TARGET, None), - logical_env=self.spark.conf.get(DLTPipelineSettingKeys.LOGICAL_ENV, "") - ) - self.logger.info("Pipeline Details: %s", json.dumps(self.pipeline_details.__dict__, indent=4)) - - # Initialize pipeline details singleton - pipeline_config.initialize_pipeline_details(self.pipeline_details) - - # Get the pipeline update id via event hook (only method at the moment) - # This only gets populated post initiliazation. Currently retrieved in operational_metadata.py - @dp.on_event_hook - def update_id_hook(event): - if self.spark.conf.get("pipeline.pipeline_update_id", "") == "": - update_id = event.get("origin", {}).get("update_id", "") - if update_id: - self.spark.conf.set("pipeline.pipeline_update_id", update_id) - - # Load dataflow filters - self.logger.info("Loading Dataflow Filters...") - self.dataflow_spec_filters = { - "data_flow_ids": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_DATA_FLOW_ID, None), - "data_flow_groups": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_DATA_FLOW_GROUP, None), - "flow_group_ids": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_FLOW_GROUP_ID, None), - "target_tables": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_TARGET_TABLE, None), - "files": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILE_FILTER, None), - } - - def _init_pipeline_components(self) -> None: - """Initialize all pipeline components.""" - - # Load and merge configurations - self._load_merged_config() - - # Initialize substitution manager - self._init_substitution_manager() - - # Initialize table migration state volume path (after substitution manager so tokens are resolved) - table_migration_path = self.pipeline_config.get("table_migration_state_volume_path", None) - if table_migration_path: - table_migration_path = self.substitution_manager.substitute_string(table_migration_path) - pipeline_config.initialize_table_migration(table_migration_path) - - # Initialize secrets manager - self._init_secrets_manager() - - # Initialize dataflow specifications - self._init_dataflow_specs() - - # Setup operational metadata - self._setup_operational_metadata() - - # Apply Spark configurations - self._apply_spark_config() - - def _load_framework_global_config(self) -> Dict[str, Any]: - """Load the framework global config, deep-merging any src/local/config/ overrides.""" - from config_resolver import load_framework_config - - override_dir = os.path.join(self.framework_path, FrameworkPaths.CONFIG_OVERRIDE_PATH) - is_override = any( - os.path.exists(os.path.join(override_dir, n)) for n in FrameworkPaths.GLOBAL_CONFIG - ) - - if is_override: - import warnings - warnings.warn( - f"{FrameworkPaths.CONFIG_OVERRIDE_PATH} is deprecated (v0.13.0) and will be " - f"removed in v1.0.0. Migrate your overrides to {FrameworkPaths.LOCAL_CONFIG_PATH} " - "— only the keys you want to change are needed (sparse files are supported). " - "See the framework configuration documentation for migration steps.", - DeprecationWarning, - stacklevel=2, - ) - self._active_config_path = FrameworkPaths.CONFIG_OVERRIDE_PATH - else: - self._active_config_path = FrameworkPaths.CONFIG_PATH - - self.logger.info( - "Retrieving Global Framework Config From: %s", - os.path.join(self.framework_path, self._active_config_path), - ) - result = load_framework_config( - FrameworkPaths.GLOBAL_CONFIG, - self.framework_path, - config_path=self._active_config_path, - fail_on_not_exists=True, - ) - self.logger.info("Global Framework Config (resolved): %s", result) - return result - - def _load_pipeline_bundle_global_config_file(self) -> Dict[str, Any]: - """Load a global config file""" - global_config_paths = [ - os.path.join(self.bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, path) for path in PipelineBundlePaths.GLOBAL_CONFIG_FILE - ] - - # Check if more than one global config exists - existing_configs = [path for path in global_config_paths if os.path.exists(path)] - if len(existing_configs) > 1: - raise ValueError(f"Multiple pipeline global config files found. Only one is allowed: {existing_configs}") - - if not existing_configs: - return {} - - pipeline_config_path = existing_configs[0] - self.logger.info("Retrieving Pipeline Global Config From: %s", pipeline_config_path) - return utility.load_config_file_auto(pipeline_config_path, False) or {} - - def _load_merged_config(self) -> None: - """Load and merge global and pipeline-specific configurations.""" - self.pipeline_config = self._load_framework_global_config() - pipeline_bundle_config = self._load_pipeline_bundle_global_config_file() - - # Initialize pipeline bundle spec format - self._init_pipeline_bundle_spec_format(pipeline_bundle_config) - - # Merge pipeline bundle config - if pipeline_bundle_config: - self.pipeline_config.update(pipeline_bundle_config) - - self.mandatory_table_properties = self.pipeline_config.get("mandatory_table_properties", {}) - - # Initialize mandatory table properties singleton - pipeline_config.initialize_mandatory_table_properties(self.mandatory_table_properties) - - # Initialize mandatory configuration singleton - pipeline_config.initialize_mandatory_configuration() - - def _init_pipeline_bundle_spec_format(self, pipeline_bundle_config: Dict[str, Any]) -> None: - """Initialize the pipeline bundle spec format.""" - valid_formats = [fmt.value for fmt in SupportedSpecFormat] - - # Process global format configuration - global_format_dict = self.pipeline_config.pop("pipeline_bundle_spec_format", None) - if global_format_dict: - self.logger.info("Global pipeline bundle spec format: %s", global_format_dict) - global_format = global_format_dict.get("format", "json") - - if global_format not in valid_formats: - raise ValueError(f"Invalid pipeline bundle spec format: {global_format}. Valid formats are: {valid_formats}") - - self.pipeline_bundle_spec_format = global_format - allow_override = global_format_dict.get("allow_override", False) - else: - allow_override = False - - # Process pipeline-specific format configuration - pipeline_format_dict = pipeline_bundle_config.pop("pipeline_bundle_spec_format", None) - if pipeline_format_dict: - self.logger.info("Pipeline bundle spec format: %s", pipeline_format_dict) - pipeline_format = pipeline_format_dict.get("format", None) - - if pipeline_format and pipeline_format not in valid_formats: - raise ValueError(f"Invalid pipeline bundle spec format: {pipeline_format}. Valid formats are: {valid_formats}") - - if pipeline_format and pipeline_format != self.pipeline_bundle_spec_format and not allow_override: - raise ValueError(f"Pipeline bundle spec format has been set at global framework level as {self.pipeline_bundle_spec_format}. Override has been disabled.") - - if pipeline_format and allow_override: - self.pipeline_bundle_spec_format = pipeline_format - - self.logger.info("Pipeline bundle spec format: %s", self.pipeline_bundle_spec_format) - - def _init_substitution_manager(self) -> None: - """Initialize the substitution manager.""" - self.logger.info("Initializing Substitution Manager...") - - workspace_env = self.pipeline_details.workspace_env or "" - - # Build framework substitutions paths - framework_subs_paths = [ - os.path.join(self.framework_path, self._framework_config_path, workspace_env + path) - for path in FrameworkPaths.GLOBAL_SUBSTITUTIONS - ] - self.logger.info("Framework substitutions paths: %s", framework_subs_paths) - - # Build pipeline substitutions paths - suffixes = utility.get_format_suffixes(self.pipeline_bundle_spec_format, "substitutions") - pipeline_subs_paths = [os.path.join( - self.bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, workspace_env + suffix - ) for suffix in suffixes - ] - self.logger.info("Pipeline substitutions paths: %s", pipeline_subs_paths) - - self.substitution_manager = SubstitutionManager( - framework_substitutions_paths=framework_subs_paths, - pipeline_substitutions_paths=pipeline_subs_paths, - additional_tokens=self.pipeline_details.__dict__ - ) - self.logger.debug("Loaded substitution config: %s", self.substitution_manager._substitutions_config) - - # Initialize substitution manager singleton - pipeline_config.initialize_substitution_manager(self.substitution_manager) - - def _init_secrets_manager(self) -> None: - """Initialize the secrets manager.""" - self.logger.info("Initializing Secrets Manager...") - - workspace_env = self.pipeline_details.workspace_env or "" - - # Build framework secrets paths - framework_secrets_config_paths = [ - os.path.join(self.framework_path, self._framework_config_path, workspace_env + path) - for path in FrameworkPaths.GLOBAL_SECRETS - ] - - # Build pipeline secrets paths - suffixes = utility.get_format_suffixes(self.pipeline_bundle_spec_format, "secrets") - pipeline_secrets_configs_paths = [os.path.join( - self.bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, workspace_env + suffix - ) for suffix in suffixes - ] - - secrets_validator_path = os.path.join( - self.framework_path, FrameworkPaths.SECRETS_SCHEMA_PATH - ) - - self.secrets_manager = SecretsManager( - json_validation_schema_path=secrets_validator_path, - framework_secrets_config_paths=framework_secrets_config_paths, - pipeline_secrets_config_paths=pipeline_secrets_configs_paths - ) - - def _init_dataflow_specs(self) -> None: - """Initialize dataflow specifications.""" - self.logger.info("Initializing Dataflow Spec Builder...") - - dataflow_spec_version = self.pipeline_config.get("dataflow_spec_version", None) - dataflow_spec_builder_max_workers = self.pipeline_config.get( - FrameworkSettings.OVERRIDE_MAX_WORKERS_KEY, - self.default_max_workers - ) - - self.dataflow_specs = DataflowSpecBuilder( - bundle_path=self.bundle_path, - framework_path=self.framework_path, - filters=self.dataflow_spec_filters, - secrets_manager=self.secrets_manager, - ignore_validation_errors=self.ignore_validation_errors, - dataflow_spec_version=dataflow_spec_version, - max_workers=dataflow_spec_builder_max_workers, - spec_file_format=self.pipeline_bundle_spec_format - ).build() - - if not self.dataflow_specs: - raise ValueError(f"No dataflow specifications found in: {self.bundle_path}") - - def _setup_operational_metadata(self) -> None: - """Set up operational metadata schema.""" - self.logger.info("Initializing Operational Metadata...") - layer = self.pipeline_details.pipeline_layer - if not layer: - self.logger.info("Layer not set in pipeline, skipping operational metadata...") - self.operational_metadata_schema = None - return - - self.logger.info("Operational Metadata: layer set to %s", layer) - config_name = f"operational_metadata_{layer}.json" - metadata_json = load_framework_config( - config_name, self.framework_path, - config_path=getattr(self, "_active_config_path", FrameworkPaths.CONFIG_PATH), - fail_on_not_exists=False, - ) - self.logger.info("Operational Metadata config (resolved): %s", metadata_json) - self.operational_metadata_schema = ( - T.StructType.fromJson(metadata_json) if metadata_json else None - ) - - # Initialize operational metadata schema singleton - pipeline_config.initialize_operational_metadata_schema(self.operational_metadata_schema) - - def _apply_spark_config(self) -> None: - """Apply Spark configuration settings.""" - spark_config = self.pipeline_config.get("spark_config", {}) - if spark_config: - self.logger.info("Initializing Spark Configs...") - for prop, value in spark_config.items(): - self.logger.info("Set Spark Config: %s = %s", prop, value) - self.spark.conf.set(prop, value) - - def initialize_pipeline(self) -> None: - """Initialize the Spark Declarative Pipeline.""" - def create_dataflow(spec): - """Create a dataflow from a specification.""" - return DataFlow(dataflow_spec=spec).create_dataflow() - - run_init_scripts(self.framework_path, self.bundle_path, "pre", self.logger) - - self.logger.info("Initializing Pipeline...") - pipeline_builder_threading_disabled = self.pipeline_config.get( - FrameworkSettings.PIPELINE_BUILDER_DISABLE_THREADING_KEY, - True - ) - - self.logger.info("Processing Dataflow Specs...") - if pipeline_builder_threading_disabled: - self.logger.info("Pipeline Builder Threading Disabled, creating dataflows sequentially...") - for spec in self.dataflow_specs: - create_dataflow(spec) - else: - pipeline_builder_max_workers = self.pipeline_config.get( - FrameworkSettings.OVERRIDE_MAX_WORKERS_KEY, - self.default_max_workers - ) - - with ThreadPoolExecutor(max_workers=pipeline_builder_max_workers) as executor: - self.logger.info("Pipeline Builder Threading Enabled. Max Workers: %s", pipeline_builder_max_workers) - futures = [ - executor.submit(create_dataflow, spec) - for spec in self.dataflow_specs - ] - for future in futures: - future.result() - - run_init_scripts(self.framework_path, self.bundle_path, "post", self.logger) +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.dlt_pipeline_builder import DLTPipelineBuilder +from lakeflow_framework.dlt_pipeline_builder import DLTPipelineBuilder # noqa: F401 diff --git a/src/lakeflow_framework/__init__.py b/src/lakeflow_framework/__init__.py new file mode 100644 index 0000000..a630ed7 --- /dev/null +++ b/src/lakeflow_framework/__init__.py @@ -0,0 +1,96 @@ +""" +Lakeflow Framework — A metadata-driven framework for building Databricks Spark Declarative Pipelines. + +Primary import path: + from lakeflow_framework import DLTPipelineBuilder + from lakeflow_framework.dlt_pipeline_builder import DLTPipelineBuilder + +Flat-deploy compat shims at ``src/utility.py``, ``src/constants.py`` etc. are kept until v1.0.0. +""" + +from importlib.metadata import PackageNotFoundError, version as _pkg_version + +try: + __version__: str = _pkg_version("lakeflow-framework") +except PackageNotFoundError: + # Flat deploy — package not pip-installed; fall back to the VERSION file. + import os as _os + _version_file = _os.path.join(_os.path.dirname(__file__), "..", "..", "VERSION") + try: + with open(_version_file) as _f: + __version__ = _f.read().strip() + except OSError: + __version__ = "unknown" + +from .dataflow import ( + DataFlow, + FlowConfig, + FlowGroup, + BaseFlow, + QuarantineManager, + TableMigrationManager, + TargetType, + SinkType, + View, + StagingTable, +) +from .dataflow.dataflow_spec import DataflowSpec +from .dataflow_spec_builder import DataflowSpecBuilder, DataQualityExpectationBuilder +from .dlt_pipeline_builder import DLTPipelineBuilder +from .pipeline_config import ( + get_spark, + get_logger, + get_operational_metadata_schema, + get_pipeline_details, + get_substitution_manager, + get_mandatory_table_properties, + initialize_core, + initialize_operational_metadata_schema, + initialize_pipeline_details, + initialize_substitution_manager, + initialize_mandatory_table_properties, +) +from .secrets_manager import SecretsManager +from .substitution_manager import SubstitutionManager +from .constants import SystemColumns, MetaDataColumnDefs +from . import utility + +__all__ = [ + "__version__", + "DLTPipelineBuilder", + "DataflowSpec", + "DataflowSpecBuilder", + "DataQualityExpectationBuilder", + # Main classes + "DataFlow", + "FlowConfig", + "FlowGroup", + "BaseFlow", + "StagingTable", + "View", + # Managers + "SecretsManager", + "SubstitutionManager", + "TableMigrationManager", + "QuarantineManager", + # Types and enums + "TargetType", + "SinkType", + # Constants + "SystemColumns", + "MetaDataColumnDefs", + # Utilities + "utility", + # Pipeline config + "get_spark", + "get_logger", + "get_operational_metadata_schema", + "get_pipeline_details", + "get_substitution_manager", + "get_mandatory_table_properties", + "initialize_core", + "initialize_operational_metadata_schema", + "initialize_pipeline_details", + "initialize_substitution_manager", + "initialize_mandatory_table_properties", +] diff --git a/src/lakeflow_framework/bundle_loader.py b/src/lakeflow_framework/bundle_loader.py new file mode 100644 index 0000000..bbf026e --- /dev/null +++ b/src/lakeflow_framework/bundle_loader.py @@ -0,0 +1,168 @@ +""" +Bundle sys.path registration and init script execution. + +Framework bundle (framework.sourcePath → src/): + local/libraries/ — org-wide cluster-install artefacts + loose .py / packages + local/python/ — org-wide Data Flow Spec-referenced Python + extensions/ — DEPRECATED (v0.13.0): flat top-level .py files; removed in v1.0.0. + Migrate importable modules to local/python/. + +Pipeline bundle (bundle.sourcePath → src/): + src/libraries/ — bundle-local cluster-install artefacts + loose .py / packages + src/python/ — bundle Data Flow Spec-referenced Python + extensions/ — DEPRECATED (v0.13.0): flat top-level .py files; removed in v1.0.0. + Migrate importable modules to src/python/. + +Init scripts executed for framework then bundle per phase: + Framework: local/init/pre/ — before SDP table/view declarations inside initialize_pipeline() + local/init/post/ — after all SDP declarations + Bundle: src/init/pre/ + src/init/post/ + +Within each directory, scripts run in sorted filename order; files whose names start +with '_' are skipped. A script that raises an exception fails the pipeline. +""" + +from __future__ import annotations + +import os +import runpy +import sys +import warnings +from typing import Literal + +from lakeflow_framework.constants import FrameworkPaths, PipelineBundlePaths + +Phase = Literal["pre", "post"] + +_FRAMEWORK_PHASE_PATH: dict[str, str] = { + "pre": FrameworkPaths.LOCAL_INIT_PRE_PATH, + "post": FrameworkPaths.LOCAL_INIT_POST_PATH, +} + +_BUNDLE_PHASE_PATH: dict[str, str] = { + "pre": PipelineBundlePaths.INIT_PRE_PATH, + "post": PipelineBundlePaths.INIT_POST_PATH, +} + +_PHASE_LABEL: dict[str, str] = { + "pre": "pre-init", + "post": "post-init", +} + + +def _add_if_dir(path: str, label: str, logger=None) -> None: + """Insert path at the front of sys.path if it is a directory and not already present.""" + if os.path.isdir(path): + if path not in sys.path: + sys.path.insert(0, path) + if logger is not None: + logger.info("Registered %s on sys.path: %s", label, path) + + +def _has_top_level_py(directory: str) -> bool: + """Return True if directory exists and contains at least one top-level .py file.""" + if not os.path.isdir(directory): + return False + return any( + os.path.isfile(os.path.join(directory, name)) and name.endswith(".py") + for name in os.listdir(directory) + ) + + +def _warn_legacy_extensions(ext_path: str, python_path: str, level: str, logger=None) -> None: + """Register legacy extensions/ on sys.path and emit a deprecation warning. + + extensions/ is always registered when it contains top-level .py files, + regardless of whether src/python/ (or local/python/) also exists. + Deprecated means warn-and-still-work until removal in v1.0.0. + """ + if not _has_top_level_py(ext_path): + return + _add_if_dir(ext_path, f"{level} extensions (legacy)", logger) + msg = ( + f"[{level}] Top-level .py files under {ext_path!r} are deprecated. " + "Move importable modules to " + + ("local/python/ (framework) or " if level == "framework" else "") + + "src/python/ (bundle). " + "Flat extensions/ sys.path support will be removed in v1.0.0." + ) + if logger is not None: + logger.warning(msg) + warnings.warn(msg, DeprecationWarning, stacklevel=3) + + +def register_bundle_sys_paths( + framework_path: str, + bundle_path: str, + logger=None, +) -> None: + """ + Register sys.path roots for framework (local/) then pipeline bundle (src/). + + Framework bundle uses local/ paths (local/libraries/, local/python/). + Pipeline bundle uses src/ paths (src/libraries/, src/python/). + Legacy extensions/ is deprecated in both contexts. + + ``logger`` is optional. When omitted, paths are registered silently — useful + for early bootstrap before the real logger is available. + """ + # Framework bundle: custom code lives under local/ only + fw_libraries = os.path.normpath(os.path.join(framework_path, FrameworkPaths.LOCAL_LIBRARIES_PATH)) + fw_python = os.path.normpath(os.path.join(framework_path, FrameworkPaths.LOCAL_PYTHON_PATH)) + fw_legacy = os.path.normpath(os.path.join(framework_path, FrameworkPaths.EXTENSIONS_PATH)) + _add_if_dir(fw_libraries, "framework local/libraries", logger) + _add_if_dir(fw_python, "framework local/python", logger) + _warn_legacy_extensions(fw_legacy, fw_python, "framework", logger) + + # Pipeline bundle: custom code lives under src/ + pipe_libraries = os.path.normpath(os.path.join(bundle_path, PipelineBundlePaths.LIBRARIES_PATH)) + pipe_python = os.path.normpath(os.path.join(bundle_path, PipelineBundlePaths.PYTHON_PATH)) + pipe_legacy = os.path.normpath(os.path.join(bundle_path, PipelineBundlePaths.EXTENSIONS_PATH)) + _add_if_dir(pipe_libraries, "bundle src/libraries", logger) + _add_if_dir(pipe_python, "bundle src/python", logger) + _warn_legacy_extensions(pipe_legacy, pipe_python, "bundle", logger) + + +def run_init_scripts( + framework_path: str, + bundle_path: str, + phase: Phase, + logger, +) -> None: + """ + Execute all .py lifecycle scripts under the init// directory for framework then bundle. + + Framework bundle uses local/init//; pipeline bundle uses src/init//. + Files are sorted by filename; files starting with '_' are skipped. + Each file is run with runpy.run_path(..., run_name='__main__'). + A script that raises an exception fails the pipeline. + + Args: + framework_path: Root of the framework bundle (framework.sourcePath). + bundle_path: Root of the pipeline bundle (bundle.sourcePath). + phase: "pre" (before SDP declarations) or "post" (after). + logger: Framework logger instance. + """ + if phase not in _FRAMEWORK_PHASE_PATH: + raise ValueError(f"Invalid init phase: {phase!r}. Expected one of {list(_FRAMEWORK_PHASE_PATH)}.") + + label = _PHASE_LABEL[phase] + + for base, level, phase_paths in ( + (framework_path, "framework", _FRAMEWORK_PHASE_PATH), + (bundle_path, "bundle", _BUNDLE_PHASE_PATH), + ): + folder = os.path.normpath(os.path.join(base, phase_paths[phase])) + if not os.path.isdir(folder): + continue + scripts = sorted( + name for name in os.listdir(folder) + if name.endswith(".py") and not name.startswith("_") + ) + for filename in scripts: + path = os.path.join(folder, filename) + if not os.path.isfile(path): + continue + logger.info("Running %s %s script: %s", level, label, path) + runpy.run_path(path, run_name="__main__") diff --git a/src/config/default/dataflow_spec_mapping/0.1.0/dataflow_spec_mapping.json b/src/lakeflow_framework/config/default/dataflow_spec_mapping/0.1.0/dataflow_spec_mapping.json similarity index 100% rename from src/config/default/dataflow_spec_mapping/0.1.0/dataflow_spec_mapping.json rename to src/lakeflow_framework/config/default/dataflow_spec_mapping/0.1.0/dataflow_spec_mapping.json diff --git a/src/config/default/dataflow_spec_mapping/0.2.0/dataflow_spec_mapping.json b/src/lakeflow_framework/config/default/dataflow_spec_mapping/0.2.0/dataflow_spec_mapping.json similarity index 100% rename from src/config/default/dataflow_spec_mapping/0.2.0/dataflow_spec_mapping.json rename to src/lakeflow_framework/config/default/dataflow_spec_mapping/0.2.0/dataflow_spec_mapping.json diff --git a/src/config/default/global.json b/src/lakeflow_framework/config/default/global.json similarity index 100% rename from src/config/default/global.json rename to src/lakeflow_framework/config/default/global.json diff --git a/src/config/default/operational_metadata_bronze.json b/src/lakeflow_framework/config/default/operational_metadata_bronze.json similarity index 100% rename from src/config/default/operational_metadata_bronze.json rename to src/lakeflow_framework/config/default/operational_metadata_bronze.json diff --git a/src/config/default/operational_metadata_gold.json b/src/lakeflow_framework/config/default/operational_metadata_gold.json similarity index 100% rename from src/config/default/operational_metadata_gold.json rename to src/lakeflow_framework/config/default/operational_metadata_gold.json diff --git a/src/config/default/operational_metadata_silver.json b/src/lakeflow_framework/config/default/operational_metadata_silver.json similarity index 100% rename from src/config/default/operational_metadata_silver.json rename to src/lakeflow_framework/config/default/operational_metadata_silver.json diff --git a/src/lakeflow_framework/config_resolver.py b/src/lakeflow_framework/config_resolver.py new file mode 100644 index 0000000..55677bc --- /dev/null +++ b/src/lakeflow_framework/config_resolver.py @@ -0,0 +1,269 @@ +"""Framework configuration resolver. + +Provides the primary API for loading framework config files and bundled +JSON schemas with ``src/local/config/`` sparse-override support, plus the +deprecated ``resolve_framework_config_path`` shim kept for backward +compatibility until v1.0.0. + +Strategy B resolver +------------------- +:func:`load_framework_default_json` implements the canonical two-source +resolution with Workspace Files-first priority: + +1. **Workspace Files** — ``{framework_path}/lakeflow_framework/config/default/`` + when *framework_path* is provided and the file exists. Explicit beats + implicit: if the customer has deployed the framework files in Workspace + Files, that copy is used. +2. **Package data** (``importlib.resources``) — always present after + ``pip install lakeflow-framework``; also resolves correctly for + flat-deploy when ``src/`` is on ``sys.path`` and the file was not found + under *framework_path*. +3. **``src/local/config/`` custom override** — deep-merged on top when + *framework_path* is provided and the sparse fragment exists. + +:func:`load_framework_schema` returns an ``importlib.resources`` traversable +for a bundled JSON schema file, suitable for ``jsonschema.RefResolver`` and +similar validators. +""" +import importlib.resources +import json +import os +import warnings +from typing import Dict, Optional, Sequence, Union + +from lakeflow_framework.constants import FrameworkPaths + + +def load_framework_default_json( + name: str, + framework_path: Optional[str] = None, +) -> Dict: + """Load a framework default JSON config file using Strategy B (disk-first). + + Resolution order: + + 1. Workspace Files — ``{framework_path}/lakeflow_framework/config/default/`` + when *framework_path* is provided and the file exists on disk. + 2. Package data — ``importlib.resources`` (wheel install or flat-deploy + with ``src/`` on ``sys.path``) if disk path is unavailable. + 3. ``src/local/config/`` custom override — deep-merged on top when + *framework_path* is provided and the sparse fragment exists. + + Args: + name: Config filename, e.g. ``"global.json"`` or + ``"operational_metadata_bronze.json"``. + framework_path: Absolute path to the framework bundle's ``src/`` + directory. When ``None`` only package data is used + (no custom override merging possible). + + Returns: + Parsed config dict (base config deep-merged with custom override if present). + + Raises: + FileNotFoundError: If the file cannot be found either on disk or via + ``importlib.resources``. + """ + from lakeflow_framework.utility import deep_merge + + # 1. Disk-first: explicit framework_path takes priority over package data. + base: Optional[Dict] = None + file_path: Optional[str] = None + if framework_path: + file_path = os.path.normpath( + os.path.join( + framework_path, + FrameworkPaths.CONFIG_PATH.lstrip("./"), + name, + ) + ) + if os.path.isfile(file_path): + with open(file_path, encoding="utf-8") as fh: + base = json.load(fh) + + # 2. importlib.resources fallback when disk path absent or no framework_path. + if base is None: + try: + ref = importlib.resources.files("lakeflow_framework").joinpath( + f"config/default/{name}" + ) + base = json.loads(ref.read_text("utf-8")) + except (FileNotFoundError, TypeError, ModuleNotFoundError) as exc: + raise FileNotFoundError( + f"Framework default config '{name}' not found on disk ({file_path!r}) " + f"or via package data." + ) from exc + + # 3. Sparse local/config override — only when framework_path is known. + if framework_path: + override_path = os.path.join( + framework_path, FrameworkPaths.LOCAL_CONFIG_PATH.lstrip("./"), name + ) + if os.path.isfile(override_path): + with open(override_path, encoding="utf-8") as fh: + override = json.load(fh) + if override: + base = deep_merge(base, override) + + return base + + +def load_framework_schema(name: str): + """Return an ``importlib.resources`` traversable for a bundled schema file. + + Resolves to ``lakeflow_framework/schemas/`` inside the installed + package, which is equivalent to the Workspace Files path when ``src/`` is + on ``sys.path`` (flat deploy) and to wheel package data when installed via + ``pip install lakeflow-framework``. + + Args: + name: Schema filename, e.g. ``"main.json"`` or ``"flow_group.json"``. + + Returns: + An ``importlib.resources.abc.Traversable`` that can be read with + ``.read_text("utf-8")`` or converted to a ``Path`` via + ``importlib.resources.as_file()``. + + Example:: + + schema = load_framework_schema("main.json") + data = json.loads(schema.read_text("utf-8")) + """ + return importlib.resources.files("lakeflow_framework").joinpath(f"schemas/{name}") + + +def load_framework_config( + name: Union[str, Sequence[str]], + framework_path: str, + config_path: str = FrameworkPaths.CONFIG_PATH, + fail_on_not_exists: bool = True, +) -> Dict: + """Load a config file from *config_path* with ``src/local/config/`` deep-merge override. + + The caller is responsible for determining the active base path (override or default) + and passing it via ``config_path``. This function contains no override-detection + logic — it only loads from the given path and applies the ``src/local/config/`` + sparse override on top. + + When *name* is a sequence (e.g. ``FrameworkPaths.GLOBAL_CONFIG``), the function + resolves to the single matching file within ``config_path``, raising ``ValueError`` + if more than one match is found. + + Args: + name: Config file name (e.g. ``"logger.json"``) **or** a sequence of alternative + filenames (e.g. ``FrameworkPaths.GLOBAL_CONFIG``). + framework_path: Absolute path to the framework bundle's ``src/`` directory. + config_path: Relative path segment for the base config directory, e.g. + ``FrameworkPaths.CONFIG_PATH`` (default) or + ``FrameworkPaths.CONFIG_OVERRIDE_PATH`` when the caller has + determined that the deprecated override directory is active. + fail_on_not_exists: When ``True`` (default) raise ``FileNotFoundError`` if the + base file is absent. When ``False`` return ``{}``. + """ + from lakeflow_framework.utility import load_config_file_auto, deep_merge + + base_dir = os.path.join(framework_path, config_path) + local_dir = os.path.join(framework_path, FrameworkPaths.LOCAL_CONFIG_PATH) + + if not isinstance(name, str): + names = name + matches = [n for n in names if os.path.exists(os.path.join(base_dir, n))] + if len(matches) > 1: + raise ValueError( + f"Multiple config files found in {config_path}. " + f"Only one is allowed: {[os.path.join(base_dir, n) for n in matches]}" + ) + if not matches: + if fail_on_not_exists: + raise FileNotFoundError( + f"Config file not found. Expected one of {list(names)} under {base_dir}" + ) + return {} + name = matches[0] + + base_path = os.path.join(base_dir, name) + local_path = os.path.join(local_dir, name) + + defaults = load_config_file_auto(base_path, fail_on_not_exists=fail_on_not_exists) or {} + + if os.path.exists(local_path): + override = load_config_file_auto(local_path, fail_on_not_exists=False) or {} + if override: + return deep_merge(defaults, override) + + return defaults + + +def resolve_framework_config_dir(subdir: str, framework_path: str) -> str: + """Resolve a config subdirectory path, checking ``src/local/config/`` first. + + If ``src/local/config//`` exists it is returned; otherwise falls + back to ``src/config/default//``. Useful for directory-based config + such as ``dataflow_spec_mapping/``. + + Args: + subdir: Subdirectory name, e.g. ``"dataflow_spec_mapping"``. + framework_path: Absolute path to the framework bundle's ``src/`` directory. + """ + local_path = os.path.join(framework_path, FrameworkPaths.LOCAL_CONFIG_PATH, subdir) + if os.path.isdir(local_path): + return local_path + return os.path.join(framework_path, FrameworkPaths.CONFIG_PATH, subdir) + + +def _has_visible_children(directory: str) -> bool: + """Return True if *directory* exists and contains at least one non-hidden child.""" + if not os.path.isdir(directory): + return False + try: + names = os.listdir(directory) + except OSError: + return False + return any(not n.startswith(".") for n in names) + + +def resolve_framework_config_path(framework_path: str) -> str: + """ + DEPRECATED (v0.13.0): Use ``load_framework_config()`` or ``resolve_framework_config_dir()`` + instead. Removed in v1.0.0. + + Returns FrameworkPaths.CONFIG_OVERRIDE_PATH when the override directory has at least one + non-hidden child and mirrors the required layout; otherwise FrameworkPaths.CONFIG_PATH. + + Raises: + FileNotFoundError: If neither default nor override config roots contain valid files, + or if the override root is active but incomplete. + """ + config_dir = os.path.join(framework_path, FrameworkPaths.CONFIG_PATH) + override_dir = os.path.join(framework_path, FrameworkPaths.CONFIG_OVERRIDE_PATH) + if not _has_visible_children(override_dir): + if not _has_visible_children(config_dir): + raise FileNotFoundError( + f"No valid files found under {FrameworkPaths.CONFIG_PATH} or " + f"{FrameworkPaths.CONFIG_OVERRIDE_PATH} in the framework bundle " + f"({framework_path!s}). Please add framework configuration under " + f"{FrameworkPaths.CONFIG_PATH} (for example a global config file, " + f"the {FrameworkPaths.DATAFLOW_SPEC_MAPPING} directory, and related files)." + ) + return FrameworkPaths.CONFIG_PATH + + warnings.warn( + f"{FrameworkPaths.CONFIG_OVERRIDE_PATH} is deprecated (v0.13.0) and will be removed " + f"in v1.0.0. Migrate your overrides to {FrameworkPaths.LOCAL_CONFIG_PATH} — only the " + "keys you want to change are needed (sparse files are supported). " + "See the framework configuration documentation for migration steps.", + DeprecationWarning, + stacklevel=2, + ) + + mapping_dir = os.path.join(override_dir, FrameworkPaths.DATAFLOW_SPEC_MAPPING) + global_paths = [ + os.path.join(override_dir, name) for name in FrameworkPaths.GLOBAL_CONFIG + ] + if not os.path.isdir(mapping_dir) or not any(os.path.isfile(p) for p in global_paths): + raise FileNotFoundError( + f"Using {FrameworkPaths.CONFIG_OVERRIDE_PATH} requires both a global config file " + f"({' or '.join(FrameworkPaths.GLOBAL_CONFIG)}) and the " + f"{FrameworkPaths.DATAFLOW_SPEC_MAPPING} directory under that path. " + f"Copy the full {FrameworkPaths.CONFIG_PATH} tree into {FrameworkPaths.CONFIG_OVERRIDE_PATH}." + ) + return FrameworkPaths.CONFIG_OVERRIDE_PATH diff --git a/src/lakeflow_framework/constants.py b/src/lakeflow_framework/constants.py new file mode 100644 index 0000000..5b6240c --- /dev/null +++ b/src/lakeflow_framework/constants.py @@ -0,0 +1,200 @@ +from dataclasses import dataclass +from enum import Enum + +@dataclass(frozen=True) +class FrameworkSettings: + """ + FrameworkSettings is a class that contains constants for the framework settings. + """ + OVERRIDE_MAX_WORKERS_KEY: str = "override_max_workers" + PIPELINE_BUILDER_DISABLE_THREADING_KEY: str = "pipeline_builder_disable_threading" + + +@dataclass(frozen=True) +class DLTPipelineSettingKeys: + """ + DLTPipelineSettingKeys is a class that contains constants for various Pipeline settings keys. + """ + BUNDLE_SOURCE_PATH: str = "bundle.sourcePath" + BUNDLE_TARGET: str = "bundle.target" + FRAMEWORK_SOURCE_PATH: str = "framework.sourcePath" + LOG_LEVEL: str = "logLevel" + LOGICAL_ENV: str = "logicalEnv" + PIPELINE_CATALOG: str = "pipelines.catalog" + PIPELINE_FILE_FILTER: str = "pipeline.fileFilter" + PIPELINE_FILTER_DATA_FLOW_GROUP: str = "pipeline.dataFlowGroupFilter" + PIPELINE_FILTER_DATA_FLOW_ID: str = "pipeline.dataFlowIdFilter" + PIPELINE_FILTER_FLOW_GROUP_ID: str = "pipeline.flowGroupIdFilter" + PIPELINE_FILTER_TARGET_TABLE: str = "pipeline.targetTableFilter" + PIPELINE_ID: str = "pipelines.id" + PIPELINE_IGNORE_VALIDATION_ERRORS: str = "pipeline.ignoreValidationErrors" + PIPELINE_LAYER: str = "pipeline.layer" + PIPELINE_TARGET: str = "pipelines.target" + PIPELINE_SCHEMA: str = "pipelines.schema" + WORKSPACE_HOST: str = "workspace.host" + + +@dataclass(frozen=True) +class FrameworkPaths: + """ + FrameworkPaths is a class that contains constants for various paths and file masks used in the Lakeflow Framework. + + All LOCAL_* paths are relative to ``framework.sourcePath`` (which resolves to the ``src/`` + directory of the framework bundle, e.g. ``.../files/src``). Customer code in the framework + bundle lives exclusively under ``local/`` — there are no top-level ``src/libraries/``, + ``src/python/`` or ``src/init/`` directories in the framework bundle. + + CONFIG_PATH and CONFIG_OVERRIDE_PATH are static path segments. At runtime, which root to use + for framework config files should be chosen using utility.resolve_framework_config_path(framework_path). + + Note: CONFIG_PATH and the schema paths point into the ``lakeflow_framework/`` package subtree + so that ``importlib.resources`` can locate them when the package is installed as a wheel, while + flat-deploy customers using ``framework.sourcePath`` continue to resolve them on disk. + + Attributes: + CONFIG_PATH (str): Path to the config/default directory (inside the lakeflow_framework package tree). + CONFIG_OVERRIDE_PATH (str): DEPRECATED (v0.13.0) — use src/local/config/ instead; removed in v1.0.0. + LOCAL_CONFIG_PATH (str): Sparse overlay directory for framework config files. Files here are deep-merged on top of config/default equivalents. + LOCAL_LIBRARIES_PATH (str): Org-wide cluster-install artefacts + loose .py/packages on sys.path. + LOCAL_PYTHON_PATH (str): Org-wide pipeline logic modules called via pythonModule/pythonTransform. + LOCAL_INIT_PRE_PATH (str): Org-wide pre-init lifecycle scripts run before SDP declarations. + LOCAL_INIT_POST_PATH (str): Org-wide post-init lifecycle scripts run after SDP declarations. + EXTENSIONS_PATH (str): DEPRECATED (v0.13.0) — flat extensions/ on sys.path; removed in v1.0.0. Migrate to src/local/python/. + GLOBAL_CONFIG (tuple): Paths to the global configuration files. + GLOBAL_SUBSTITUTIONS (tuple): Paths to the global substitutions files. + GLOBAL_SECRETS (tuple): Paths to the global secrets files. + DATAFLOW_SPEC_MAPPING (str): Directory segment for dataflow spec mapping (under the resolved root). + LOGGER_CONFIG (str): Basename of the pluggable logger configuration file (under the resolved config root). + MAIN_SPEC_SCHEMA_PATH (str): Path to the main specification schema file. + FLOW_GROUP_SPEC_SCHEMA_PATH (str): Path to the flow group specification schema file. + EXPECTATIONS_SPEC_SCHEMA_PATH (str): Path to the expectations specification schema file. + SECRETS_SCHEMA_PATH (str): Path to the secrets specification schema file. + TEMPLATE_DEFINITION_SPEC_SCHEMA_PATH (str): Path to the template definition specification schema file. + TEMPLATE_SPEC_SCHEMA_PATH (str): Path to the template specification schema file. + """ + CONFIG_PATH: str = "./lakeflow_framework/config/default" + CONFIG_OVERRIDE_PATH: str = "./config/override" # DEPRECATED (v0.13.0) — use local/config/; removed v1.0.0 + LOCAL_CONFIG_PATH: str = "./local/config" + LOCAL_LIBRARIES_PATH: str = "./local/libraries" + LOCAL_PYTHON_PATH: str = "./local/python" + LOCAL_INIT_PRE_PATH: str = "./local/init/pre" + LOCAL_INIT_POST_PATH: str = "./local/init/post" + EXTENSIONS_PATH: str = "./extensions" # DEPRECATED (v0.13.0) — migrate to local/python/; removed v1.0.0 + GLOBAL_CONFIG: tuple = ("global.json", "global.yaml", "global.yml") + GLOBAL_SUBSTITUTIONS: tuple = ("_substitutions.json", "_substitutions.yaml", "_substitutions.yml") + GLOBAL_SECRETS: tuple = ("_secrets.json", "_secrets.yaml", "_secrets.yml") + DATAFLOW_SPEC_MAPPING: str = "dataflow_spec_mapping" + LOGGER_CONFIG: str = "logger.json" + REQUIREMENTS_FILE: str = "requirements.txt" + + # Spec schema definitions paths (inside the lakeflow_framework package tree) + SPEC_MAPPING_SCHEMA_PATH: str = "./lakeflow_framework/schemas/spec_mapping.json" + MAIN_SPEC_SCHEMA_PATH: str = "./lakeflow_framework/schemas/main.json" + FLOW_GROUP_SPEC_SCHEMA_PATH: str = "./lakeflow_framework/schemas/flow_group.json" + EXPECTATIONS_SPEC_SCHEMA_PATH: str = "./lakeflow_framework/schemas/expectations.json" + SECRETS_SCHEMA_PATH: str = "./lakeflow_framework/schemas/secrets.json" + TEMPLATE_DEFINITION_SPEC_SCHEMA_PATH: str = "./lakeflow_framework/schemas/spec_template_definition.json" + TEMPLATE_SPEC_SCHEMA_PATH: str = "./lakeflow_framework/schemas/spec_template.json" + + +class SupportedSpecFormat(str, Enum): + """Supported specification file formats.""" + JSON = "json" + YAML = "yaml" + + +@dataclass(frozen=True) +class PipelineBundleSuffixesJson: + """ + PipelineBundleSuffixesJson is a class that contains constants for various file suffixes used in the Pipeline Bundles in JSON format. + """ + MAIN_SPEC_FILE_SUFFIX: tuple = ("_main.json") + FLOW_GROUP_FILE_SUFFIX: tuple = ("_flow.json") + EXPECTATIONS_FILE_SUFFIX: tuple = (".json") + SECRETS_FILE_SUFFIX: tuple = ("_secrets.json") + SUBSTITUTIONS_FILE_SUFFIX: tuple = ("_substitutions.json") + + +@dataclass(frozen=True) +class PipelineBundleSuffixesYaml: + """ + PipelineBundleSuffixesYaml is a class that contains constants for various file suffixes used in the Pipeline Bundles in YAML format. + """ + MAIN_SPEC_FILE_SUFFIX: tuple = ("_main.yaml", "_main.yml") + FLOW_GROUP_FILE_SUFFIX: tuple = ("_flow.yaml", "_flow.yml") + EXPECTATIONS_FILE_SUFFIX: tuple = ("_expectations.yaml", "_expectations.yml") + SECRETS_FILE_SUFFIX: tuple = ("_secrets.yaml", "_secrets.yml") + SUBSTITUTIONS_FILE_SUFFIX: tuple = ("_substitutions.yaml", "_substitutions.yml") + + +@dataclass(frozen=True) +class PipelineBundlePaths: + """ + PipelineBundlePaths is a class that contains constants for various paths and file masks + used in the Pipeline Bundles. + + Attributes: + DATAFLOWS_BASE_PATH (str): The base path for dataflows. + DATAFLOW_SPEC_PATH (str): The path for dataflow specifications. + DML_PATH (str): The path for DML (Data Manipulation Language) files. + DQE_PATH (str): The path for data quality expectations. + LIBRARIES_PATH (str): Cluster-install artefacts + loose .py/packages on sys.path (primary). + PYTHON_PATH (str): Data Flow Spec-referenced Python — modules called via pythonModule/pythonTransform. + INIT_PRE_PATH (str): Pre-init lifecycle scripts run before SDP declarations. + INIT_POST_PATH (str): Post-init lifecycle scripts run after SDP declarations. + EXTENSIONS_PATH (str): DEPRECATED (v0.13.0) — flat extensions/ on sys.path; removed in v1.0.0. Migrate to src/python/. + GLOBAL_CONFIG_FILE (tuple): The file names for global configuration files. + PIPELINE_CONFIGS_PATH (str): The path for pipeline configuration files. + LOGGER_CONFIG (str): Basename of the pluggable logger configuration file (under pipeline_configs). + PYTHON_FUNCTION_PATH (str): The path for python functions. + SCHEMA_PATH (str): The path for schema files. + TEMPLATE_PATH (str): Path to the template directory. + """ + DATAFLOWS_BASE_PATH: str = "./dataflows" + DATAFLOW_SPEC_PATH: str = "dataflowspec" + DML_PATH: str = "./dml" + DQE_PATH: str = "./expectations" + LIBRARIES_PATH: str = "./libraries" + PYTHON_PATH: str = "./python" + INIT_PRE_PATH: str = "./init/pre" + INIT_POST_PATH: str = "./init/post" + EXTENSIONS_PATH: str = "./extensions" # DEPRECATED (v0.13.0) — migrate to python/; removed v1.0.0 + GLOBAL_CONFIG_FILE: tuple = ("./global.json", "./global.yaml", "./global.yml") + PIPELINE_CONFIGS_PATH: str = "./pipeline_configs" + LOGGER_CONFIG: str = "logger.json" + PYTHON_FUNCTION_PATH: str = "./python_functions" + SCHEMA_PATH: str = "./schemas" + TEMPLATE_PATH: str = "./templates" + REQUIREMENTS_FILE: str = "requirements.txt" + + +class SystemColumns: + """ + SystemColumns is a container for constants related to SDP and Framework system columns. + + Classes: + CDFColumns (Enum): Contains constants for Change Data Feed (CDF) system columns. + SCD2Columns (Enum): Contains constants for Slowly Changing Dimension Type 2 (SCD2) system columns. + """ + class CDFColumns(Enum): + """Change Data Feed system columns.""" + CDF_CHANGE_TYPE = "_change_type" + CDF_COMMIT_VERSION = "_commit_version" + CDF_COMMIT_TIMESTAMP = "_commit_timestamp" + + class SCD2Columns(Enum): + """SCD2 system columns.""" + SCD2_START_AT = "__START_AT" + SCD2_END_AT = "__END_AT" + +class MetaDataColumnDefs: + """MetaDataColumnDefs is a class that contains constants for all Framework metadata columns.""" + + QUARANTINE_FLAG = { + "name": "is_quarantined", + "type": "boolean", + "nullable": True, + "metadata": {} + } + +FILE_METADATA_COLUMN = "_metadata" \ No newline at end of file diff --git a/src/lakeflow_framework/contrib/README.rst b/src/lakeflow_framework/contrib/README.rst new file mode 100644 index 0000000..75e8ed2 --- /dev/null +++ b/src/lakeflow_framework/contrib/README.rst @@ -0,0 +1,35 @@ +``lakeflow_framework.contrib`` — Community Extensions +===================================================== + +.. warning:: + + This sub-package is **experimental**. Modules here are community-maintained, + may change without a deprecation window, and are not covered by the core + stability guarantee. + +Support policy +-------------- + +* ``contrib`` modules are gated behind the ``[contrib]`` extra: + + .. code-block:: bash + + pip install "lakeflow-framework[contrib]" + +* Stability label: **experimental** — breaking changes are allowed between + minor releases while a module carries this label. +* Each module documents its own external dependencies; they must be listed as + optional extras in ``pyproject.toml`` under the relevant sub-extra (e.g. + ``[contrib.myintegration]``) so that a plain ``pip install lakeflow-framework`` + does not pull them in. +* Community contributions follow the same PR process as core; the maintainers + review for safety and conformance, not for feature completeness. + +Adding a new contrib module +--------------------------- + +1. Create ``src/lakeflow_framework/contrib//``. +2. Add any required third-party dependencies as a new extra in ``pyproject.toml``. +3. Include a README (``README.rst`` or ``README.md``) describing the integration, + its status, and its external requirements. +4. Add CI tests gated behind the ``[contrib]`` install. diff --git a/src/lakeflow_framework/contrib/__init__.py b/src/lakeflow_framework/contrib/__init__.py new file mode 100644 index 0000000..73ecf90 --- /dev/null +++ b/src/lakeflow_framework/contrib/__init__.py @@ -0,0 +1 @@ +# lakeflow_framework.contrib — optional community-maintained integrations diff --git a/src/dataflow/__init__.py b/src/lakeflow_framework/dataflow/__init__.py similarity index 100% rename from src/dataflow/__init__.py rename to src/lakeflow_framework/dataflow/__init__.py diff --git a/src/dataflow/cdc.py b/src/lakeflow_framework/dataflow/cdc.py similarity index 98% rename from src/dataflow/cdc.py rename to src/lakeflow_framework/dataflow/cdc.py index 7515db8..5ce807b 100644 --- a/src/dataflow/cdc.py +++ b/src/lakeflow_framework/dataflow/cdc.py @@ -5,7 +5,7 @@ from pyspark.sql import functions as F import pyspark.sql.types as T -import pipeline_config +import lakeflow_framework.pipeline_config as pipeline_config @dataclass diff --git a/src/lakeflow_framework/dataflow/cdc_snapshot.py b/src/lakeflow_framework/dataflow/cdc_snapshot.py new file mode 100644 index 0000000..1625ad7 --- /dev/null +++ b/src/lakeflow_framework/dataflow/cdc_snapshot.py @@ -0,0 +1,651 @@ +import bisect +from dataclasses import dataclass, field +from datetime import datetime +import re +from typing import Dict, List, Literal, Optional, Union + +from pyspark import pipelines as dp +from pyspark.sql import DataFrame +from pyspark.sql import functions as F +import pyspark.sql.types as T + +import lakeflow_framework.pipeline_config as pipeline_config + +from .dataflow_config import DataFlowConfig +from .sources import SourceDelta, SourceBatchFiles, ReadConfig + + +@dataclass(frozen=True) +class CDCSnapshotTypes: + """Constants for the types of CDC Snapshot.""" + HISTORICAL = "historical" + PERIODIC = "periodic" + + +@dataclass(frozen=True) +class CDCSnapshotSourceTypes: + """Constants for the types of CDC Snapshot source types.""" + FILE = "file" + TABLE = "table" + + +@dataclass(frozen=True) +class CDCSnapshotVersionTypes: + """Constants for the types of CDC Snapshot version types.""" + DATE = "date" + INTEGER = "integer" + LONG = "long" + TIMESTAMP = "timestamp" + + +@dataclass(frozen=True) +class DeduplicateMode: + """Deduplication strategy for CDC snapshot source data. + - off: no deduplication (default). + - full_row: deduplicate based on the full row using dropDuplicates(); deterministic as the full row is deduplicated. + - keys_only: deduplicate based on the keys using dropDuplicates(keys); non-deterministic as it preserves the first row per key(s) without ordering on any other columns.""" + OFF = "off" + FULL_ROW = "full_row" + KEYS_ONLY = "keys_only" + + +@dataclass +class VersionInfo: + """A structure to hold version information with both raw and formatted values.""" + raw_value: Union[str, int, datetime] + version_type: str + datetime_format: Optional[str] = None + micro_second_mask_length: Optional[int] = None + + @property + def formatted_value(self) -> str: + """Get formatted value based on version type and datetime format.""" + if self.version_type == CDCSnapshotVersionTypes.TIMESTAMP: + if isinstance(self.raw_value, datetime): + if self.datetime_format: + if '%f' in self.datetime_format and self.micro_second_mask_length: + truncate_from_right = 6 - self.micro_second_mask_length + return self.raw_value.strftime(self.datetime_format)[:-truncate_from_right] + else: + return self.raw_value.strftime(self.datetime_format) + else: + return self.raw_value.strftime('%Y-%m-%d %H:%M:%S') + else: + return str(self.raw_value) + else: + return str(self.raw_value) + + @property + def sql_formatted_value(self) -> str: + """Get SQL formatted value with appropriate quotes.""" + if self.version_type in [CDCSnapshotVersionTypes.TIMESTAMP, CDCSnapshotVersionTypes.DATE]: + return f"'{self.formatted_value}'" + elif self.version_type in [CDCSnapshotVersionTypes.INTEGER, CDCSnapshotVersionTypes.LONG]: + return f"'{self.formatted_value}'" + else: + raise ValueError(f"Unsupported version type: {self.version_type}") + +@dataclass +class FilePathInfo: + """A structure to hold file path information.""" + full_path: str + filename_with_version_path: str + +@dataclass +class CDCSnapshotFileSource: + """A structure to hold the source configuration for CDC Snapshot.""" + format: str + path: str + readerOptions: Dict = field(default_factory=dict) + filter: Optional[str] = None + versionType: Optional[str] = None + startingVersion: Optional[Union[int, str]] = None + datetimeFormat: Optional[str] = None + microSecondMaskLength: Optional[int] = None + schemaPath: Optional[str] = None + selectExp: Optional[List[str]] = None + recursiveFileLookup: bool = False + deduplicateMode: Optional[ + Literal[ + DeduplicateMode.OFF, + DeduplicateMode.FULL_ROW, + DeduplicateMode.KEYS_ONLY, + ] + ] = DeduplicateMode.OFF + + +@dataclass +class CDCSnapshotTableSource: + """A structure to hold the source configuration for CDC Snapshot.""" + table: str + versionColumn: str + versionType: str + startingVersion: Optional[Union[int, str]] = None + selectExp: Optional[List[str]] = None + deduplicateMode: Optional[ + Literal[ + DeduplicateMode.OFF, + DeduplicateMode.FULL_ROW, + DeduplicateMode.KEYS_ONLY, + ] + ] = DeduplicateMode.OFF + + +@dataclass +class CDCSnapshotSettings: + """CDC Settings for the SDP auto CDC Snapshot API.""" + keys: List[str] + scd_type: str + snapshotType: str + sourceType: str = None + source: Dict = field(default_factory=dict) + track_history_column_list: Optional[List[str]] = None + track_history_except_column_list: Optional[List[str]] = None + sequence_by_data_type: T.DataType = None + + def __post_init__(self): + if self.snapshotType == CDCSnapshotTypes.HISTORICAL and not self.source: + raise ValueError("Source is required for Historical CDC from Snapshot") + + if self.scd_type == "2": + # TODO: implement dynamic sequence by type + self.sequence_by_data_type = T.TimestampType() + + if self.snapshotType == CDCSnapshotTypes.HISTORICAL: + version_type = self.get_source().versionType + if version_type == CDCSnapshotVersionTypes.INTEGER: + self.sequence_by_data_type = T.IntegerType() + + def get_source(self) -> Optional[CDCSnapshotFileSource]: + """Get source configuration for CDC from Snapshot.""" + if self.sourceType == CDCSnapshotSourceTypes.FILE: + return CDCSnapshotFileSource(**self.source) + elif self.sourceType == CDCSnapshotSourceTypes.TABLE: + return CDCSnapshotTableSource(**self.source) + else: + raise ValueError(f"Unsupported source type: {self.sourceType}") + + def is_historical(self) -> bool: + """Is the CDC snapshot type historical.""" + return self.snapshotType == CDCSnapshotTypes.HISTORICAL + + def is_file_source(self) -> bool: + """Is the CDC snapshot source type file.""" + return self.sourceType == CDCSnapshotSourceTypes.FILE + + +class CDCSnapshotFlow: + """A class to create a CDC Snapshot flow.""" + + def __init__(self, settings: CDCSnapshotSettings): + self.settings = settings + self.logger = pipeline_config.get_logger() + + # Core CDC settings + self.keys = settings.keys + self.scd_type = settings.scd_type + self.snapshotType = settings.snapshotType + self.track_history_column_list = settings.track_history_column_list + self.track_history_except_column_list = settings.track_history_except_column_list + self.sequence_by_data_type = settings.sequence_by_data_type + + # Historical snapshot specific settings + self.sourceType = None + self.source = None + if self.snapshotType == CDCSnapshotTypes.HISTORICAL: + self.sourceType = settings.sourceType + self.source = settings.get_source() + + if self.source is None: + raise ValueError("Source configuration is required for historical snapshots") + + # Cached version data + self._available_versions: Optional[List[VersionInfo]] = None + self._sorted_versions: Optional[List[VersionInfo]] = None + self._version_values: Optional[List[Union[int, datetime]]] = None + + @property + def sorted_versions(self) -> List[VersionInfo]: + """Get sorted versions.""" + if self._sorted_versions is None and self._available_versions: + self._sorted_versions = sorted(self._available_versions, key=lambda x: x.raw_value) + return self._sorted_versions or [] + + @property + def version_values(self) -> List[Union[int, datetime]]: + """Get version values.""" + if self._version_values is None: + self._version_values = [v.raw_value for v in self.sorted_versions] + return self._version_values + + def _deduplicate_by_keys(self, df: DataFrame) -> DataFrame: + """Deduplicate by configured keys, retaining first row per key. WARNING: This is non-deterministic.""" + self.logger.warning( + "CDC Snapshot: deduplicateMode=keys_only is non-deterministic as it peserves the first row per key." + ) + return df.dropDuplicates(self.keys) + + def _deduplicate_full_row(self, df: DataFrame) -> DataFrame: + """Deduplicate by full row besides metadata column if present.""" + columns_to_deduplicate_on = [col for col in df.columns if col not in ["_metadata"]] + return df.dropDuplicates(columns_to_deduplicate_on) + + def create( + self, + dataflow_config: DataFlowConfig, + target_table: str, + source_view_name: Optional[str] = None, + target_config_flags: Optional[List[str]] = None, + flow_name: Optional[str] = None # TODO: Add flow name + ) -> None: + """Create CDC from snapshot flow. + + Args: + dataflow_config: DataFlow configuration + target_table: Name of the target table + source_view_name: Name of the source view + flow_name: Optional name for the flow + """ + self.logger.debug(f"CDC From Snapshot: {self.source}") + + try: + if self.snapshotType == CDCSnapshotTypes.PERIODIC: + self._apply_periodic_changes(target_table, source_view_name) + elif self.snapshotType == CDCSnapshotTypes.HISTORICAL: + self._apply_historical_changes(target_table, dataflow_config, target_config_flags) + else: + raise ValueError(f"Unsupported snapshot type: {self.snapshotType}") + except Exception as e: + self.logger.error(f"Failed to create CDC snapshot flow: {e}") + raise + + def _apply_periodic_changes(self, target_table: str, source_view_name: str) -> None: + """Apply periodic changes from snapshot.""" + dp.create_auto_cdc_from_snapshot_flow( + target=target_table, + source=source_view_name, + keys=self.keys, + stored_as_scd_type=self.scd_type, + track_history_column_list=self.track_history_column_list, + track_history_except_column_list=self.track_history_except_column_list + ) + + def _apply_historical_changes(self, target_table: str, dataflow_config: DataFlowConfig, target_config_flags: Optional[List[str]] = None): + """Apply historical changes from snapshot.""" + dp.create_auto_cdc_from_snapshot_flow( + target=target_table, + snapshot_and_version=lambda version: self._next_snapshot_and_version(version, dataflow_config, target_config_flags), + keys=self.keys, + stored_as_scd_type=self.scd_type, + track_history_column_list=self.track_history_column_list, + track_history_except_column_list=self.track_history_except_column_list + ) + + def _next_snapshot_and_version(self, latest_snapshot_version, dataflow_config: DataFlowConfig, target_config_flags: Optional[List[str]] = None): + """Get the next snapshot and version.""" + try: + if self._available_versions is None: + self._available_versions = self._get_available_versions(latest_snapshot_version) + + if not self._available_versions: + self.logger.warning("CDC Snapshot: No valid versions found") + return None + + version_info = self._get_next_version(latest_snapshot_version) + if version_info is None: + self.logger.debug("CDC Snapshot: Retrieving next version was None") + return None + + self.logger.info(f"CDC Snapshot: Reading file version: {version_info.formatted_value}") + df = self._read_snapshot_dataframe(version_info, dataflow_config, target_config_flags) + if df is None or df.isEmpty(): + self.logger.debug("CDC Snapshot: Retrieving snapshot dataframe was None or empty") + return None + + self.logger.info(f"CDC Snapshot: Returning dataframe with version: {version_info.formatted_value}. Raw version: {version_info.raw_value}.") + return (df, version_info.raw_value) + + except Exception as e: + self.logger.error(f"CDC Snapshot: Error processing snapshots: {e}") + raise + + + def _get_available_versions(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> List[VersionInfo]: + """Get list of available versions from source.""" + if self.sourceType == CDCSnapshotSourceTypes.FILE: + return self._get_available_file_versions(latest_snapshot_version) + elif self.sourceType == CDCSnapshotSourceTypes.TABLE: + return self._get_available_table_versions(latest_snapshot_version) + else: + raise ValueError(f"Unsupported source type: {self.sourceType}") + + def _list_files(self, path, recursive=True): + """List files in a directory, with optional recursive file lookup. + + Args: + path: Directory path to list files from + recursive: If True, list files recursively. If False, list only files in the immediate directory. + + Returns: + List of file objects from dbutils.fs.ls() + """ + dbutils = pipeline_config.get_dbutils() + all_files = [] + + for f in dbutils.fs().ls(path): + all_files.append(f) + + if recursive and f.isDir(): + all_files.extend(self._list_files(f.path, recursive=True)) + + return all_files + + def _path_to_regex_pattern(self, path: str) -> str: + """Convert path to normalized regex pattern with named groups. + + Curly-brace syntax is converted to named capture groups: + - {version} -> (?P.+) (single capture group for version) + - {fragment} -> (?P.*?) (single capture group for fragment) + + If path already contains regex named groups (?P, + it is returned as-is (already a regex pattern). + """ + if re.search(r'\(\?P', path): + return path + if '{version}' not in path and '{fragment}' not in path: + return path + escaped = re.escape(path) + normalized = ( + escaped + .replace(r'\{version\}', r'(?P.+)') + .replace(r'\{fragment\}', r'(?P.*?)') + ) + return normalized + + def _get_dynamic_path_index(self, path: str) -> int: + """Return index of first path segment that contains version or fragment (curly or regex).""" + path_parts = path.split('/') + for idx, part in enumerate(path_parts): + if '{version}' in part or '{fragment}' in part: + return idx + if '(?P' in part: + return idx + raise ValueError(f"No version or fragment placeholder found in path: {path}") + + def _get_version_string_from_match(self, match: re.Match) -> Optional[str]: + """From a regex match, concatenate all groups named version_* (sorted) to form version string.""" + groupdict = match.groupdict() + version_keys = [k for k in groupdict if k.startswith('version_')] + if not version_keys: + return None + return ''.join(groupdict[k] or '' for k in version_keys) + + def _has_fragment_group(self, pattern: str) -> bool: + """Return True if pattern has a fragment capture group.""" + if '{fragment}' in pattern: + return True + return bool(re.search(r'\(\?P', pattern)) + + def _get_available_file_versions(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> List[VersionInfo]: + """Get list of available versions from file path (supports regex and {version}/{fragment} syntax).""" + path = self.source.path + pattern = self._path_to_regex_pattern(path) + dynamic_idx = self._get_dynamic_path_index(path) + path_parts = path.split('/') + parent_dir = '/'.join(path_parts[:dynamic_idx]) or '.' + # Anchor pattern so we only match full path (e.g. exclude customer.parquet/_SUCCESS) + file_pattern_regex = '/'.join(pattern.split('/')[dynamic_idx:]) + r'/?$' + + self.logger.debug(f"CDC Snapshot: Listing files in {parent_dir} with pattern {file_pattern_regex}") + + # List files using the configured recursive file lookup option + recursive_file_lookup = self.source.recursiveFileLookup + self.logger.debug(f"CDC Snapshot: Using recursive file lookup: {recursive_file_lookup}") + files_list = self._list_files(parent_dir, recursive=recursive_file_lookup) + files_with_path_info = [FilePathInfo(full_path=f.path, filename_with_version_path='/'.join(f.path.split('/')[dynamic_idx:])) for f in files_list] + + self.logger.debug(f"CDC Snapshot: Found {len(files_with_path_info)} files") + + # Extract version from filename and filter by latest_snapshot_version if provided + available_versions = [] + for file in files_with_path_info: + self.logger.debug(f"CDC Snapshot: Processing file: {file.filename_with_version_path}") + try: + version_info = self._extract_version_from_filename(file.filename_with_version_path, file_pattern_regex) + if version_info is None: + continue + + self.logger.debug(f"CDC Snapshot: Extracted version from filename: {version_info.formatted_value}. Raw version: {version_info.raw_value}") + + if latest_snapshot_version is None and self.source.startingVersion is not None and version_info.raw_value < self.source.startingVersion: + continue + + if latest_snapshot_version is not None and version_info.raw_value <= latest_snapshot_version: + continue + + # Dedupe by version (multiple fragments can share same version) + if any(v.raw_value == version_info.raw_value for v in available_versions): + continue + available_versions.append(version_info) + self.logger.debug(f"CDC Snapshot: Added version {version_info.formatted_value} to available versions") + + except ValueError as e: + self.logger.warning(f"CDC Snapshot: Skipping file '{file.filename_with_version_path}' - {e}") + continue + + return available_versions + + def _get_available_table_versions(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> List[VersionInfo]: + """Get list of available versions from table.""" + spark = pipeline_config.get_spark() + table_name = self.source.table + + self.logger.info(f"CDC Snapshot: Getting versions from table: {table_name}") + try: + df = spark.table(table_name) + except Exception as e: + self.logger.error(f"CDC Snapshot: Error getting versions from table: {e}") + raise + + # Get the version column + version_column = self.source.versionColumn + + # Check if the version column is a valid data type + valid_data_types = ["timestamp", "date", "integer","long"] + version_column_type = df.schema[version_column].dataType.typeName() + if version_column_type not in valid_data_types: + raise ValueError(f"Version column: {version_column}, type: {version_column_type}, is not a valid data type: {valid_data_types}") + if version_column_type != self.source.versionType: + raise ValueError(f"Version column: {version_column}, type: {version_column_type}, does not match specified version type: {self.source.versionType}") + + # Get the version values and filter by latest_snapshot_version if provided + if latest_snapshot_version is not None: + latest_version_info = VersionInfo( + raw_value=latest_snapshot_version, + version_type=self.source.versionType) + version_df = df.select(version_column).where(f"{version_column} > {latest_version_info.sql_formatted_value}").distinct() + else: + version_df = df.select(version_column).distinct() + + available_versions = [] + for row in version_df.collect(): + version = row[version_column] + + if version is None: + continue + + if self.source.startingVersion is not None and version < self.source.startingVersion: + self.logger.debug(f"CDC Snapshot: Skipping version {version} because it is less than the starting version {self.source.startingVersion}") + continue + + if latest_snapshot_version is not None and version <= latest_snapshot_version: + self.logger.debug(f"CDC Snapshot: Skipping version {version} because it is less than or equal to the latest snapshot version {latest_snapshot_version}") + continue + + version_info = VersionInfo( + raw_value=version, + version_type=self.source.versionType, + datetime_format=None + ) + + available_versions.append(version_info) + + self.logger.debug(f"CDC Snapshot: Found {len(available_versions)} available versions") + + return available_versions + + def _extract_version_from_filename(self, filename: str, file_pattern: str) -> Optional[VersionInfo]: + """Extract version from filename using pattern (regex or curly-brace; normalized to regex). + + Version is taken from all named groups starting with version_, concatenated in name order. + Curly-brace {version} is converted to (?P.+); {fragment} to (?P.*?). + """ + regex_pattern = self._path_to_regex_pattern(file_pattern) + match = re.match(regex_pattern, filename) + + if not match: + self.logger.debug(f"CDC Snapshot: No match for filename: {filename}") + self.logger.debug(f"CDC Snapshot: Regex pattern: {regex_pattern}") + return None + + version_str = self._get_version_string_from_match(match) + + if not version_str: + self.logger.debug(f"CDC Snapshot: No version_* groups in pattern for filename: {filename}") + return None + + self.logger.debug(f"CDC Snapshot: Version string match found: {version_str}") + + try: + if self.source.versionType == CDCSnapshotVersionTypes.TIMESTAMP: + raw_value = datetime.strptime(version_str, self.source.datetimeFormat) + else: + raw_value = int(version_str) + + return VersionInfo( + raw_value=raw_value, + version_type=self.source.versionType, + datetime_format=self.source.datetimeFormat if self.source.versionType == CDCSnapshotVersionTypes.TIMESTAMP else None, + micro_second_mask_length=self.source.microSecondMaskLength \ + if self.source.versionType == CDCSnapshotVersionTypes.TIMESTAMP and self.source.microSecondMaskLength else None + ) + except (ValueError, TypeError) as e: + self.logger.error(f"CDC Snapshot: Failed to parse version '{version_str}': {e}") + raise + + def _get_next_version(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> Optional[VersionInfo]: + """Get the next version to process.""" + # If no previous version exists yet + if latest_snapshot_version is None: + if not self.sorted_versions: + return None + version_info = self.sorted_versions[0] + self.logger.debug(f"CDC Snapshot: Using initial version: {version_info.formatted_value}") + return version_info + + # If a previous version exists, + # use bisect to find the first version greater than latest_snapshot_version + index = bisect.bisect_right(self.version_values, latest_snapshot_version) + if index < len(self.sorted_versions): + version_info = self.sorted_versions[index] + self.logger.debug(f"CDC Snapshot: Using next version: {version_info.formatted_value}") + return version_info + else: + self.logger.debug("CDC Snapshot: No more versions available") + return None + + def _read_snapshot_dataframe(self, version_info: VersionInfo, dataflow_config: DataFlowConfig, target_config_flags: Optional[List[str]] = None) -> Optional[DataFrame]: + """Read snapshot data into dataframe.""" + read_config = ReadConfig( + features=dataflow_config.features, + mode="batch", + target_config_flags=target_config_flags + ) + + if self.sourceType == CDCSnapshotSourceTypes.FILE: + path = self.source.path + pattern = self._path_to_regex_pattern(path) + dynamic_idx = self._get_dynamic_path_index(path) + path_parts = path.split('/') + parent_dir = '/'.join(path_parts[:dynamic_idx]) or '.' + # Anchor pattern so we only match full path (e.g. exclude customer.parquet/_SUCCESS) + file_pattern_regex = '/'.join(pattern.split('/')[dynamic_idx:]) + r'/?$' + regex_pattern = self._path_to_regex_pattern(file_pattern_regex) + + recursive_file_lookup = self.source.recursiveFileLookup + files_list = self._list_files(parent_dir, recursive=recursive_file_lookup) + + target_version = version_info.formatted_value + files_to_read: List[str] = [] + + for f in files_list: + full_path = f.path + relative_path = '/'.join(full_path.split('/')[dynamic_idx:]) + match = re.match(regex_pattern, relative_path) + if not match: + continue + version_str = self._get_version_string_from_match(match) + if version_str is None or version_str != target_version: + continue + files_to_read.append(full_path) + + if not files_to_read: + self.logger.warning(f"CDC Snapshot: No files found for version {target_version}") + return None + + self.logger.info(f"CDC Snapshot: Reading {len(files_to_read)} file(s) for version {target_version}") + schema_path = self.source.schemaPath + select_exp = self.source.selectExp + df = None + for file_path in sorted(files_to_read): + self.logger.debug(f"CDC Snapshot: Reading file: {file_path}") + file_df = SourceBatchFiles( + path=file_path, + format=self.source.format, + readerOptions=self.source.readerOptions, + schemaPath=schema_path, + selectExp=select_exp + ).read_source(read_config) + if df is None: + df = file_df + else: + df = df.union(file_df) + + # Apply filter if specified + if self.source.filter: + df = df.where(self.source.filter.replace("{version}", version_info.formatted_value)) + + elif self.sourceType == CDCSnapshotSourceTypes.TABLE: + table_parts = self.source.table.split(".") + if not 1 < len(table_parts) < 4: + raise ValueError(f"Invalid table name format: {self.source.table}. Accepted formats are: schema.table & database.schema.table") + + if len(table_parts) == 3: + database = f"{table_parts[0]}.{table_parts[1]}" + else: + # set to the specified target catalog by default + _pipeline_details = pipeline_config.get_pipeline_details() + database = f"{_pipeline_details.pipeline_catalog}.{table_parts[0]}" + + table = table_parts[-1] + + select_exp = self.source.selectExp + where_clause = [ + f"{self.source.versionColumn} = {version_info.sql_formatted_value}"] + + self.logger.info(f"CDC Snapshot: Reading table: {database}.{table} with where clause: {where_clause}") + df = SourceDelta( + database=database, + table=table, + whereClause=where_clause, + selectExp=select_exp + ).read_source(read_config) + + if self.source.deduplicateMode == DeduplicateMode.KEYS_ONLY: + df = self._deduplicate_by_keys(df) + self.logger.debug("CDC Snapshot: Applied deduplication by keys") + elif self.source.deduplicateMode == DeduplicateMode.FULL_ROW: + df = self._deduplicate_full_row(df) + self.logger.debug("CDC Snapshot: Applied full-row deduplication") + + return df diff --git a/src/dataflow/dataflow.py b/src/lakeflow_framework/dataflow/dataflow.py similarity index 99% rename from src/dataflow/dataflow.py rename to src/lakeflow_framework/dataflow/dataflow.py index 7cef5c8..2066531 100644 --- a/src/dataflow/dataflow.py +++ b/src/lakeflow_framework/dataflow/dataflow.py @@ -2,8 +2,8 @@ import pyspark.sql.types as T -from constants import SystemColumns -import pipeline_config +from lakeflow_framework.constants import SystemColumns +import lakeflow_framework.pipeline_config as pipeline_config from .cdc_snapshot import CDCSnapshotFlow from .dataflow_config import DataFlowConfig diff --git a/src/dataflow/dataflow_config.py b/src/lakeflow_framework/dataflow/dataflow_config.py similarity index 100% rename from src/dataflow/dataflow_config.py rename to src/lakeflow_framework/dataflow/dataflow_config.py diff --git a/src/dataflow/dataflow_spec.py b/src/lakeflow_framework/dataflow/dataflow_spec.py similarity index 90% rename from src/dataflow/dataflow_spec.py rename to src/lakeflow_framework/dataflow/dataflow_spec.py index c32b0c7..fbd9a72 100644 --- a/src/dataflow/dataflow_spec.py +++ b/src/lakeflow_framework/dataflow/dataflow_spec.py @@ -1,17 +1,17 @@ from dataclasses import dataclass, field from typing import Dict, List, Any -import utility - -from dataflow.cdc import CDCSettings -from dataflow.cdc_snapshot import CDCSnapshotSettings -from dataflow.enums import SourceType -from dataflow.expectations import DataQualityExpectations -from dataflow.features import Features -from dataflow.flow_group import FlowGroup -from dataflow.flows import BaseFlowWithViews -from dataflow.targets import TargetFactory -from dataflow.view import View +import lakeflow_framework.utility as utility + +from lakeflow_framework.dataflow.cdc import CDCSettings +from lakeflow_framework.dataflow.cdc_snapshot import CDCSnapshotSettings +from lakeflow_framework.dataflow.enums import SourceType +from lakeflow_framework.dataflow.expectations import DataQualityExpectations +from lakeflow_framework.dataflow.features import Features +from lakeflow_framework.dataflow.flow_group import FlowGroup +from lakeflow_framework.dataflow.flows import BaseFlowWithViews +from lakeflow_framework.dataflow.targets import TargetFactory +from lakeflow_framework.dataflow.view import View @dataclass diff --git a/src/dataflow/enums.py b/src/lakeflow_framework/dataflow/enums.py similarity index 100% rename from src/dataflow/enums.py rename to src/lakeflow_framework/dataflow/enums.py diff --git a/src/dataflow/expectations.py b/src/lakeflow_framework/dataflow/expectations.py similarity index 98% rename from src/dataflow/expectations.py rename to src/lakeflow_framework/dataflow/expectations.py index 4034975..afa8b01 100644 --- a/src/dataflow/expectations.py +++ b/src/lakeflow_framework/dataflow/expectations.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field from typing import Dict -import utility +import lakeflow_framework.utility as utility @dataclass(frozen=True) diff --git a/src/dataflow/features.py b/src/lakeflow_framework/dataflow/features.py similarity index 100% rename from src/dataflow/features.py rename to src/lakeflow_framework/dataflow/features.py diff --git a/src/dataflow/flow_group.py b/src/lakeflow_framework/dataflow/flow_group.py similarity index 100% rename from src/dataflow/flow_group.py rename to src/lakeflow_framework/dataflow/flow_group.py diff --git a/src/dataflow/flows/__init__.py b/src/lakeflow_framework/dataflow/flows/__init__.py similarity index 100% rename from src/dataflow/flows/__init__.py rename to src/lakeflow_framework/dataflow/flows/__init__.py diff --git a/src/dataflow/flows/append_sql.py b/src/lakeflow_framework/dataflow/flows/append_sql.py similarity index 100% rename from src/dataflow/flows/append_sql.py rename to src/lakeflow_framework/dataflow/flows/append_sql.py diff --git a/src/dataflow/flows/append_view.py b/src/lakeflow_framework/dataflow/flows/append_view.py similarity index 97% rename from src/dataflow/flows/append_view.py rename to src/lakeflow_framework/dataflow/flows/append_view.py index f3c7501..33f28c3 100644 --- a/src/dataflow/flows/append_view.py +++ b/src/lakeflow_framework/dataflow/flows/append_view.py @@ -2,7 +2,7 @@ from pyspark import pipelines as dp -import pipeline_config +import lakeflow_framework.pipeline_config as pipeline_config from ..dataflow_config import DataFlowConfig diff --git a/src/dataflow/flows/base.py b/src/lakeflow_framework/dataflow/flows/base.py similarity index 98% rename from src/dataflow/flows/base.py rename to src/lakeflow_framework/dataflow/flows/base.py index 1e7917a..6979669 100644 --- a/src/dataflow/flows/base.py +++ b/src/lakeflow_framework/dataflow/flows/base.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from typing import Dict, List, Optional -import pipeline_config +import lakeflow_framework.pipeline_config as pipeline_config from ..cdc import CDCSettings from ..cdc_snapshot import CDCSnapshotSettings diff --git a/src/dataflow/flows/factory.py b/src/lakeflow_framework/dataflow/flows/factory.py similarity index 100% rename from src/dataflow/flows/factory.py rename to src/lakeflow_framework/dataflow/flows/factory.py diff --git a/src/dataflow/flows/materialized_view.py b/src/lakeflow_framework/dataflow/flows/materialized_view.py similarity index 100% rename from src/dataflow/flows/materialized_view.py rename to src/lakeflow_framework/dataflow/flows/materialized_view.py diff --git a/src/dataflow/flows/merge.py b/src/lakeflow_framework/dataflow/flows/merge.py similarity index 100% rename from src/dataflow/flows/merge.py rename to src/lakeflow_framework/dataflow/flows/merge.py diff --git a/src/dataflow/operational_metadata.py b/src/lakeflow_framework/dataflow/operational_metadata.py similarity index 98% rename from src/dataflow/operational_metadata.py rename to src/lakeflow_framework/dataflow/operational_metadata.py index 4cc9887..d9e80ee 100644 --- a/src/dataflow/operational_metadata.py +++ b/src/lakeflow_framework/dataflow/operational_metadata.py @@ -5,7 +5,7 @@ import pyspark.sql.types as T from pyspark.sql import functions as F -import utility +import lakeflow_framework.utility as utility class MetadataMappingType(Enum): diff --git a/src/dataflow/quarantine.py b/src/lakeflow_framework/dataflow/quarantine.py similarity index 98% rename from src/dataflow/quarantine.py rename to src/lakeflow_framework/dataflow/quarantine.py index fd4112f..618b7b7 100644 --- a/src/dataflow/quarantine.py +++ b/src/lakeflow_framework/dataflow/quarantine.py @@ -3,9 +3,9 @@ from pyspark import pipelines as dp import pyspark.sql.functions as F -from constants import MetaDataColumnDefs, SystemColumns -import pipeline_config -import utility +from lakeflow_framework.constants import MetaDataColumnDefs, SystemColumns +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility from .enums import QuarantineMode, TableType, TargetType, Mode from .targets import ( diff --git a/src/dataflow/schema.py b/src/lakeflow_framework/dataflow/schema.py similarity index 97% rename from src/dataflow/schema.py rename to src/lakeflow_framework/dataflow/schema.py index a4b5009..eb5f827 100644 --- a/src/dataflow/schema.py +++ b/src/lakeflow_framework/dataflow/schema.py @@ -5,8 +5,8 @@ import pyspark.sql.types as T -import pipeline_config -import utility +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility @dataclass diff --git a/src/dataflow/sources/__init__.py b/src/lakeflow_framework/dataflow/sources/__init__.py similarity index 100% rename from src/dataflow/sources/__init__.py rename to src/lakeflow_framework/dataflow/sources/__init__.py diff --git a/src/dataflow/sources/base.py b/src/lakeflow_framework/dataflow/sources/base.py similarity index 98% rename from src/dataflow/sources/base.py rename to src/lakeflow_framework/dataflow/sources/base.py index 7b3428b..200a8b1 100644 --- a/src/dataflow/sources/base.py +++ b/src/lakeflow_framework/dataflow/sources/base.py @@ -8,9 +8,9 @@ from pyspark.sql import functions as F import pyspark.sql.types as T -from constants import MetaDataColumnDefs, SystemColumns -import pipeline_config -import utility +from lakeflow_framework.constants import MetaDataColumnDefs, SystemColumns +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility from ..enums import TargetConfigFlags from ..features import Features diff --git a/src/dataflow/sources/batch_files.py b/src/lakeflow_framework/dataflow/sources/batch_files.py similarity index 100% rename from src/dataflow/sources/batch_files.py rename to src/lakeflow_framework/dataflow/sources/batch_files.py diff --git a/src/dataflow/sources/cloud_files.py b/src/lakeflow_framework/dataflow/sources/cloud_files.py similarity index 100% rename from src/dataflow/sources/cloud_files.py rename to src/lakeflow_framework/dataflow/sources/cloud_files.py diff --git a/src/dataflow/sources/delta.py b/src/lakeflow_framework/dataflow/sources/delta.py similarity index 97% rename from src/dataflow/sources/delta.py rename to src/lakeflow_framework/dataflow/sources/delta.py index 962c872..778750f 100644 --- a/src/dataflow/sources/delta.py +++ b/src/lakeflow_framework/dataflow/sources/delta.py @@ -3,9 +3,9 @@ from pyspark.sql import DataFrame -import pipeline_config +import lakeflow_framework.pipeline_config as pipeline_config -from constants import SystemColumns +from lakeflow_framework.constants import SystemColumns from .base import BaseSourceWithSchemaOnRead, ReadConfig from ..enums import Mode diff --git a/src/dataflow/sources/delta_join.py b/src/lakeflow_framework/dataflow/sources/delta_join.py similarity index 100% rename from src/dataflow/sources/delta_join.py rename to src/lakeflow_framework/dataflow/sources/delta_join.py diff --git a/src/dataflow/sources/factory.py b/src/lakeflow_framework/dataflow/sources/factory.py similarity index 100% rename from src/dataflow/sources/factory.py rename to src/lakeflow_framework/dataflow/sources/factory.py diff --git a/src/dataflow/sources/kafka.py b/src/lakeflow_framework/dataflow/sources/kafka.py similarity index 100% rename from src/dataflow/sources/kafka.py rename to src/lakeflow_framework/dataflow/sources/kafka.py diff --git a/src/dataflow/sources/python.py b/src/lakeflow_framework/dataflow/sources/python.py similarity index 96% rename from src/dataflow/sources/python.py rename to src/lakeflow_framework/dataflow/sources/python.py index b7c9451..7663cb6 100644 --- a/src/dataflow/sources/python.py +++ b/src/lakeflow_framework/dataflow/sources/python.py @@ -4,9 +4,9 @@ from pyspark.sql import DataFrame import pyspark.sql.functions as F -from constants import MetaDataColumnDefs, SystemColumns -import pipeline_config -import utility +from lakeflow_framework.constants import MetaDataColumnDefs, SystemColumns +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility from .base import ReadConfig from ..operational_metadata import OperationalMetadataMixin diff --git a/src/dataflow/sources/sql.py b/src/lakeflow_framework/dataflow/sources/sql.py similarity index 100% rename from src/dataflow/sources/sql.py rename to src/lakeflow_framework/dataflow/sources/sql.py diff --git a/src/dataflow/sql.py b/src/lakeflow_framework/dataflow/sql.py similarity index 100% rename from src/dataflow/sql.py rename to src/lakeflow_framework/dataflow/sql.py diff --git a/src/dataflow/table_import.py b/src/lakeflow_framework/dataflow/table_import.py similarity index 96% rename from src/dataflow/table_import.py rename to src/lakeflow_framework/dataflow/table_import.py index e47c400..c2dc192 100644 --- a/src/dataflow/table_import.py +++ b/src/lakeflow_framework/dataflow/table_import.py @@ -1,9 +1,9 @@ from pyspark import pipelines as dp from typing import Any -from constants import SystemColumns -from dataflow.sources.factory import SourceFactory -import pipeline_config +from lakeflow_framework.constants import SystemColumns +from lakeflow_framework.dataflow.sources.factory import SourceFactory +import lakeflow_framework.pipeline_config as pipeline_config from pyspark.sql import functions as F from .cdc import CDCFlow, CDCSettings diff --git a/src/dataflow/table_migration.py b/src/lakeflow_framework/dataflow/table_migration.py similarity index 99% rename from src/dataflow/table_migration.py rename to src/lakeflow_framework/dataflow/table_migration.py index 711172e..056d348 100644 --- a/src/dataflow/table_migration.py +++ b/src/lakeflow_framework/dataflow/table_migration.py @@ -7,8 +7,8 @@ from pyspark.sql import types as T from pyspark.sql.utils import AnalysisException -import pipeline_config -import utility +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility from .cdc import CDCSettings from .dataflow_config import DataFlowConfig diff --git a/src/dataflow/targets/__init__.py b/src/lakeflow_framework/dataflow/targets/__init__.py similarity index 100% rename from src/dataflow/targets/__init__.py rename to src/lakeflow_framework/dataflow/targets/__init__.py diff --git a/src/dataflow/targets/base.py b/src/lakeflow_framework/dataflow/targets/base.py similarity index 99% rename from src/dataflow/targets/base.py rename to src/lakeflow_framework/dataflow/targets/base.py index aa31f75..dbcf65a 100644 --- a/src/dataflow/targets/base.py +++ b/src/lakeflow_framework/dataflow/targets/base.py @@ -6,8 +6,8 @@ from pyspark import pipelines as dp import pyspark.sql.types as T -import pipeline_config -import utility +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility from ..enums import TableType, TargetConfigFlags from ..features import Features diff --git a/src/dataflow/targets/delta_materialized_view.py b/src/lakeflow_framework/dataflow/targets/delta_materialized_view.py similarity index 100% rename from src/dataflow/targets/delta_materialized_view.py rename to src/lakeflow_framework/dataflow/targets/delta_materialized_view.py diff --git a/src/dataflow/targets/delta_streaming_table.py b/src/lakeflow_framework/dataflow/targets/delta_streaming_table.py similarity index 100% rename from src/dataflow/targets/delta_streaming_table.py rename to src/lakeflow_framework/dataflow/targets/delta_streaming_table.py diff --git a/src/dataflow/targets/factory.py b/src/lakeflow_framework/dataflow/targets/factory.py similarity index 100% rename from src/dataflow/targets/factory.py rename to src/lakeflow_framework/dataflow/targets/factory.py diff --git a/src/dataflow/targets/sink_custom_python.py b/src/lakeflow_framework/dataflow/targets/sink_custom_python.py similarity index 100% rename from src/dataflow/targets/sink_custom_python.py rename to src/lakeflow_framework/dataflow/targets/sink_custom_python.py diff --git a/src/dataflow/targets/sink_delta.py b/src/lakeflow_framework/dataflow/targets/sink_delta.py similarity index 100% rename from src/dataflow/targets/sink_delta.py rename to src/lakeflow_framework/dataflow/targets/sink_delta.py diff --git a/src/dataflow/targets/sink_foreach_batch.py b/src/lakeflow_framework/dataflow/targets/sink_foreach_batch.py similarity index 99% rename from src/dataflow/targets/sink_foreach_batch.py rename to src/lakeflow_framework/dataflow/targets/sink_foreach_batch.py index b0c90a9..4a9316e 100644 --- a/src/dataflow/targets/sink_foreach_batch.py +++ b/src/lakeflow_framework/dataflow/targets/sink_foreach_batch.py @@ -3,7 +3,7 @@ from pyspark import pipelines as dp -import utility +import lakeflow_framework.utility as utility from .base import BaseSink from ..enums import SinkType from ..sql import SqlMixin diff --git a/src/dataflow/targets/sink_kafka.py b/src/lakeflow_framework/dataflow/targets/sink_kafka.py similarity index 100% rename from src/dataflow/targets/sink_kafka.py rename to src/lakeflow_framework/dataflow/targets/sink_kafka.py diff --git a/src/dataflow/targets/staging_table.py b/src/lakeflow_framework/dataflow/targets/staging_table.py similarity index 100% rename from src/dataflow/targets/staging_table.py rename to src/lakeflow_framework/dataflow/targets/staging_table.py diff --git a/src/dataflow/view.py b/src/lakeflow_framework/dataflow/view.py similarity index 98% rename from src/dataflow/view.py rename to src/lakeflow_framework/dataflow/view.py index 18effe6..615e978 100644 --- a/src/dataflow/view.py +++ b/src/lakeflow_framework/dataflow/view.py @@ -4,7 +4,7 @@ from pyspark import pipelines as dp from pyspark.sql import DataFrame -import pipeline_config +import lakeflow_framework.pipeline_config as pipeline_config from .dataflow_config import DataFlowConfig from .enums import SourceType diff --git a/src/dataflow_spec_builder/__init__.py b/src/lakeflow_framework/dataflow_spec_builder/__init__.py similarity index 100% rename from src/dataflow_spec_builder/__init__.py rename to src/lakeflow_framework/dataflow_spec_builder/__init__.py diff --git a/src/dataflow_spec_builder/dataflow_spec_builder.py b/src/lakeflow_framework/dataflow_spec_builder/dataflow_spec_builder.py similarity index 99% rename from src/dataflow_spec_builder/dataflow_spec_builder.py rename to src/lakeflow_framework/dataflow_spec_builder/dataflow_spec_builder.py index 8199310..01c31e3 100644 --- a/src/dataflow_spec_builder/dataflow_spec_builder.py +++ b/src/lakeflow_framework/dataflow_spec_builder/dataflow_spec_builder.py @@ -4,13 +4,13 @@ from typing import Dict, List, Optional from concurrent.futures import ThreadPoolExecutor, as_completed -from constants import ( +from lakeflow_framework.constants import ( FrameworkPaths, PipelineBundlePaths, SupportedSpecFormat, PipelineBundleSuffixesJson, PipelineBundleSuffixesYaml ) -from dataflow.dataflow_spec import DataflowSpec -import pipeline_config -from secrets_manager import SecretsManager -import utility +from lakeflow_framework.dataflow.dataflow_spec import DataflowSpec +import lakeflow_framework.pipeline_config as pipeline_config +from lakeflow_framework.secrets_manager import SecretsManager +import lakeflow_framework.utility as utility from .expectations_builder import DataQualityExpectationBuilder from .spec_mapper import SpecMapper diff --git a/src/dataflow_spec_builder/expectations_builder.py b/src/lakeflow_framework/dataflow_spec_builder/expectations_builder.py similarity index 96% rename from src/dataflow_spec_builder/expectations_builder.py rename to src/lakeflow_framework/dataflow_spec_builder/expectations_builder.py index ebe5ccf..946e1ea 100644 --- a/src/dataflow_spec_builder/expectations_builder.py +++ b/src/lakeflow_framework/dataflow_spec_builder/expectations_builder.py @@ -2,10 +2,10 @@ import os from typing import Dict -import utility +import lakeflow_framework.utility as utility -from constants import SupportedSpecFormat -from dataflow.expectations import DataQualityExpectations, ExpectationType +from lakeflow_framework.constants import SupportedSpecFormat +from lakeflow_framework.dataflow.expectations import DataQualityExpectations, ExpectationType class DataQualityExpectationBuilder: diff --git a/src/dataflow_spec_builder/spec_mapper.py b/src/lakeflow_framework/dataflow_spec_builder/spec_mapper.py similarity index 98% rename from src/dataflow_spec_builder/spec_mapper.py rename to src/lakeflow_framework/dataflow_spec_builder/spec_mapper.py index 724350e..d49feb5 100644 --- a/src/dataflow_spec_builder/spec_mapper.py +++ b/src/lakeflow_framework/dataflow_spec_builder/spec_mapper.py @@ -2,10 +2,10 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Dict, Tuple, Optional, Any -from config_resolver import resolve_framework_config_dir -from constants import FrameworkPaths -import pipeline_config -import utility +from lakeflow_framework.config_resolver import resolve_framework_config_dir +from lakeflow_framework.constants import FrameworkPaths +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility class SpecMapper: diff --git a/src/dataflow_spec_builder/template_processor.py b/src/lakeflow_framework/dataflow_spec_builder/template_processor.py similarity index 98% rename from src/dataflow_spec_builder/template_processor.py rename to src/lakeflow_framework/dataflow_spec_builder/template_processor.py index b092ab0..1fd6e60 100644 --- a/src/dataflow_spec_builder/template_processor.py +++ b/src/lakeflow_framework/dataflow_spec_builder/template_processor.py @@ -3,9 +3,9 @@ import re from typing import Dict, Any -from constants import FrameworkPaths, PipelineBundlePaths, SupportedSpecFormat -import pipeline_config -import utility +from lakeflow_framework.constants import FrameworkPaths, PipelineBundlePaths, SupportedSpecFormat +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility class TemplateProcessor: diff --git a/src/dataflow_spec_builder/transformer/__init__.py b/src/lakeflow_framework/dataflow_spec_builder/transformer/__init__.py similarity index 100% rename from src/dataflow_spec_builder/transformer/__init__.py rename to src/lakeflow_framework/dataflow_spec_builder/transformer/__init__.py diff --git a/src/dataflow_spec_builder/transformer/base.py b/src/lakeflow_framework/dataflow_spec_builder/transformer/base.py similarity index 96% rename from src/dataflow_spec_builder/transformer/base.py rename to src/lakeflow_framework/dataflow_spec_builder/transformer/base.py index c250b2e..e4a313d 100644 --- a/src/dataflow_spec_builder/transformer/base.py +++ b/src/lakeflow_framework/dataflow_spec_builder/transformer/base.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from typing import Dict, List, Union -import pipeline_config +import lakeflow_framework.pipeline_config as pipeline_config class BaseSpecTransformer(ABC): diff --git a/src/dataflow_spec_builder/transformer/factory.py b/src/lakeflow_framework/dataflow_spec_builder/transformer/factory.py similarity index 100% rename from src/dataflow_spec_builder/transformer/factory.py rename to src/lakeflow_framework/dataflow_spec_builder/transformer/factory.py diff --git a/src/dataflow_spec_builder/transformer/flow.py b/src/lakeflow_framework/dataflow_spec_builder/transformer/flow.py similarity index 100% rename from src/dataflow_spec_builder/transformer/flow.py rename to src/lakeflow_framework/dataflow_spec_builder/transformer/flow.py diff --git a/src/dataflow_spec_builder/transformer/materialized_views.py b/src/lakeflow_framework/dataflow_spec_builder/transformer/materialized_views.py similarity index 98% rename from src/dataflow_spec_builder/transformer/materialized_views.py rename to src/lakeflow_framework/dataflow_spec_builder/transformer/materialized_views.py index cd9ffa4..8852cd3 100644 --- a/src/dataflow_spec_builder/transformer/materialized_views.py +++ b/src/lakeflow_framework/dataflow_spec_builder/transformer/materialized_views.py @@ -1,7 +1,7 @@ from typing import Dict, List from .base import BaseSpecTransformer -from dataflow.enums import FlowType, Mode, SourceType, TargetType, TableType +from lakeflow_framework.dataflow.enums import FlowType, Mode, SourceType, TargetType, TableType class MaterializedViewSpecTransformer(BaseSpecTransformer): diff --git a/src/dataflow_spec_builder/transformer/standard.py b/src/lakeflow_framework/dataflow_spec_builder/transformer/standard.py similarity index 98% rename from src/dataflow_spec_builder/transformer/standard.py rename to src/lakeflow_framework/dataflow_spec_builder/transformer/standard.py index 612fd6b..0c7b474 100644 --- a/src/dataflow_spec_builder/transformer/standard.py +++ b/src/lakeflow_framework/dataflow_spec_builder/transformer/standard.py @@ -1,7 +1,7 @@ from typing import Dict from .base import BaseSpecTransformer -from dataflow.enums import FlowType, Mode, TableType +from lakeflow_framework.dataflow.enums import FlowType, Mode, TableType class StandardSpecTransformer(BaseSpecTransformer): diff --git a/src/lakeflow_framework/dlt_pipeline_builder.py b/src/lakeflow_framework/dlt_pipeline_builder.py new file mode 100644 index 0000000..5e66d75 --- /dev/null +++ b/src/lakeflow_framework/dlt_pipeline_builder.py @@ -0,0 +1,464 @@ +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +import json +import os + +from pyspark import pipelines as dp +from pyspark.dbutils import DBUtils +import pyspark.sql.types as T +from pyspark.sql import SparkSession +from typing import Dict, Any + +from lakeflow_framework.config_resolver import load_framework_config, resolve_framework_config_path +from lakeflow_framework.constants import ( + FrameworkPaths, FrameworkSettings, PipelineBundlePaths, DLTPipelineSettingKeys, SupportedSpecFormat +) +from lakeflow_framework.dataflow import DataFlow +from lakeflow_framework.dataflow_spec_builder import DataflowSpecBuilder +from lakeflow_framework.bundle_loader import register_bundle_sys_paths, run_init_scripts +from lakeflow_framework.pipeline_details import PipelineDetails +from lakeflow_framework.secrets_manager import SecretsManager +from lakeflow_framework.substitution_manager import SubstitutionManager + +import lakeflow_framework.logger as pipeline_logger +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility + + +class DLTPipelineBuilder: + """ + Initializes a dataflow in a Spark Declarative Pipeline based on the pipeline configuration and dataflow specifications. + + Args: + spark (SparkSession): The Spark session to use for the pipeline. + dbutils (DBUtils): The DBUtils to use for the pipeline. + + Attributes: + spark (SparkSession): The Spark session to use for the pipeline. + dbutils (DBUtils): The DBUtils to use for the pipeline. + + logger (Logger): The logger to use for the pipeline. + context (NotebookContext): The notebook context to use for the pipeline. + token (str): The token to use for the pipeline. + + pipeline_config (Dict[str, Any]): The pipeline configuration to use for the pipeline. + framework_path (str): The path to the framework to use for the pipeline. + bundle_path (str): The path to the bundle to use for the pipeline. + dataflow_path (str): The path to the dataflow to use for the pipeline. + workspace_host (str): The host to use for the pipeline. + + dataflow_specs (List[DataflowSpec]): The dataflow specifications to use for the pipeline. + dataflow_spec_filters (Dict[str, Any]): The filters to use for the dataflow specifications. + + pipeline_details (PipelineDetails): The pipeline details to use for the pipeline. + mandatory_table_properties (Dict[str, Any]): The mandatory table properties to use for the pipeline. + operational_metadata_schema (StructType): The operational metadata schema to use for the pipeline. + substitution_manager (SubstitutionManager): The substitution manager to use for the pipeline. + + Methods: + initialize_pipeline(): Initializes a dataflow in a Spark Declarative Pipeline. + """ + + MANDATORY_CONFIG_PARAMS = [ + DLTPipelineSettingKeys.BUNDLE_SOURCE_PATH, + DLTPipelineSettingKeys.FRAMEWORK_SOURCE_PATH, + DLTPipelineSettingKeys.WORKSPACE_HOST + ] + + def __init__(self, spark: SparkSession, dbutils: DBUtils): + """Initialize the pipeline builder with Spark session and utilities.""" + # Initialize Spark context + self.spark = spark + self.dbutils = dbutils + self.context = self.dbutils.entry_point.getDbutils().notebook().getContext() + self.token = self.context.apiToken().get() + + self.pipeline_bundle_spec_format = "json" # Default to JSON format + self.dataflow_specs = [] + self.dataflow_spec_filters = {} + self.mandatory_table_properties = {} + self.operational_metadata_schema = None + self.pipeline_config = {} + self.secrets_manager = None + self.substitution_manager = None + self.driver_cores = os.cpu_count() + self.default_max_workers = self.driver_cores - 1 if self.driver_cores else 1 + + # Bootstrap mandatory paths and register sys.path + self._load_mandatory_paths() + register_bundle_sys_paths(self.framework_path, self.bundle_path) + + # Resolve the main logger + log_level = self.spark.conf.get(DLTPipelineSettingKeys.LOG_LEVEL, "INFO").upper() + self.logger = pipeline_logger.resolve_pipeline_logger( + spark=self.spark, + dbutils=self.dbutils, + framework_path=self.framework_path, + bundle_path=self.bundle_path, + spark_log_level=log_level, + ) + self.logger.info("Initializing Pipeline...") + self.logger.info("Logical cores (threads): %s", self.driver_cores) + self.logger.info("Max workers: %s", self.default_max_workers) + + # Initialize core singletons + pipeline_config.initialize_core( + spark=self.spark, + dbutils=self.dbutils, + logger=self.logger, + ) + + # Load configurations and initialize components + self._init_configurations() + self._init_pipeline_components() + + def _load_mandatory_paths(self) -> None: + """Load mandatory Spark conf paths required before logger and config init.""" + config_values = { + param: self.spark.conf.get(param, None) + for param in self.MANDATORY_CONFIG_PARAMS + } + + missing_params = [param for param, value in config_values.items() if not value] + if missing_params: + raise ValueError(f"Missing mandatory config parameters: {missing_params}") + + self.bundle_path = config_values[DLTPipelineSettingKeys.BUNDLE_SOURCE_PATH] + self.framework_path = config_values[DLTPipelineSettingKeys.FRAMEWORK_SOURCE_PATH] + self._framework_config_path = resolve_framework_config_path(self.framework_path) + self.workspace_host = config_values[DLTPipelineSettingKeys.WORKSPACE_HOST] + + def _init_configurations(self) -> None: + """Load and validate all necessary configurations.""" + # Load optional parameters + ignore_validation_errors = self.spark.conf.get( + DLTPipelineSettingKeys.PIPELINE_IGNORE_VALIDATION_ERRORS, "false" + ) + self.ignore_validation_errors = (ignore_validation_errors.lower() == "true") + + # Load pipeline details + self.logger.info("Loading Pipeline Details...") + self.pipeline_details = PipelineDetails( + pipeline_id=self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_ID, None), + pipeline_catalog=self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_CATALOG, None), + pipeline_schema=( + self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_SCHEMA, None) + or self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_TARGET, None) + ), + pipeline_layer=self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_LAYER, None), + start_utc_timestamp=datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S.%f'), + workspace_env=self.spark.conf.get(DLTPipelineSettingKeys.BUNDLE_TARGET, None), + logical_env=self.spark.conf.get(DLTPipelineSettingKeys.LOGICAL_ENV, "") + ) + self.logger.info("Pipeline Details: %s", json.dumps(self.pipeline_details.__dict__, indent=4)) + + # Initialize pipeline details singleton + pipeline_config.initialize_pipeline_details(self.pipeline_details) + + # Get the pipeline update id via event hook (only method at the moment) + # This only gets populated post initiliazation. Currently retrieved in operational_metadata.py + @dp.on_event_hook + def update_id_hook(event): + if self.spark.conf.get("pipeline.pipeline_update_id", "") == "": + update_id = event.get("origin", {}).get("update_id", "") + if update_id: + self.spark.conf.set("pipeline.pipeline_update_id", update_id) + + # Load dataflow filters + self.logger.info("Loading Dataflow Filters...") + self.dataflow_spec_filters = { + "data_flow_ids": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_DATA_FLOW_ID, None), + "data_flow_groups": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_DATA_FLOW_GROUP, None), + "flow_group_ids": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_FLOW_GROUP_ID, None), + "target_tables": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILTER_TARGET_TABLE, None), + "files": self.spark.conf.get(DLTPipelineSettingKeys.PIPELINE_FILE_FILTER, None), + } + + def _init_pipeline_components(self) -> None: + """Initialize all pipeline components.""" + + # Load and merge configurations + self._load_merged_config() + + # Initialize substitution manager + self._init_substitution_manager() + + # Initialize table migration state volume path (after substitution manager so tokens are resolved) + table_migration_path = self.pipeline_config.get("table_migration_state_volume_path", None) + if table_migration_path: + table_migration_path = self.substitution_manager.substitute_string(table_migration_path) + pipeline_config.initialize_table_migration(table_migration_path) + + # Initialize secrets manager + self._init_secrets_manager() + + # Initialize dataflow specifications + self._init_dataflow_specs() + + # Setup operational metadata + self._setup_operational_metadata() + + # Apply Spark configurations + self._apply_spark_config() + + def _load_framework_global_config(self) -> Dict[str, Any]: + """Load the framework global config, deep-merging any src/local/config/ overrides.""" + from lakeflow_framework.config_resolver import load_framework_config + + override_dir = os.path.join(self.framework_path, FrameworkPaths.CONFIG_OVERRIDE_PATH) + is_override = any( + os.path.exists(os.path.join(override_dir, n)) for n in FrameworkPaths.GLOBAL_CONFIG + ) + + if is_override: + import warnings + warnings.warn( + f"{FrameworkPaths.CONFIG_OVERRIDE_PATH} is deprecated (v0.13.0) and will be " + f"removed in v1.0.0. Migrate your overrides to {FrameworkPaths.LOCAL_CONFIG_PATH} " + "— only the keys you want to change are needed (sparse files are supported). " + "See the framework configuration documentation for migration steps.", + DeprecationWarning, + stacklevel=2, + ) + self._active_config_path = FrameworkPaths.CONFIG_OVERRIDE_PATH + else: + self._active_config_path = FrameworkPaths.CONFIG_PATH + + self.logger.info( + "Retrieving Global Framework Config From: %s", + os.path.join(self.framework_path, self._active_config_path), + ) + result = load_framework_config( + FrameworkPaths.GLOBAL_CONFIG, + self.framework_path, + config_path=self._active_config_path, + fail_on_not_exists=True, + ) + self.logger.info("Global Framework Config (resolved): %s", result) + return result + + def _load_pipeline_bundle_global_config_file(self) -> Dict[str, Any]: + """Load a global config file""" + global_config_paths = [ + os.path.join(self.bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, path) for path in PipelineBundlePaths.GLOBAL_CONFIG_FILE + ] + + # Check if more than one global config exists + existing_configs = [path for path in global_config_paths if os.path.exists(path)] + if len(existing_configs) > 1: + raise ValueError(f"Multiple pipeline global config files found. Only one is allowed: {existing_configs}") + + if not existing_configs: + return {} + + pipeline_config_path = existing_configs[0] + self.logger.info("Retrieving Pipeline Global Config From: %s", pipeline_config_path) + return utility.load_config_file_auto(pipeline_config_path, False) or {} + + def _load_merged_config(self) -> None: + """Load and merge global and pipeline-specific configurations.""" + self.pipeline_config = self._load_framework_global_config() + pipeline_bundle_config = self._load_pipeline_bundle_global_config_file() + + # Initialize pipeline bundle spec format + self._init_pipeline_bundle_spec_format(pipeline_bundle_config) + + # Merge pipeline bundle config + if pipeline_bundle_config: + self.pipeline_config.update(pipeline_bundle_config) + + self.mandatory_table_properties = self.pipeline_config.get("mandatory_table_properties", {}) + + # Initialize mandatory table properties singleton + pipeline_config.initialize_mandatory_table_properties(self.mandatory_table_properties) + + # Initialize mandatory configuration singleton + pipeline_config.initialize_mandatory_configuration() + + def _init_pipeline_bundle_spec_format(self, pipeline_bundle_config: Dict[str, Any]) -> None: + """Initialize the pipeline bundle spec format.""" + valid_formats = [fmt.value for fmt in SupportedSpecFormat] + + # Process global format configuration + global_format_dict = self.pipeline_config.pop("pipeline_bundle_spec_format", None) + if global_format_dict: + self.logger.info("Global pipeline bundle spec format: %s", global_format_dict) + global_format = global_format_dict.get("format", "json") + + if global_format not in valid_formats: + raise ValueError(f"Invalid pipeline bundle spec format: {global_format}. Valid formats are: {valid_formats}") + + self.pipeline_bundle_spec_format = global_format + allow_override = global_format_dict.get("allow_override", False) + else: + allow_override = False + + # Process pipeline-specific format configuration + pipeline_format_dict = pipeline_bundle_config.pop("pipeline_bundle_spec_format", None) + if pipeline_format_dict: + self.logger.info("Pipeline bundle spec format: %s", pipeline_format_dict) + pipeline_format = pipeline_format_dict.get("format", None) + + if pipeline_format and pipeline_format not in valid_formats: + raise ValueError(f"Invalid pipeline bundle spec format: {pipeline_format}. Valid formats are: {valid_formats}") + + if pipeline_format and pipeline_format != self.pipeline_bundle_spec_format and not allow_override: + raise ValueError(f"Pipeline bundle spec format has been set at global framework level as {self.pipeline_bundle_spec_format}. Override has been disabled.") + + if pipeline_format and allow_override: + self.pipeline_bundle_spec_format = pipeline_format + + self.logger.info("Pipeline bundle spec format: %s", self.pipeline_bundle_spec_format) + + def _init_substitution_manager(self) -> None: + """Initialize the substitution manager.""" + self.logger.info("Initializing Substitution Manager...") + + workspace_env = self.pipeline_details.workspace_env or "" + + # Build framework substitutions paths + framework_subs_paths = [ + os.path.join(self.framework_path, self._framework_config_path, workspace_env + path) + for path in FrameworkPaths.GLOBAL_SUBSTITUTIONS + ] + self.logger.info("Framework substitutions paths: %s", framework_subs_paths) + + # Build pipeline substitutions paths + suffixes = utility.get_format_suffixes(self.pipeline_bundle_spec_format, "substitutions") + pipeline_subs_paths = [os.path.join( + self.bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, workspace_env + suffix + ) for suffix in suffixes + ] + self.logger.info("Pipeline substitutions paths: %s", pipeline_subs_paths) + + self.substitution_manager = SubstitutionManager( + framework_substitutions_paths=framework_subs_paths, + pipeline_substitutions_paths=pipeline_subs_paths, + additional_tokens=self.pipeline_details.__dict__ + ) + self.logger.debug("Loaded substitution config: %s", self.substitution_manager._substitutions_config) + + # Initialize substitution manager singleton + pipeline_config.initialize_substitution_manager(self.substitution_manager) + + def _init_secrets_manager(self) -> None: + """Initialize the secrets manager.""" + self.logger.info("Initializing Secrets Manager...") + + workspace_env = self.pipeline_details.workspace_env or "" + + # Build framework secrets paths + framework_secrets_config_paths = [ + os.path.join(self.framework_path, self._framework_config_path, workspace_env + path) + for path in FrameworkPaths.GLOBAL_SECRETS + ] + + # Build pipeline secrets paths + suffixes = utility.get_format_suffixes(self.pipeline_bundle_spec_format, "secrets") + pipeline_secrets_configs_paths = [os.path.join( + self.bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, workspace_env + suffix + ) for suffix in suffixes + ] + + secrets_validator_path = os.path.join( + self.framework_path, FrameworkPaths.SECRETS_SCHEMA_PATH + ) + + self.secrets_manager = SecretsManager( + json_validation_schema_path=secrets_validator_path, + framework_secrets_config_paths=framework_secrets_config_paths, + pipeline_secrets_config_paths=pipeline_secrets_configs_paths + ) + + def _init_dataflow_specs(self) -> None: + """Initialize dataflow specifications.""" + self.logger.info("Initializing Dataflow Spec Builder...") + + dataflow_spec_version = self.pipeline_config.get("dataflow_spec_version", None) + dataflow_spec_builder_max_workers = self.pipeline_config.get( + FrameworkSettings.OVERRIDE_MAX_WORKERS_KEY, + self.default_max_workers + ) + + self.dataflow_specs = DataflowSpecBuilder( + bundle_path=self.bundle_path, + framework_path=self.framework_path, + filters=self.dataflow_spec_filters, + secrets_manager=self.secrets_manager, + ignore_validation_errors=self.ignore_validation_errors, + dataflow_spec_version=dataflow_spec_version, + max_workers=dataflow_spec_builder_max_workers, + spec_file_format=self.pipeline_bundle_spec_format + ).build() + + if not self.dataflow_specs: + raise ValueError(f"No dataflow specifications found in: {self.bundle_path}") + + def _setup_operational_metadata(self) -> None: + """Set up operational metadata schema.""" + self.logger.info("Initializing Operational Metadata...") + layer = self.pipeline_details.pipeline_layer + if not layer: + self.logger.info("Layer not set in pipeline, skipping operational metadata...") + self.operational_metadata_schema = None + return + + self.logger.info("Operational Metadata: layer set to %s", layer) + config_name = f"operational_metadata_{layer}.json" + metadata_json = load_framework_config( + config_name, self.framework_path, + config_path=getattr(self, "_active_config_path", FrameworkPaths.CONFIG_PATH), + fail_on_not_exists=False, + ) + self.logger.info("Operational Metadata config (resolved): %s", metadata_json) + self.operational_metadata_schema = ( + T.StructType.fromJson(metadata_json) if metadata_json else None + ) + + # Initialize operational metadata schema singleton + pipeline_config.initialize_operational_metadata_schema(self.operational_metadata_schema) + + def _apply_spark_config(self) -> None: + """Apply Spark configuration settings.""" + spark_config = self.pipeline_config.get("spark_config", {}) + if spark_config: + self.logger.info("Initializing Spark Configs...") + for prop, value in spark_config.items(): + self.logger.info("Set Spark Config: %s = %s", prop, value) + self.spark.conf.set(prop, value) + + def initialize_pipeline(self) -> None: + """Initialize the Spark Declarative Pipeline.""" + def create_dataflow(spec): + """Create a dataflow from a specification.""" + return DataFlow(dataflow_spec=spec).create_dataflow() + + run_init_scripts(self.framework_path, self.bundle_path, "pre", self.logger) + + self.logger.info("Initializing Pipeline...") + pipeline_builder_threading_disabled = self.pipeline_config.get( + FrameworkSettings.PIPELINE_BUILDER_DISABLE_THREADING_KEY, + True + ) + + self.logger.info("Processing Dataflow Specs...") + if pipeline_builder_threading_disabled: + self.logger.info("Pipeline Builder Threading Disabled, creating dataflows sequentially...") + for spec in self.dataflow_specs: + create_dataflow(spec) + else: + pipeline_builder_max_workers = self.pipeline_config.get( + FrameworkSettings.OVERRIDE_MAX_WORKERS_KEY, + self.default_max_workers + ) + + with ThreadPoolExecutor(max_workers=pipeline_builder_max_workers) as executor: + self.logger.info("Pipeline Builder Threading Enabled. Max Workers: %s", pipeline_builder_max_workers) + futures = [ + executor.submit(create_dataflow, spec) + for spec in self.dataflow_specs + ] + for future in futures: + future.result() + + run_init_scripts(self.framework_path, self.bundle_path, "post", self.logger) diff --git a/src/lakeflow_framework/logger.py b/src/lakeflow_framework/logger.py new file mode 100644 index 0000000..8c2127e --- /dev/null +++ b/src/lakeflow_framework/logger.py @@ -0,0 +1,319 @@ +""" +Pluggable pipeline logging: default stdout logger, optional custom logger from logger.json, +and CompositeLogger (custom primary + framework stdout mirror). +""" + +from __future__ import annotations + +import importlib +import importlib.util +import logging +import os +import sys +from copy import deepcopy +from typing import TYPE_CHECKING, Any, Dict, Optional + +from lakeflow_framework.constants import FrameworkPaths, PipelineBundlePaths + +if TYPE_CHECKING: + from pyspark.dbutils import DBUtils + from pyspark.sql import SparkSession + +_REQUIRED_LOG_METHODS = ("debug", "info", "warning", "error", "critical", "exception") + + +class CustomLoggerConfigWarning(UserWarning): + """Emitted when the custom logger fails to initialise and the framework falls back to the default logger.""" + +_DEFAULT_LOGGER_CONFIG: Dict[str, Any] = { + "enabled": False, + "factory_args": {}, + "level": "INFO", + "mirror_to_stdout": False, + "allow_pipeline_logger_override": False, +} + + +def create_default_logger(logger_name: str, log_level: str = "INFO") -> logging.Logger: + """Create a framework default logger writing to stdout.""" + logger = logging.getLogger(logger_name) + level = getattr(logging, log_level.upper(), None) or logging.INFO + logger.setLevel(level) + + if logger.hasHandlers(): + logger.handlers.clear() + + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + logger.addHandler(handler) + return logger + + +def get_effective_logger_config( + framework_config: Optional[Dict[str, Any]], + bundle_config: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """ + Merge framework and bundle logger.json. + + allow_pipeline_logger_override (framework only, default false): + false -> framework wins on conflicts + true -> bundle wins on conflicts + """ + framework = deepcopy(framework_config or {}) + bundle = deepcopy(bundle_config or {}) + bundle.pop("allow_pipeline_logger_override", None) + allow_override = bool(framework.get("allow_pipeline_logger_override", False)) + + merged = deepcopy(_DEFAULT_LOGGER_CONFIG) + + if allow_override: + # Bundle config wins: apply framework first, then bundle on top. + merged.update(framework) + merged.update(bundle) + merged["factory_args"] = { + **(framework.get("factory_args") or {}), + **(bundle.get("factory_args") or {}), + } + else: + # Framework config wins: bundle config is ignored entirely. + merged.update(framework) + merged["factory_args"] = { + **(framework.get("factory_args") or {}), + } + + merged["allow_pipeline_logger_override"] = allow_override + return merged + + +def _validate_logger_api(logger_obj: Any) -> None: + for name in _REQUIRED_LOG_METHODS: + method = getattr(logger_obj, name, None) + if not callable(method): + raise TypeError( + f"Custom logger must provide a callable '{name}' method; got {type(method).__name__}" + ) + + +def _resolve_log_level(config: Dict[str, Any], spark_log_level: Optional[str]) -> str: + if spark_log_level: + return spark_log_level.upper() + level = config.get("level", "INFO") + return str(level).upper() if level else "INFO" + + +class CompositeLogger: + """Custom logger as primary; framework default logger mirrors to stdout.""" + + def __init__(self, primary: Any, mirror: logging.Logger) -> None: + self._primary = primary + self._mirror = mirror + + def close(self) -> None: + closer = getattr(self._primary, "close", None) + if callable(closer): + closer() + + def _call_primary(self, method_name: str, message: str, args: tuple, kwargs: Dict[str, Any]) -> None: + method = getattr(self._primary, method_name) + if kwargs: + method(message, **kwargs) + elif args: + method(message, *args) + else: + method(message) + + def _call_mirror(self, method_name: str, message: str, args: tuple, kwargs: Dict[str, Any]) -> None: + method = getattr(self._mirror, method_name) + if kwargs: + method(message, *args, **kwargs) + elif args: + method(message, *args) + else: + method(message) + + def _log(self, method_name: str, message: str, *args: Any, **kwargs: Any) -> None: + try: + self._call_primary(method_name, message, args, kwargs) + except Exception as exc: + self._mirror.warning( + "Custom logger failed in %s: %s", + method_name, + exc, + exc_info=True, + ) + try: + self._call_mirror(method_name, message, args, kwargs) + except Exception: + pass + + def debug(self, message: str, *args: Any, **kwargs: Any) -> None: + self._log("debug", message, *args, **kwargs) + + def info(self, message: str, *args: Any, **kwargs: Any) -> None: + self._log("info", message, *args, **kwargs) + + def warning(self, message: str, *args: Any, **kwargs: Any) -> None: + self._log("warning", message, *args, **kwargs) + + def error(self, message: str, *args: Any, **kwargs: Any) -> None: + self._log("error", message, *args, **kwargs) + + def critical(self, message: str, *args: Any, **kwargs: Any) -> None: + self._log("critical", message, *args, **kwargs) + + def exception(self, message: str, *args: Any, **kwargs: Any) -> None: + self._log("exception", message, *args, **kwargs) + + +def _get_logger_config_from_file(config_path: str) -> Dict[str, Any]: + """ + Load logger.json from disk. Missing or invalid files return {} and log a warning + via a one-off default logger when possible. + """ + if not config_path or not os.path.isfile(config_path): + return {} + + try: + from lakeflow_framework.utility import get_json_from_file + + data = get_json_from_file(config_path, fail_on_not_exists=False) + return data if isinstance(data, dict) else {} + except Exception as exc: + bootstrap = create_default_logger("lakeflowframework.config") + bootstrap.warning( + "Failed to load logger config from %s: %s. Using empty logger config for this source.", + config_path, + exc, + ) + return {} + + +def _get_framework_logger_config(framework_path: str) -> Dict[str, Any]: + """Load framework logger.json configuration. + + Logger defaults are defined in-code as ``_DEFAULT_LOGGER_CONFIG`` — there is no + ``config/default/logger.json`` on disk. This function therefore returns only the + *override keys* that should be merged on top of those defaults inside + ``merge_logger_config``. + + Resolution order (first match wins): + + 1. ``config/override/logger.json`` — DEPRECATED (v0.13.0), removed v1.0.0. + 2. ``src/local/config/logger.json`` — the canonical location. + 3. Neither present — returns ``{}``; ``merge_logger_config`` applies + ``_DEFAULT_LOGGER_CONFIG`` as the base. + """ + override_path = os.path.join( + framework_path, FrameworkPaths.CONFIG_OVERRIDE_PATH, FrameworkPaths.LOGGER_CONFIG + ) + if os.path.exists(override_path): + import warnings as _warnings + _warnings.warn( + f"{FrameworkPaths.CONFIG_OVERRIDE_PATH}/{FrameworkPaths.LOGGER_CONFIG} is deprecated " + f"(v0.13.0) and will be removed in v1.0.0. Migrate to " + f"{FrameworkPaths.LOCAL_CONFIG_PATH}/{FrameworkPaths.LOGGER_CONFIG} — only the keys " + "you want to change are needed (sparse files are supported). " + "See the framework configuration documentation for migration steps.", + DeprecationWarning, + stacklevel=2, + ) + return _get_logger_config_from_file(override_path) + + local_path = os.path.join( + framework_path, FrameworkPaths.LOCAL_CONFIG_PATH, FrameworkPaths.LOGGER_CONFIG + ) + return _get_logger_config_from_file(local_path) + + +def _get_bundle_logger_config(bundle_path: str) -> Dict[str, Any]: + """Load pipeline bundle logger.json from pipeline_configs.""" + path = os.path.join(bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, PipelineBundlePaths.LOGGER_CONFIG) + return _get_logger_config_from_file(path) + + +def get_logger( + spark: "SparkSession", + dbutils: "DBUtils", + effective_config: Dict[str, Any], + spark_log_level: Optional[str] = None, + logger_name: str = "lakeflowframework", +) -> Any: + """ + Resolve the pipeline logger: default only, custom only, or CompositeLogger. + Never raises; falls back to default logger and logs warnings on failure. + """ + level = _resolve_log_level(effective_config, spark_log_level) + default_logger = create_default_logger(logger_name, level) + + if not effective_config.get("enabled", False): + return default_logger + + library = effective_config.get("library") + if library and not importlib.util.find_spec(str(library)): + default_logger.warning( + "Logger library %r not found; using framework default logger only.", + library, + ) + return default_logger + + module_name = effective_config.get("module") + factory_name = effective_config.get("factory") + if not module_name or not factory_name: + default_logger.warning( + "Custom logger enabled but module/factory not configured; using framework default logger only." + ) + return default_logger + + try: + module = importlib.import_module(str(module_name)) + factory = getattr(module, str(factory_name)) + factory_args = dict(effective_config.get("factory_args") or {}) + # Inject the resolved log level so pipeline-level logLevel settings + # (e.g. spark.conf logLevel = "DEBUG") propagate to the custom logger. + # Only set if the factory accepts a "level" kwarg (detected by checking + # factory_args; the factory itself must handle the kwarg). + factory_args["level"] = level + custom = factory(dbutils, spark, **factory_args) + _validate_logger_api(custom) + except Exception as exc: + import warnings as _warnings + _warnings.warn( + f"Custom logger failed to initialise — falling back to framework default logger. " + f"Check 'module' and 'factory' in your logger.json. Error: {exc}", + CustomLoggerConfigWarning, + stacklevel=2, + ) + default_logger.error( + "Custom logger failed to initialise (falling back to default): %s", + exc, + exc_info=True, + ) + return default_logger + + if effective_config.get("mirror_to_stdout", False): + return CompositeLogger(primary=custom, mirror=default_logger) + + # Custom logger owns output. Silence the Python logging registry entry so + # any code holding a direct logging.getLogger() reference stays quiet. + default_logger.handlers.clear() + default_logger.addHandler(logging.NullHandler()) + return custom + + +def resolve_pipeline_logger( + spark: "SparkSession", + dbutils: "DBUtils", + framework_path: str, + bundle_path: str, + spark_log_level: Optional[str] = None, +) -> Any: + """Load, merge, and resolve logger from framework and bundle logger.json files.""" + framework_cfg = _get_framework_logger_config(framework_path) + bundle_cfg = _get_bundle_logger_config(bundle_path) + effective = get_effective_logger_config(framework_cfg, bundle_cfg) + logger = get_logger(spark, dbutils, effective, spark_log_level=spark_log_level) + logger.info("Resolved pipeline logger config: %s", effective) + return logger diff --git a/src/lakeflow_framework/pipeline_config.py b/src/lakeflow_framework/pipeline_config.py new file mode 100644 index 0000000..095914c --- /dev/null +++ b/src/lakeflow_framework/pipeline_config.py @@ -0,0 +1,139 @@ +import os +import json +import logging +from typing import Dict, Any, Optional + +from pyspark.dbutils import DBUtils +import pyspark.sql.types as T +from pyspark.sql import SparkSession + +from lakeflow_framework.constants import DLTPipelineSettingKeys +from lakeflow_framework.pipeline_details import PipelineDetails +from lakeflow_framework.substitution_manager import SubstitutionManager + +# Module-level singletons +_spark = None +_dbutils = None +_logger = None +_substitution_manager = None +_pipeline_details = None +_mandatory_table_properties = None +_operational_metadata_schema = None + + +def initialize_core( + spark: SparkSession, + dbutils: DBUtils, + logger: logging.Logger +) -> None: + """Initialize the pipeline configuration.""" + global _spark, _dbutils, _logger + _spark = spark + _dbutils = dbutils + _logger = logger + + +def initialize_substitution_manager( + substitution_manager: SubstitutionManager +) -> None: + """Initialize the substitution manager.""" + global _substitution_manager + _substitution_manager = substitution_manager + + +def initialize_pipeline_details( + pipeline_details: PipelineDetails +) -> None: + """Initialize the pipeline details.""" + global _pipeline_details + _pipeline_details = pipeline_details + + +def initialize_mandatory_table_properties( + mandatory_table_properties: Dict[str, Any] +) -> None: + """Initialize the mandatory table properties.""" + global _mandatory_table_properties + _mandatory_table_properties = mandatory_table_properties + +def initialize_mandatory_configuration() -> None: + """Initialize the mandatory configuration.""" + verion_file = os.path.join( + os.path.dirname(_spark.conf.get(DLTPipelineSettingKeys.FRAMEWORK_SOURCE_PATH, ".")), + 'VERSION' + ) + with open(verion_file, mode='r', encoding='utf-8') as f: + version = f.read().strip() + + mandatory_configuration = { + 'version': version + } + _spark.conf.set('tag.lakeflow_framework', json.dumps(mandatory_configuration)) + +def initialize_operational_metadata_schema( + operational_metadata_schema: Optional[T.StructType] = None +) -> None: + """Initialize the operational metadata schema.""" + global _operational_metadata_schema + _operational_metadata_schema = operational_metadata_schema + +def initialize_table_migration( + table_migration_state_volume_path: str +) -> None: + """Initialize the table migration state volume path.""" + global _table_migration_state_volume_path + _table_migration_state_volume_path = table_migration_state_volume_path + + +def get_spark() -> SparkSession: + """Get the Spark instance.""" + if _spark is None: + raise RuntimeError("Spark has not been initialized. Call initialize_pipeline_config() first.") + return _spark + + +def get_dbutils() -> DBUtils: + """Get the DBUtils instance.""" + if _dbutils is None: + raise RuntimeError("DBUtils has not been initialized. Call initialize_pipeline_config() first.") + return _dbutils + + +def get_logger() -> logging.Logger: + """Get the logger instance.""" + if _logger is None: + raise RuntimeError("Logger has not been initialized. Call initialize_pipeline_config() first.") + return _logger + + +def get_substitution_manager() -> SubstitutionManager: + """Get the substitution manager instance.""" + if _substitution_manager is None: + raise RuntimeError("Substitution manager has not been initialized. Call initialize_pipeline_config() first.") + return _substitution_manager + + +def get_pipeline_details() -> PipelineDetails: + """Get the pipeline details instance.""" + if _pipeline_details is None: + raise RuntimeError("Pipeline details has not been initialized. Call initialize_pipeline_config() first.") + return _pipeline_details + + +def get_mandatory_table_properties() -> Dict[str, Any]: + """Get the mandatory table properties.""" + if _mandatory_table_properties is None: + raise RuntimeError("Mandatory table properties have not been initialized. Call initialize_pipeline_config() first.") + return _mandatory_table_properties + + +def get_operational_metadata_schema() -> Optional[T.StructType]: + """Get the operational metadata schema.""" + return _operational_metadata_schema + + +def get_table_migration_state_volume_path() -> str: + """Get the table migration state volume path.""" + if _table_migration_state_volume_path is None: + raise RuntimeError("Table migration state volume path has not been initialized. Call initialize_pipeline_config() first.") + return _table_migration_state_volume_path \ No newline at end of file diff --git a/src/lakeflow_framework/pipeline_details.py b/src/lakeflow_framework/pipeline_details.py new file mode 100644 index 0000000..a517a58 --- /dev/null +++ b/src/lakeflow_framework/pipeline_details.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class PipelineDetails: + """Container for pipeline configuration details.""" + pipeline_id: Optional[str] + pipeline_catalog: Optional[str] + pipeline_schema: Optional[str] + pipeline_layer: Optional[str] + start_utc_timestamp: str + workspace_env: Optional[str] + logical_env: str \ No newline at end of file diff --git a/src/schemas/definitions_flows.json b/src/lakeflow_framework/schemas/definitions_flows.json similarity index 100% rename from src/schemas/definitions_flows.json rename to src/lakeflow_framework/schemas/definitions_flows.json diff --git a/src/schemas/definitions_main.json b/src/lakeflow_framework/schemas/definitions_main.json similarity index 100% rename from src/schemas/definitions_main.json rename to src/lakeflow_framework/schemas/definitions_main.json diff --git a/src/schemas/definitions_sources.json b/src/lakeflow_framework/schemas/definitions_sources.json similarity index 100% rename from src/schemas/definitions_sources.json rename to src/lakeflow_framework/schemas/definitions_sources.json diff --git a/src/schemas/definitions_targets.json b/src/lakeflow_framework/schemas/definitions_targets.json similarity index 100% rename from src/schemas/definitions_targets.json rename to src/lakeflow_framework/schemas/definitions_targets.json diff --git a/src/schemas/expectations.json b/src/lakeflow_framework/schemas/expectations.json similarity index 100% rename from src/schemas/expectations.json rename to src/lakeflow_framework/schemas/expectations.json diff --git a/src/schemas/flow_group.json b/src/lakeflow_framework/schemas/flow_group.json similarity index 100% rename from src/schemas/flow_group.json rename to src/lakeflow_framework/schemas/flow_group.json diff --git a/src/schemas/main.json b/src/lakeflow_framework/schemas/main.json similarity index 100% rename from src/schemas/main.json rename to src/lakeflow_framework/schemas/main.json diff --git a/src/schemas/secrets.json b/src/lakeflow_framework/schemas/secrets.json similarity index 100% rename from src/schemas/secrets.json rename to src/lakeflow_framework/schemas/secrets.json diff --git a/src/schemas/spec_flows.json b/src/lakeflow_framework/schemas/spec_flows.json similarity index 100% rename from src/schemas/spec_flows.json rename to src/lakeflow_framework/schemas/spec_flows.json diff --git a/src/schemas/spec_mapping.json b/src/lakeflow_framework/schemas/spec_mapping.json similarity index 100% rename from src/schemas/spec_mapping.json rename to src/lakeflow_framework/schemas/spec_mapping.json diff --git a/src/schemas/spec_materialized_views.json b/src/lakeflow_framework/schemas/spec_materialized_views.json similarity index 100% rename from src/schemas/spec_materialized_views.json rename to src/lakeflow_framework/schemas/spec_materialized_views.json diff --git a/src/schemas/spec_standard.json b/src/lakeflow_framework/schemas/spec_standard.json similarity index 100% rename from src/schemas/spec_standard.json rename to src/lakeflow_framework/schemas/spec_standard.json diff --git a/src/schemas/spec_template.json b/src/lakeflow_framework/schemas/spec_template.json similarity index 100% rename from src/schemas/spec_template.json rename to src/lakeflow_framework/schemas/spec_template.json diff --git a/src/schemas/spec_template_definition.json b/src/lakeflow_framework/schemas/spec_template_definition.json similarity index 100% rename from src/schemas/spec_template_definition.json rename to src/lakeflow_framework/schemas/spec_template_definition.json diff --git a/src/lakeflow_framework/secrets_manager.py b/src/lakeflow_framework/secrets_manager.py new file mode 100644 index 0000000..fce2805 --- /dev/null +++ b/src/lakeflow_framework/secrets_manager.py @@ -0,0 +1,196 @@ +from dataclasses import dataclass +from typing import Dict, Pattern, Any, List +import re +import os + +from pyspark.dbutils import DBUtils + +import lakeflow_framework.pipeline_config as pipeline_config +import lakeflow_framework.utility as utility + + +@dataclass(frozen=True) +class SecretConfig: + """ + Immutable configuration for a secret. + + Attributes: + scope: The secret scope name + key: The secret key name + exceptionEnabled: Whether to raise exceptions on secret retrieval failure + """ + scope: str + key: str + exceptionEnabled: bool = False + + def __post_init__(self): + """Validate the secret configuration.""" + if not self.scope or not isinstance(self.scope, str): + raise ValueError("Secret scope must be a non-empty string") + if not self.key or not isinstance(self.key, str): + raise ValueError("Secret key must be a non-empty string") + + def get_secret(self, dbutils: DBUtils) -> str: + """Get the secret value, retrieving it if not already cached.""" + try: + return SecretValue( + dbutils.secrets.get( + scope=self.scope, + key=self.key + ) + ) + except Exception as e: + if self.exceptionEnabled: + raise RuntimeError( + f"Failed to retrieve secret '{self.key}' from scope '{self.scope}': {str(e)}" + ) from e + return "" + + +class SecretValue: + """ + A wrapper class that lazily retrieves secrets when accessed. + This prevents secrets from being stored in memory and only retrieves them when needed. + """ + def __init__(self, secret: str): + self.__secret = secret + + def __str__(self) -> str: + """Return the secret value when converted to string.""" + return self.__secret + + def __repr__(self) -> str: + """Return a redacted string representation.""" + return "[REDACTED]" + + def __dict__(self) -> Dict[str, str]: + """Prevent conversion to dict from exposing the secret value.""" + return {"secret": "[REDACTED]"} + + def __getstate__(self) -> Dict[str, str]: + """Prevent pickling from exposing the secret value.""" + return {"secret": "[REDACTED]"} + + def clear(self) -> None: + """Clear the cached secret value.""" + self.__secret = None + + +class SecretsManager: + """ + This class provides a centralized way to manage secrets. + + Attributes: + dbutils: DBUtils instance for accessing secrets + logger: Logger instance for logging + framework_secrets_config_path: Path to framework secrets config + pipeline_secrets_config_path: Path to pipeline secrets config + + Example: + manager = SecretsManager(dbutils, "./config/framework_secrets.json", "./config/pipeline_secrets.json") + secrets = manager.get_secrets() # Get all secrets + value = manager.get_secret("db_password") # Get specific secret + """ + + SECRET_PATTERN: Pattern = re.compile(r"^\$\{secret\.([a-zA-Z0-9_]+)\}$") + + def __init__( + self, + json_validation_schema_path: str, + framework_secrets_config_paths: List[str], + pipeline_secrets_config_paths: List[str] + ): + self.dbutils = pipeline_config.get_dbutils() + self.logger = pipeline_config.get_logger() + self.json_validation_schema_path = json_validation_schema_path + self.framework_secrets_config_paths = framework_secrets_config_paths + self.pipeline_secrets_config_paths = pipeline_secrets_config_paths + self.validator = utility.JSONValidator(json_validation_schema_path) + + # Load secret configurations only + self._secret_configs = self._load_secrets() + + def _load_file(self, paths: List[str], config_type: str) -> Dict[str, Any]: + """Load a single secret file from a list of possible paths.""" + existing_files = [path for path in paths if os.path.exists(path)] + + if len(existing_files) > 1: + raise ValueError(f"Multiple {config_type} secrets files found. Only one is allowed: {existing_files}") + + if len(existing_files) == 1: + file_path = existing_files[0] + + self.logger.info("Retrieving %s secrets from: %s", config_type, file_path) + return utility.load_config_file_auto(file_path, False) + + self.logger.warning("No %s secrets file found.", config_type) + return {} + + def _load_secrets_config_from_files(self) -> Dict[str, Any]: + """Load and merge framework and pipeline secret configurations.""" + framework_secrets = self._load_file(self.framework_secrets_config_paths, "framework") + pipeline_secrets = self._load_file(self.pipeline_secrets_config_paths, "pipeline") + + return utility.merge_dicts_recursively(pipeline_secrets, framework_secrets) + + def _load_secrets(self) -> Dict[str, SecretConfig]: + """Load and merge framework and pipeline secret configurations.""" + json_data = self._load_secrets_config_from_files() + errors = self.validator.validate(json_data) + if errors: + raise ValueError(f"Secrets validation errors: {errors}") + + try: + return { + alias: SecretConfig(**config) + for alias, config + in json_data.items() + } + except KeyError as e: + raise ValueError(f"Invalid secret configuration: {e}") from e + except TypeError as e: + raise ValueError(f"Invalid secret configuration: {e}") from e + except Exception as e: + raise e + + def get_secret(self, alias: str) -> SecretValue: + """ + Get a SecretValue object for a secret by its alias. + + Args: + alias: The alias name of the secret + + Returns: + A SecretValue object that will lazily retrieve the secret when accessed + """ + if alias not in self._secret_configs: + raise KeyError(f"Secret alias '{alias}' not found") + + return self._secret_configs[alias].get_secret(self.dbutils) + + def substitute_secrets(self, data: Any) -> Any: + """ + Substitute secret references in a dictionary with SecretValue objects. + + Args: + data: The data to process (dict, list, or any other type) + + Returns: + Processed data with secret references replaced by SecretValue objects + """ + def substitute_value(value: Any) -> Any: + match = self.SECRET_PATTERN.match(value) + if match: + secret_alias = match.group(1) + return self.get_secret(secret_alias) + else: + return value + + if isinstance(data, dict): + return {k: self.substitute_secrets(v) for k, v in data.items()} + elif isinstance(data, list): + return [self.substitute_secrets(item) for item in data] + elif isinstance(data, str): + return substitute_value(data) + else: + return data diff --git a/src/lakeflow_framework/substitution_manager.py b/src/lakeflow_framework/substitution_manager.py new file mode 100644 index 0000000..9602d1c --- /dev/null +++ b/src/lakeflow_framework/substitution_manager.py @@ -0,0 +1,186 @@ +import re +import os +from typing import Dict, Any, Optional, Pattern, List +from functools import cached_property + +import lakeflow_framework.utility as utility +import lakeflow_framework.pipeline_config as pipeline_config + +class SubstitutionManager(): + """ + Manages token substitutions in strings and nested dictionaries. + + This class handles token replacements in configuration files, supporting both + framework-level and pipeline-level substitutions with optional additional tokens. + + Attributes: + framework_substitutions_path (str): Path to framework substitutions JSON + pipeline_substitutions_path (str): Path to pipeline substitutions JSON + additional_tokens (Optional[Dict[str, str]]): Optional dictionary of additional token replacements + + Methods: + substitute_string(input_string: str) -> str: + Substitute tokens in a string. + + substitute_dict(input_dict: Dict[str, Any]) -> Dict[str, Any]: + Substitute tokens in a nested dictionary. + """ + + # Compiled regex patterns for token substitution + DEFAULT_TOKEN_PATTERN: Pattern = re.compile(r'\{(\w+)\}') + + def __init__( + self, + framework_substitutions_paths: List[str], + pipeline_substitutions_paths: List[str], + additional_tokens: Optional[Dict[str, str]] = None + ): + """Initialize the SubstitutionManager. + + Args: + framework_substitutions_paths: List of paths to framework substitutions files. + pipeline_substitutions_paths: List of paths to pipeline substitutions files. + additional_tokens: Optional dictionary of additional token replacements + """ + if not framework_substitutions_paths or not pipeline_substitutions_paths: + raise ValueError("Framework and pipeline substitution paths must be provided as a list.") + + self.framework_substitutions_paths = framework_substitutions_paths + self.pipeline_substitutions_paths = pipeline_substitutions_paths + self.additional_tokens = additional_tokens or {} + + self.logger = pipeline_config.get_logger() + + self._substitutions_config = self._load_substitution_config() + + def _load_file(self, paths: List[str], config_type: str) -> Dict[str, Any]: + """Load a single substitution file from a list of possible paths.""" + existing_files = [path for path in paths if os.path.exists(path)] + + if len(existing_files) > 1: + raise ValueError(f"Multiple {config_type} substitutions files found. Only one is allowed: {existing_files}") + + if len(existing_files) == 1: + file_path = existing_files[0] + self.logger.info("Retrieving %s substitutions from: %s", config_type, file_path) + return utility.load_config_file_auto(file_path, False) or {} + + self.logger.warning("No %s substitutions file found.", config_type) + return {} + + def _load_substitution_config(self) -> Dict[str, Any]: + """Load and merge framework and pipeline substitutions.""" + framework_subs = self._load_file(self.framework_substitutions_paths, "framework") + pipeline_subs = self._load_file(self.pipeline_substitutions_paths, "pipeline") + + return utility.merge_dicts_recursively(pipeline_subs, framework_subs) + + @cached_property + def tokens(self) -> Dict[str, str]: + """Get merged tokens with additional tokens applied.""" + tokens = self._substitutions_config.get('tokens', {}) + if self.additional_tokens: + tokens = utility.merge_dicts_recursively(tokens, self.additional_tokens) + + # Apply substitutions to token values themselves + return { + k: self._substitute_tokens_in_string(v, tokens) + for k, v in tokens.items() + } + + @cached_property + def prefix_suffix_rules(self) -> Dict[str, Dict[str, str]]: + """Get prefix/suffix rules with tokens substituted.""" + rules = self._substitutions_config.get('prefix_suffix', {}) + + # Apply substitutions to prefix/suffix values + return { + k: { + rule_k: self._substitute_tokens_in_string(rule_v, self.tokens) + for rule_k, rule_v in v.items() + } + for k, v in rules.items() + } + + def substitute_string(self, input_string: str, additional_tokens: Optional[Dict[str, Any]] = None) -> str: + """Substitute tokens in a string. + + Args: + input_string: String containing tokens to replace + additional_tokens: Optional dictionary of additional token replacements. + Dictionary values will be converted to strings, with nested dictionaries being JSON serialized. + + Returns: + str: String with all tokens substituted + """ + if not isinstance(input_string, str): + raise TypeError(f"Expected string input, got {type(input_string)}") + + tokens = self.tokens + if additional_tokens: + tokens = utility.merge_dicts_recursively(self.tokens.copy(), additional_tokens) + + return self._substitute_tokens_in_string(input_string, tokens) + + def substitute_dict(self, data: Any) -> Any: + """ + Substitute tokens and apply prefix/suffix rules to a string or dictionary. + + Args: + data: The data to process (dict, list, or string) + + Returns: + Processed data with tokens references replaced by their values + """ + result = self._substitute_tokens(data) + result = self._apply_prefix_suffix(result) + return result + + def _substitute_tokens_in_string( + self, + value: str, + tokens: Dict[str, str], + pattern: Optional[Pattern] = None + ) -> str: + """Replace tokens in a string using regex substitution.""" + if not isinstance(value, str): + return value + + def replace_token(match): + token = match.group(1) + if token in self.additional_tokens: + return self.additional_tokens[token] + elif token in tokens: + return tokens[token] + return match.group(0) + + return (pattern or self.DEFAULT_TOKEN_PATTERN).sub(replace_token, value) + + def _substitute_tokens(self, data: Any) -> Any: + """Substitute tokens in a string or dictionary""" + if isinstance(data, dict): + return {k: self._substitute_tokens(v) for k, v in data.items()} + elif isinstance(data, list): + return [self._substitute_tokens(item) for item in data] + elif isinstance(data, str): + return self._substitute_tokens_in_string(data, self.tokens, self.DEFAULT_TOKEN_PATTERN) + else: + return data + + def _apply_prefix_suffix(self, data: Any) -> Any: + """Apply prefix/suffix rules to a string or dictionary.""" + def apply_value(key: str, value: Any) -> Any: + if isinstance(value, str) and key in self.prefix_suffix_rules: + rules = self.prefix_suffix_rules[key] + if 'prefix' in rules: + value = rules['prefix'] + value + if 'suffix' in rules: + value = value + rules['suffix'] + return value + + if isinstance(data, dict): + return {k: apply_value(k, v) for k, v in data.items()} + elif isinstance(data, list): + return [self._apply_prefix_suffix(item) for item in data] + else: + return data \ No newline at end of file diff --git a/src/lakeflow_framework/utility.py b/src/lakeflow_framework/utility.py new file mode 100644 index 0000000..961c4de --- /dev/null +++ b/src/lakeflow_framework/utility.py @@ -0,0 +1,535 @@ +import concurrent.futures +import importlib.util +import inspect +from functools import reduce +import logging +import os +from typing import Callable, Dict, List + +import json +import jsonschema as js +import yaml + +from pyspark.sql import DataFrame +from pyspark.sql import SparkSession +from pyspark.sql.types import StructType + +from lakeflow_framework.constants import ( + SupportedSpecFormat, + PipelineBundleSuffixesJson, + PipelineBundleSuffixesYaml, +) + + +def get_format_suffixes(file_format: str, suffix_type: str) -> list: + """Get file suffixes based on file format and suffix type. + + This is a centralized utility to retrieve the appropriate file suffixes + for different configuration and specification files based on the format + (JSON or YAML) and the type of file (substitutions, secrets, specs, etc.).""" + suffix_map = { + SupportedSpecFormat.JSON.value: { + "substitutions": PipelineBundleSuffixesJson.SUBSTITUTIONS_FILE_SUFFIX, + "secrets": PipelineBundleSuffixesJson.SECRETS_FILE_SUFFIX, + "main_spec": PipelineBundleSuffixesJson.MAIN_SPEC_FILE_SUFFIX, + "flow_group": PipelineBundleSuffixesJson.FLOW_GROUP_FILE_SUFFIX, + "expectations": PipelineBundleSuffixesJson.EXPECTATIONS_FILE_SUFFIX + }, + SupportedSpecFormat.YAML.value: { + "substitutions": PipelineBundleSuffixesYaml.SUBSTITUTIONS_FILE_SUFFIX, + "secrets": PipelineBundleSuffixesYaml.SECRETS_FILE_SUFFIX, + "main_spec": PipelineBundleSuffixesYaml.MAIN_SPEC_FILE_SUFFIX, + "flow_group": PipelineBundleSuffixesYaml.FLOW_GROUP_FILE_SUFFIX, + "expectations": PipelineBundleSuffixesYaml.EXPECTATIONS_FILE_SUFFIX + } + } + + if file_format not in suffix_map: + valid_formats = list(suffix_map.keys()) + raise ValueError( + f"Invalid file format: '{file_format}'. " + f"Valid formats are: {valid_formats}" + ) + + if suffix_type not in suffix_map[file_format]: + valid_types = list(suffix_map[file_format].keys()) + raise ValueError( + f"Invalid suffix type: '{suffix_type}'. " + f"Valid types are: {valid_types}" + ) + + result = suffix_map[file_format][suffix_type] + + # Always return a list for consistency + if isinstance(result, list): + return result + elif isinstance(result, (tuple, set)): + return list(result) + elif isinstance(result, str): + return [result] + else: + raise TypeError( + f"Invalid suffix type for '{suffix_type}': expected str, tuple, list, or set, " + f"but got {type(result).__name__}" + ) + +class JSONValidator: + """ + A JSON schema validator class. + + Attributes: + schema (dict): The JSON schema loaded from a file. + base_uri (str): The base URI for resolving schema references. + resolver (RefResolver): The JSON schema resolver. + validator (Draft7Validator): The JSON schema validator. + + Methods: + validate(json_data: Dict) -> List: + Validates the provided JSON data against the loaded schema and returns a list of validation errors. + """ + + def __init__(self, schema_path: str): + try: + with open(schema_path, "r", encoding="utf-8") as schema_file: + self.schema = json.load(schema_file) + except Exception as e: + raise ValueError(f"JSON Schema not found: {schema_path}") from e + + # Resolve references + self.base_uri = "file://" + os.path.abspath(os.path.dirname(schema_path)) + "/" + self.resolver = js.RefResolver(base_uri=self.base_uri, referrer=self.schema) + self.validator = js.Draft7Validator(self.schema, resolver=self.resolver) + + def validate(self, json_data: Dict) -> List: + """Validate the provided JSON data against the loaded schema and returns a list of validation errors.""" + return list(self.validator.iter_errors(json_data)) + + +def add_struct_field(struct: StructType, column: Dict): + """Add a field to a StructType schema.""" + return struct.jsonValue()["fields"].append(column) + + +def drop_columns(df: DataFrame, columns_to_drop: List) -> DataFrame: + """Drop columns from a DataFrame.""" + drop_column_list = [] + for column in columns_to_drop: + if column in df.columns: + drop_column_list.append(column) + if drop_column_list: + df = df.drop(*drop_column_list) + return df + + +def load_config_file(file_path: str, file_format: str = "json", fail_on_not_exists: bool = True) -> Dict: + """Load JSON or YAML data from a file.""" + if not os.path.exists(file_path): + if fail_on_not_exists: + raise ValueError(f"Path does not exist: {file_path}") + return {} + + with open(file_path, 'r', encoding='utf-8') as file: + try: + if file_format == "json": + return json.load(file) + elif file_format == "yaml": + return yaml.safe_load(file) + else: + raise ValueError(f"Invalid file format: {file_format}. Only 'json' and 'yaml' are supported.") + except json.JSONDecodeError as e: + raise ValueError( + f"Error loading JSON file '{file_path}': {e.msg} at line {e.lineno}, column {e.colno}" + ) from e + except yaml.YAMLError as e: + raise yaml.YAMLError(f"Error loading YAML file '{file_path}': {e}") + + +def load_config_files( + path: str, + file_format: str = "json", + file_suffix: str | List[str] = None, + recursive: bool = False +) -> Dict: + """Load configuration data from files with a specific suffix.""" + if file_suffix is None: + file_suffix = f".{file_format}" + + data = {} + if not path or path.strip() == "" or not os.path.exists(path): + raise ValueError(f"Path does not exist: {path}") + + file_suffix_list = [file_suffix] if isinstance(file_suffix, str) else file_suffix + + if recursive: + for root, _, filenames in os.walk(path): + for filename in filenames: + if any(filename.endswith(suffix) for suffix in file_suffix_list): + file_path = os.path.join(root, filename) + data[file_path] = load_config_file(file_path, file_format) + else: + for filename in os.listdir(path): + if any(filename.endswith(suffix) for suffix in file_suffix_list): + file_path = os.path.join(path, filename) + data[file_path] = load_config_file(file_path, file_format) + + return data + + +def load_config_file_auto(file_path: str, fail_on_not_exists: bool = True) -> Dict: + """Load JSON or YAML data from a file with automatic format detection. + + The file format is automatically detected based on the file extension: + - .json -> JSON format + - .yaml, .yml -> YAML format + + Args: + file_path: Path to the configuration file + fail_on_not_exists: Whether to raise an error if file doesn't exist + + Returns: + Dict containing the loaded configuration data + + Raises: + ValueError: If file extension is not recognized or path doesn't exist + """ + file_ext = os.path.splitext(file_path)[1].lower() + + if file_ext == '.json': + file_format = 'json' + elif file_ext in ('.yaml', '.yml'): + file_format = 'yaml' + else: + raise ValueError( + f"Unable to detect file format from extension '{file_ext}'. " + f"Supported extensions: .json, .yaml, .yml" + ) + + return load_config_file(file_path, file_format, fail_on_not_exists) + + +# TODO: Legacy wrapper around load_config_file. Remove this in a future release. +def get_json_from_file(file_path: str, fail_on_not_exists: bool = True) -> Dict: + """Load JSON data from a file.""" + return load_config_file(file_path, "json", fail_on_not_exists) + + +# TODO: Legacy wrapper around load_config_files. Remove this in a future release. +def get_json_from_files(path: str, file_suffix: str | List[str] = ".json", recursive: bool = False) -> Dict: + """Load JSON data from files that have a specific suffix.""" + return load_config_files(path, "json", file_suffix, recursive) + + +# TODO: Legacy wrapper around load_config_file. Remove this in a future release. +def get_yaml_from_file(file_path: str, fail_on_not_exists: bool = True) -> Dict: + """Load YAML data from a file.""" + return load_config_file(file_path, "yaml", fail_on_not_exists) + + +# TODO: Legacy wrapper around load_config_files. Remove this in a future release. +def get_yaml_from_files(path: str, file_suffix: str | List[str] = ".yaml", recursive: bool = False) -> Dict: + """Load YAML data from files with a specific suffix.""" + return load_config_files(path, "yaml", file_suffix, recursive) + + +def get_data_from_files_parallel( + path: str, + file_format: str, + file_suffix: str | List[str], + recursive: bool = False, + max_workers: int = 10, + logger=None, +) -> Dict: + """ + Load data from JSON or YAML files that have a specific suffix using parallel processing. + + Args: + path: Directory path to search for files + file_format: File format to load. ["json", "yaml"] + file_suffix: File suffixes to filter by e.g. ".json" or ["_main.json", "_flow.json"] + recursive: Whether to search recursively in subdirectories + max_workers: Maximum number of worker threads for parallel loading + + Returns: + Dict mapping file suffix to a list of file paths to their loaded JSON data + + Example output: + { + ".json": { + "/path/to/file1.json": { + "data": { + "key": "value" + } + }, + "/path/to/file2.json": { + "data": { + "key": "value" + } + } + } + } + + Raises: + ValueError: If the path doesn't exist + """ + def _discover_files(path: str, file_suffix: str | List[str], recursive: bool) -> List[str]: + file_path_list = [] + if recursive: + for root, _, filenames in os.walk(path): + for filename in filenames: + if filename.endswith(file_suffix): + file_path_list.append(os.path.join(root, filename)) + else: + for filename in os.listdir(path): + if filename.endswith(file_suffix): + file_path_list.append(os.path.join(path, filename)) + + return file_path_list + + def load_single_file(file_path): + try: + return file_path, load_config_file(file_path, file_format), None + except Exception as e: + return file_path, None, str(e) + + # Validate path + if not path or path.strip() == "" or not os.path.exists(path): + raise ValueError(f"Path does not exist: {path}") + + file_suffix_list = [file_suffix] if isinstance(file_suffix, str) else file_suffix + file_paths = {} + data = {} + errors = {} + for suffix in file_suffix_list: + file_paths[suffix] = _discover_files(path, suffix, recursive) + + for file_suffix, file_paths in file_paths.items(): + if logger is not None: + logger.debug("Loading %s files...", file_suffix) + logger.debug("File paths: %s", file_paths) + data[file_suffix] = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_path = { + executor.submit(load_single_file, file_path): file_path + for file_path in file_paths + } + + for future in concurrent.futures.as_completed(future_to_path): + file_path, file_data, error = future.result() + if error: + errors[file_path] = error + if logger is not None: + logger.debug("Failed to load %s: %s", file_path, error) + elif file_data is not None: + data[file_suffix][file_path] = file_data + + if errors: + if logger is not None: + logger.debug("%d files failed to load.", len(errors)) + for file_path, error in errors.items(): + logger.debug("Failed: %s: %s", file_path, error) + + return data + + +def get_pipeline_update_id(spark: SparkSession) -> str: + """ + Get the pipeline update id from the spark conf. + This is only populated post initialisation of the pipeline, by an event hook in DltPipelineBuilder. + + Args: + spark (SparkSession): The Spark session to use for the pipeline. + + Returns: + str: The pipeline update id. + """ + return spark.conf.get("pipeline.pipeline_update_id", None) + + +def get_table_versions(spark, source_view_dict: Dict[str, str]) -> DataFrame: + """Get table versions from a Spark DataFrame.""" + df_list = [] + for view_name, source_table in source_view_dict.items(): + sql = f"DESCRIBE HISTORY {source_table} LIMIT 1" + select_expr = [ + f"'{view_name}' AS viewName", + f"'{source_table}' AS tableName", + "version" + ] + df = spark.sql(sql).selectExpr(select_expr) + df_list.append(df) + return reduce(DataFrame.unionAll, df_list) if len(df_list) > 1 else df_list[0] + + +def load_python_function( + python_function_path: str, + function_name: str, + required_params: List[str] = None +) -> Callable: + """Load and validate a Python function from a file.""" + if required_params is None: + required_params = [] + + spec = importlib.util.spec_from_file_location("module", python_function_path) + if not spec or not spec.loader: + raise ImportError(f"Could not load Python function from {python_function_path}") + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # Validate function exists + if not hasattr(module, function_name): + raise AttributeError( + f"Python function file '{python_function_path}' must contain a " + f"'{function_name}' function with parameters: {', '.join(required_params)}" + ) + + function = getattr(module, function_name) + if not callable(function): + raise TypeError(f"'{function_name}' in '{python_function_path}' is not callable") + + # Inspect signature + sig = inspect.signature(function) + func_params = list(sig.parameters.keys()) + missing_params = [p for p in required_params if p not in func_params] + if missing_params: + raise ValueError( + f"Function '{function_name}' in '{python_function_path}' is missing " + f"required parameters: {', '.join(missing_params)}" + ) + + return function + + +def load_python_function_from_module( + python_module: str, + required_params: List[str] = None +) -> Callable: + """ + Load and validate a Python function from an extension module. + + The module must be importable via sys.path (typically from extensions/libraries, + which is added to sys.path during pipeline initialization). + + Args: + python_module: Module and function reference in format 'module_name.function_name' + (e.g., 'transforms.customer_source' or 'my_module.sub.get_data') + required_params: List of required parameter names for validation + + Returns: + The callable function from the module + + Raises: + ValueError: If python_module format is invalid + ImportError: If module cannot be imported + AttributeError: If function is not found in module + TypeError: If the attribute is not callable + """ + if required_params is None: + required_params = [] + + # Parse module and function name + if "." not in python_module: + raise ValueError( + f"Invalid pythonModule format: '{python_module}'. " + f"Expected format: 'module_name.function_name' (e.g., 'transforms.get_df')" + ) + + # Split on last dot to get module path and function name + module_path, function_name = python_module.rsplit(".", 1) + + # Import the module + try: + module = importlib.import_module(module_path) + except ModuleNotFoundError as e: + raise ImportError( + f"Could not import module '{module_path}' for pythonModule '{python_module}'. " + f"Ensure the module exists in the extensions directory. Original error: {e}" + ) from e + + # Get the function + if not hasattr(module, function_name): + raise AttributeError( + f"Module '{module_path}' does not contain function '{function_name}'. " + f"Available attributes: {[a for a in dir(module) if not a.startswith('_')]}" + ) + + function = getattr(module, function_name) + if not callable(function): + raise TypeError( + f"'{function_name}' in module '{module_path}' is not callable" + ) + + # Validate required parameters + if required_params: + sig = inspect.signature(function) + func_params = list(sig.parameters.keys()) + missing_params = [p for p in required_params if p not in func_params] + if missing_params: + raise ValueError( + f"Function '{function_name}' in module '{module_path}' is missing " + f"required parameters: {', '.join(missing_params)}" + ) + + return function + + +def list_sub_paths(path: str) -> List[str]: + """List subdirectories in a given directory path.""" + return [x for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))] + + +def merge_dicts(*dicts: Dict) -> Dict: + """Merge dictionaries into a single dictionary.""" + return reduce(lambda a, b: {**a, **b} if b is not None else a, dicts, {}) + + +def merge_dicts_recursively(d1: Dict, d2: Dict) -> Dict: + """Recursively merges two dictionaries. Keys in d1 take precedence over d2.""" + d = d1.copy() + + for key in d2: + if key in d1: + if isinstance(d1[key], dict) and isinstance(d2[key], dict): + d[key] = merge_dicts_recursively(d1[key], d2[key]) + else: + d[key] = d2[key] + + return d + + +def replace_dict_key_value(spec: Dict, target_key: str, new_value: str) -> Dict: + """Replace values of a specific key in a nested dictionary.""" + if isinstance(spec, dict): + for key, value in spec.items(): + if key == target_key: + if spec[key] is not None and spec[key].strip() != "": + spec[key] = f"{new_value}/{spec[key]}" + elif isinstance(value, dict) or isinstance(value, list): + replace_dict_key_value(value, target_key, new_value) + elif isinstance(spec, list): + for item in spec: + replace_dict_key_value(item, target_key, new_value) + return spec + + +def set_logger(logger_name: str, log_level: str = "INFO") -> logging.Logger: + """Here for backwards compatibility. Use resolve_logger instead. Set up and return a logger with a specified name and log level.""" + from logger import create_default_logger + + return create_default_logger(logger_name, log_level) + + +def deep_merge(base: Dict, overlay: Dict) -> Dict: + """Recursively merge *overlay* into *base*, returning a new dict. + + - Dict values are merged recursively; overlay wins on key conflicts. + - Non-dict values (including lists) are replaced wholesale by the overlay value. + - Neither input is mutated. + """ + result = base.copy() + for key, val in overlay.items(): + if key in result and isinstance(result[key], dict) and isinstance(val, dict): + result[key] = deep_merge(result[key], val) + else: + result[key] = val + return result diff --git a/src/logger.py b/src/logger.py index b90378b..bb60b0a 100644 --- a/src/logger.py +++ b/src/logger.py @@ -1,319 +1,10 @@ -""" -Pluggable pipeline logging: default stdout logger, optional custom logger from logger.json, -and CompositeLogger (custom primary + framework stdout mirror). -""" - -from __future__ import annotations - -import importlib -import importlib.util -import logging -import os -import sys -from copy import deepcopy -from typing import TYPE_CHECKING, Any, Dict, Optional - -from constants import FrameworkPaths, PipelineBundlePaths - -if TYPE_CHECKING: - from pyspark.dbutils import DBUtils - from pyspark.sql import SparkSession - -_REQUIRED_LOG_METHODS = ("debug", "info", "warning", "error", "critical", "exception") - - -class CustomLoggerConfigWarning(UserWarning): - """Emitted when the custom logger fails to initialise and the framework falls back to the default logger.""" - -_DEFAULT_LOGGER_CONFIG: Dict[str, Any] = { - "enabled": False, - "factory_args": {}, - "level": "INFO", - "mirror_to_stdout": False, - "allow_pipeline_logger_override": False, -} - - -def create_default_logger(logger_name: str, log_level: str = "INFO") -> logging.Logger: - """Create a framework default logger writing to stdout.""" - logger = logging.getLogger(logger_name) - level = getattr(logging, log_level.upper(), None) or logging.INFO - logger.setLevel(level) - - if logger.hasHandlers(): - logger.handlers.clear() - - handler = logging.StreamHandler(sys.stdout) - handler.setFormatter( - logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - ) - logger.addHandler(handler) - return logger - - -def get_effective_logger_config( - framework_config: Optional[Dict[str, Any]], - bundle_config: Optional[Dict[str, Any]], -) -> Dict[str, Any]: - """ - Merge framework and bundle logger.json. - - allow_pipeline_logger_override (framework only, default false): - false -> framework wins on conflicts - true -> bundle wins on conflicts - """ - framework = deepcopy(framework_config or {}) - bundle = deepcopy(bundle_config or {}) - bundle.pop("allow_pipeline_logger_override", None) - allow_override = bool(framework.get("allow_pipeline_logger_override", False)) - - merged = deepcopy(_DEFAULT_LOGGER_CONFIG) - - if allow_override: - # Bundle config wins: apply framework first, then bundle on top. - merged.update(framework) - merged.update(bundle) - merged["factory_args"] = { - **(framework.get("factory_args") or {}), - **(bundle.get("factory_args") or {}), - } - else: - # Framework config wins: bundle config is ignored entirely. - merged.update(framework) - merged["factory_args"] = { - **(framework.get("factory_args") or {}), - } - - merged["allow_pipeline_logger_override"] = allow_override - return merged - - -def _validate_logger_api(logger_obj: Any) -> None: - for name in _REQUIRED_LOG_METHODS: - method = getattr(logger_obj, name, None) - if not callable(method): - raise TypeError( - f"Custom logger must provide a callable '{name}' method; got {type(method).__name__}" - ) - - -def _resolve_log_level(config: Dict[str, Any], spark_log_level: Optional[str]) -> str: - if spark_log_level: - return spark_log_level.upper() - level = config.get("level", "INFO") - return str(level).upper() if level else "INFO" - - -class CompositeLogger: - """Custom logger as primary; framework default logger mirrors to stdout.""" - - def __init__(self, primary: Any, mirror: logging.Logger) -> None: - self._primary = primary - self._mirror = mirror - - def close(self) -> None: - closer = getattr(self._primary, "close", None) - if callable(closer): - closer() - - def _call_primary(self, method_name: str, message: str, args: tuple, kwargs: Dict[str, Any]) -> None: - method = getattr(self._primary, method_name) - if kwargs: - method(message, **kwargs) - elif args: - method(message, *args) - else: - method(message) - - def _call_mirror(self, method_name: str, message: str, args: tuple, kwargs: Dict[str, Any]) -> None: - method = getattr(self._mirror, method_name) - if kwargs: - method(message, *args, **kwargs) - elif args: - method(message, *args) - else: - method(message) - - def _log(self, method_name: str, message: str, *args: Any, **kwargs: Any) -> None: - try: - self._call_primary(method_name, message, args, kwargs) - except Exception as exc: - self._mirror.warning( - "Custom logger failed in %s: %s", - method_name, - exc, - exc_info=True, - ) - try: - self._call_mirror(method_name, message, args, kwargs) - except Exception: - pass - - def debug(self, message: str, *args: Any, **kwargs: Any) -> None: - self._log("debug", message, *args, **kwargs) - - def info(self, message: str, *args: Any, **kwargs: Any) -> None: - self._log("info", message, *args, **kwargs) - - def warning(self, message: str, *args: Any, **kwargs: Any) -> None: - self._log("warning", message, *args, **kwargs) - - def error(self, message: str, *args: Any, **kwargs: Any) -> None: - self._log("error", message, *args, **kwargs) - - def critical(self, message: str, *args: Any, **kwargs: Any) -> None: - self._log("critical", message, *args, **kwargs) - - def exception(self, message: str, *args: Any, **kwargs: Any) -> None: - self._log("exception", message, *args, **kwargs) - - -def _get_logger_config_from_file(config_path: str) -> Dict[str, Any]: - """ - Load logger.json from disk. Missing or invalid files return {} and log a warning - via a one-off default logger when possible. - """ - if not config_path or not os.path.isfile(config_path): - return {} - - try: - from utility import get_json_from_file - - data = get_json_from_file(config_path, fail_on_not_exists=False) - return data if isinstance(data, dict) else {} - except Exception as exc: - bootstrap = create_default_logger("lakeflowframework.config") - bootstrap.warning( - "Failed to load logger config from %s: %s. Using empty logger config for this source.", - config_path, - exc, - ) - return {} - - -def _get_framework_logger_config(framework_path: str) -> Dict[str, Any]: - """Load framework logger.json configuration. - - Logger defaults are defined in-code as ``_DEFAULT_LOGGER_CONFIG`` — there is no - ``config/default/logger.json`` on disk. This function therefore returns only the - *override keys* that should be merged on top of those defaults inside - ``merge_logger_config``. - - Resolution order (first match wins): - - 1. ``config/override/logger.json`` — DEPRECATED (v0.13.0), removed v1.0.0. - 2. ``src/local/config/logger.json`` — the canonical location. - 3. Neither present — returns ``{}``; ``merge_logger_config`` applies - ``_DEFAULT_LOGGER_CONFIG`` as the base. - """ - override_path = os.path.join( - framework_path, FrameworkPaths.CONFIG_OVERRIDE_PATH, FrameworkPaths.LOGGER_CONFIG - ) - if os.path.exists(override_path): - import warnings as _warnings - _warnings.warn( - f"{FrameworkPaths.CONFIG_OVERRIDE_PATH}/{FrameworkPaths.LOGGER_CONFIG} is deprecated " - f"(v0.13.0) and will be removed in v1.0.0. Migrate to " - f"{FrameworkPaths.LOCAL_CONFIG_PATH}/{FrameworkPaths.LOGGER_CONFIG} — only the keys " - "you want to change are needed (sparse files are supported). " - "See the framework configuration documentation for migration steps.", - DeprecationWarning, - stacklevel=2, - ) - return _get_logger_config_from_file(override_path) - - local_path = os.path.join( - framework_path, FrameworkPaths.LOCAL_CONFIG_PATH, FrameworkPaths.LOGGER_CONFIG - ) - return _get_logger_config_from_file(local_path) - - -def _get_bundle_logger_config(bundle_path: str) -> Dict[str, Any]: - """Load pipeline bundle logger.json from pipeline_configs.""" - path = os.path.join(bundle_path, PipelineBundlePaths.PIPELINE_CONFIGS_PATH, PipelineBundlePaths.LOGGER_CONFIG) - return _get_logger_config_from_file(path) - - -def get_logger( - spark: "SparkSession", - dbutils: "DBUtils", - effective_config: Dict[str, Any], - spark_log_level: Optional[str] = None, - logger_name: str = "lakeflowframework", -) -> Any: - """ - Resolve the pipeline logger: default only, custom only, or CompositeLogger. - Never raises; falls back to default logger and logs warnings on failure. - """ - level = _resolve_log_level(effective_config, spark_log_level) - default_logger = create_default_logger(logger_name, level) - - if not effective_config.get("enabled", False): - return default_logger - - library = effective_config.get("library") - if library and not importlib.util.find_spec(str(library)): - default_logger.warning( - "Logger library %r not found; using framework default logger only.", - library, - ) - return default_logger - - module_name = effective_config.get("module") - factory_name = effective_config.get("factory") - if not module_name or not factory_name: - default_logger.warning( - "Custom logger enabled but module/factory not configured; using framework default logger only." - ) - return default_logger - - try: - module = importlib.import_module(str(module_name)) - factory = getattr(module, str(factory_name)) - factory_args = dict(effective_config.get("factory_args") or {}) - # Inject the resolved log level so pipeline-level logLevel settings - # (e.g. spark.conf logLevel = "DEBUG") propagate to the custom logger. - # Only set if the factory accepts a "level" kwarg (detected by checking - # factory_args; the factory itself must handle the kwarg). - factory_args["level"] = level - custom = factory(dbutils, spark, **factory_args) - _validate_logger_api(custom) - except Exception as exc: - import warnings as _warnings - _warnings.warn( - f"Custom logger failed to initialise — falling back to framework default logger. " - f"Check 'module' and 'factory' in your logger.json. Error: {exc}", - CustomLoggerConfigWarning, - stacklevel=2, - ) - default_logger.error( - "Custom logger failed to initialise (falling back to default): %s", - exc, - exc_info=True, - ) - return default_logger - - if effective_config.get("mirror_to_stdout", False): - return CompositeLogger(primary=custom, mirror=default_logger) - - # Custom logger owns output. Silence the Python logging registry entry so - # any code holding a direct logging.getLogger() reference stays quiet. - default_logger.handlers.clear() - default_logger.addHandler(logging.NullHandler()) - return custom - - -def resolve_pipeline_logger( - spark: "SparkSession", - dbutils: "DBUtils", - framework_path: str, - bundle_path: str, - spark_log_level: Optional[str] = None, -) -> Any: - """Load, merge, and resolve logger from framework and bundle logger.json files.""" - framework_cfg = _get_framework_logger_config(framework_path) - bundle_cfg = _get_bundle_logger_config(bundle_path) - effective = get_effective_logger_config(framework_cfg, bundle_cfg) - logger = get_logger(spark, dbutils, effective, spark_log_level=spark_log_level) - logger.info("Resolved pipeline logger config: %s", effective) - return logger +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.logger import ... +from lakeflow_framework.logger import ( # noqa: F401 + CustomLoggerConfigWarning, + create_default_logger, + get_effective_logger_config, + CompositeLogger, + get_logger, + resolve_pipeline_logger, +) diff --git a/src/pipeline_config.py b/src/pipeline_config.py index ac7a52a..0114379 100644 --- a/src/pipeline_config.py +++ b/src/pipeline_config.py @@ -1,139 +1,4 @@ -import os -import json -import logging -from typing import Dict, Any, Optional - -from pyspark.dbutils import DBUtils -import pyspark.sql.types as T -from pyspark.sql import SparkSession - -from constants import DLTPipelineSettingKeys -from pipeline_details import PipelineDetails -from substitution_manager import SubstitutionManager - -# Module-level singletons -_spark = None -_dbutils = None -_logger = None -_substitution_manager = None -_pipeline_details = None -_mandatory_table_properties = None -_operational_metadata_schema = None - - -def initialize_core( - spark: SparkSession, - dbutils: DBUtils, - logger: logging.Logger -) -> None: - """Initialize the pipeline configuration.""" - global _spark, _dbutils, _logger - _spark = spark - _dbutils = dbutils - _logger = logger - - -def initialize_substitution_manager( - substitution_manager: SubstitutionManager -) -> None: - """Initialize the substitution manager.""" - global _substitution_manager - _substitution_manager = substitution_manager - - -def initialize_pipeline_details( - pipeline_details: PipelineDetails -) -> None: - """Initialize the pipeline details.""" - global _pipeline_details - _pipeline_details = pipeline_details - - -def initialize_mandatory_table_properties( - mandatory_table_properties: Dict[str, Any] -) -> None: - """Initialize the mandatory table properties.""" - global _mandatory_table_properties - _mandatory_table_properties = mandatory_table_properties - -def initialize_mandatory_configuration() -> None: - """Initialize the mandatory configuration.""" - verion_file = os.path.join( - os.path.dirname(_spark.conf.get(DLTPipelineSettingKeys.FRAMEWORK_SOURCE_PATH, ".")), - 'VERSION' - ) - with open(verion_file, mode='r', encoding='utf-8') as f: - version = f.read().strip() - - mandatory_configuration = { - 'version': version - } - _spark.conf.set('tag.lakeflow_framework', json.dumps(mandatory_configuration)) - -def initialize_operational_metadata_schema( - operational_metadata_schema: Optional[T.StructType] = None -) -> None: - """Initialize the operational metadata schema.""" - global _operational_metadata_schema - _operational_metadata_schema = operational_metadata_schema - -def initialize_table_migration( - table_migration_state_volume_path: str -) -> None: - """Initialize the table migration state volume path.""" - global _table_migration_state_volume_path - _table_migration_state_volume_path = table_migration_state_volume_path - - -def get_spark() -> SparkSession: - """Get the Spark instance.""" - if _spark is None: - raise RuntimeError("Spark has not been initialized. Call initialize_pipeline_config() first.") - return _spark - - -def get_dbutils() -> DBUtils: - """Get the DBUtils instance.""" - if _dbutils is None: - raise RuntimeError("DBUtils has not been initialized. Call initialize_pipeline_config() first.") - return _dbutils - - -def get_logger() -> logging.Logger: - """Get the logger instance.""" - if _logger is None: - raise RuntimeError("Logger has not been initialized. Call initialize_pipeline_config() first.") - return _logger - - -def get_substitution_manager() -> SubstitutionManager: - """Get the substitution manager instance.""" - if _substitution_manager is None: - raise RuntimeError("Substitution manager has not been initialized. Call initialize_pipeline_config() first.") - return _substitution_manager - - -def get_pipeline_details() -> PipelineDetails: - """Get the pipeline details instance.""" - if _pipeline_details is None: - raise RuntimeError("Pipeline details has not been initialized. Call initialize_pipeline_config() first.") - return _pipeline_details - - -def get_mandatory_table_properties() -> Dict[str, Any]: - """Get the mandatory table properties.""" - if _mandatory_table_properties is None: - raise RuntimeError("Mandatory table properties have not been initialized. Call initialize_pipeline_config() first.") - return _mandatory_table_properties - - -def get_operational_metadata_schema() -> Optional[T.StructType]: - """Get the operational metadata schema.""" - return _operational_metadata_schema - - -def get_table_migration_state_volume_path() -> str: - """Get the table migration state volume path.""" - if _table_migration_state_volume_path is None: - raise RuntimeError("Table migration state volume path has not been initialized. Call initialize_pipeline_config() first.") - return _table_migration_state_volume_path \ No newline at end of file +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework import pipeline_config or from lakeflow_framework.pipeline_config import ... +from lakeflow_framework import pipeline_config # noqa: F401 +from lakeflow_framework.pipeline_config import * # noqa: F401, F403 diff --git a/src/pipeline_details.py b/src/pipeline_details.py index a517a58..f13453a 100644 --- a/src/pipeline_details.py +++ b/src/pipeline_details.py @@ -1,14 +1,3 @@ -from dataclasses import dataclass -from typing import Optional - - -@dataclass -class PipelineDetails: - """Container for pipeline configuration details.""" - pipeline_id: Optional[str] - pipeline_catalog: Optional[str] - pipeline_schema: Optional[str] - pipeline_layer: Optional[str] - start_utc_timestamp: str - workspace_env: Optional[str] - logical_env: str \ No newline at end of file +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.pipeline_details import PipelineDetails +from lakeflow_framework.pipeline_details import PipelineDetails # noqa: F401 diff --git a/src/secrets_manager.py b/src/secrets_manager.py index 68ed8c6..1687b68 100644 --- a/src/secrets_manager.py +++ b/src/secrets_manager.py @@ -1,196 +1,7 @@ -from dataclasses import dataclass -from typing import Dict, Pattern, Any, List -import re -import os - -from pyspark.dbutils import DBUtils - -import pipeline_config -import utility - - -@dataclass(frozen=True) -class SecretConfig: - """ - Immutable configuration for a secret. - - Attributes: - scope: The secret scope name - key: The secret key name - exceptionEnabled: Whether to raise exceptions on secret retrieval failure - """ - scope: str - key: str - exceptionEnabled: bool = False - - def __post_init__(self): - """Validate the secret configuration.""" - if not self.scope or not isinstance(self.scope, str): - raise ValueError("Secret scope must be a non-empty string") - if not self.key or not isinstance(self.key, str): - raise ValueError("Secret key must be a non-empty string") - - def get_secret(self, dbutils: DBUtils) -> str: - """Get the secret value, retrieving it if not already cached.""" - try: - return SecretValue( - dbutils.secrets.get( - scope=self.scope, - key=self.key - ) - ) - except Exception as e: - if self.exceptionEnabled: - raise RuntimeError( - f"Failed to retrieve secret '{self.key}' from scope '{self.scope}': {str(e)}" - ) from e - return "" - - -class SecretValue: - """ - A wrapper class that lazily retrieves secrets when accessed. - This prevents secrets from being stored in memory and only retrieves them when needed. - """ - def __init__(self, secret: str): - self.__secret = secret - - def __str__(self) -> str: - """Return the secret value when converted to string.""" - return self.__secret - - def __repr__(self) -> str: - """Return a redacted string representation.""" - return "[REDACTED]" - - def __dict__(self) -> Dict[str, str]: - """Prevent conversion to dict from exposing the secret value.""" - return {"secret": "[REDACTED]"} - - def __getstate__(self) -> Dict[str, str]: - """Prevent pickling from exposing the secret value.""" - return {"secret": "[REDACTED]"} - - def clear(self) -> None: - """Clear the cached secret value.""" - self.__secret = None - - -class SecretsManager: - """ - This class provides a centralized way to manage secrets. - - Attributes: - dbutils: DBUtils instance for accessing secrets - logger: Logger instance for logging - framework_secrets_config_path: Path to framework secrets config - pipeline_secrets_config_path: Path to pipeline secrets config - - Example: - manager = SecretsManager(dbutils, "./config/framework_secrets.json", "./config/pipeline_secrets.json") - secrets = manager.get_secrets() # Get all secrets - value = manager.get_secret("db_password") # Get specific secret - """ - - SECRET_PATTERN: Pattern = re.compile(r"^\$\{secret\.([a-zA-Z0-9_]+)\}$") - - def __init__( - self, - json_validation_schema_path: str, - framework_secrets_config_paths: List[str], - pipeline_secrets_config_paths: List[str] - ): - self.dbutils = pipeline_config.get_dbutils() - self.logger = pipeline_config.get_logger() - self.json_validation_schema_path = json_validation_schema_path - self.framework_secrets_config_paths = framework_secrets_config_paths - self.pipeline_secrets_config_paths = pipeline_secrets_config_paths - self.validator = utility.JSONValidator(json_validation_schema_path) - - # Load secret configurations only - self._secret_configs = self._load_secrets() - - def _load_file(self, paths: List[str], config_type: str) -> Dict[str, Any]: - """Load a single secret file from a list of possible paths.""" - existing_files = [path for path in paths if os.path.exists(path)] - - if len(existing_files) > 1: - raise ValueError(f"Multiple {config_type} secrets files found. Only one is allowed: {existing_files}") - - if len(existing_files) == 1: - file_path = existing_files[0] - - self.logger.info("Retrieving %s secrets from: %s", config_type, file_path) - return utility.load_config_file_auto(file_path, False) - - self.logger.warning("No %s secrets file found.", config_type) - return {} - - def _load_secrets_config_from_files(self) -> Dict[str, Any]: - """Load and merge framework and pipeline secret configurations.""" - framework_secrets = self._load_file(self.framework_secrets_config_paths, "framework") - pipeline_secrets = self._load_file(self.pipeline_secrets_config_paths, "pipeline") - - return utility.merge_dicts_recursively(pipeline_secrets, framework_secrets) - - def _load_secrets(self) -> Dict[str, SecretConfig]: - """Load and merge framework and pipeline secret configurations.""" - json_data = self._load_secrets_config_from_files() - errors = self.validator.validate(json_data) - if errors: - raise ValueError(f"Secrets validation errors: {errors}") - - try: - return { - alias: SecretConfig(**config) - for alias, config - in json_data.items() - } - except KeyError as e: - raise ValueError(f"Invalid secret configuration: {e}") from e - except TypeError as e: - raise ValueError(f"Invalid secret configuration: {e}") from e - except Exception as e: - raise e - - def get_secret(self, alias: str) -> SecretValue: - """ - Get a SecretValue object for a secret by its alias. - - Args: - alias: The alias name of the secret - - Returns: - A SecretValue object that will lazily retrieve the secret when accessed - """ - if alias not in self._secret_configs: - raise KeyError(f"Secret alias '{alias}' not found") - - return self._secret_configs[alias].get_secret(self.dbutils) - - def substitute_secrets(self, data: Any) -> Any: - """ - Substitute secret references in a dictionary with SecretValue objects. - - Args: - data: The data to process (dict, list, or any other type) - - Returns: - Processed data with secret references replaced by SecretValue objects - """ - def substitute_value(value: Any) -> Any: - match = self.SECRET_PATTERN.match(value) - if match: - secret_alias = match.group(1) - return self.get_secret(secret_alias) - else: - return value - - if isinstance(data, dict): - return {k: self.substitute_secrets(v) for k, v in data.items()} - elif isinstance(data, list): - return [self.substitute_secrets(item) for item in data] - elif isinstance(data, str): - return substitute_value(data) - else: - return data +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.secrets_manager import SecretsManager +from lakeflow_framework.secrets_manager import ( # noqa: F401 + SecretConfig, + SecretValue, + SecretsManager, +) diff --git a/src/substitution_manager.py b/src/substitution_manager.py index 4afb3e3..66dd72b 100644 --- a/src/substitution_manager.py +++ b/src/substitution_manager.py @@ -1,186 +1,3 @@ -import re -import os -from typing import Dict, Any, Optional, Pattern, List -from functools import cached_property - -import utility -import pipeline_config - -class SubstitutionManager(): - """ - Manages token substitutions in strings and nested dictionaries. - - This class handles token replacements in configuration files, supporting both - framework-level and pipeline-level substitutions with optional additional tokens. - - Attributes: - framework_substitutions_path (str): Path to framework substitutions JSON - pipeline_substitutions_path (str): Path to pipeline substitutions JSON - additional_tokens (Optional[Dict[str, str]]): Optional dictionary of additional token replacements - - Methods: - substitute_string(input_string: str) -> str: - Substitute tokens in a string. - - substitute_dict(input_dict: Dict[str, Any]) -> Dict[str, Any]: - Substitute tokens in a nested dictionary. - """ - - # Compiled regex patterns for token substitution - DEFAULT_TOKEN_PATTERN: Pattern = re.compile(r'\{(\w+)\}') - - def __init__( - self, - framework_substitutions_paths: List[str], - pipeline_substitutions_paths: List[str], - additional_tokens: Optional[Dict[str, str]] = None - ): - """Initialize the SubstitutionManager. - - Args: - framework_substitutions_paths: List of paths to framework substitutions files. - pipeline_substitutions_paths: List of paths to pipeline substitutions files. - additional_tokens: Optional dictionary of additional token replacements - """ - if not framework_substitutions_paths or not pipeline_substitutions_paths: - raise ValueError("Framework and pipeline substitution paths must be provided as a list.") - - self.framework_substitutions_paths = framework_substitutions_paths - self.pipeline_substitutions_paths = pipeline_substitutions_paths - self.additional_tokens = additional_tokens or {} - - self.logger = pipeline_config.get_logger() - - self._substitutions_config = self._load_substitution_config() - - def _load_file(self, paths: List[str], config_type: str) -> Dict[str, Any]: - """Load a single substitution file from a list of possible paths.""" - existing_files = [path for path in paths if os.path.exists(path)] - - if len(existing_files) > 1: - raise ValueError(f"Multiple {config_type} substitutions files found. Only one is allowed: {existing_files}") - - if len(existing_files) == 1: - file_path = existing_files[0] - self.logger.info("Retrieving %s substitutions from: %s", config_type, file_path) - return utility.load_config_file_auto(file_path, False) or {} - - self.logger.warning("No %s substitutions file found.", config_type) - return {} - - def _load_substitution_config(self) -> Dict[str, Any]: - """Load and merge framework and pipeline substitutions.""" - framework_subs = self._load_file(self.framework_substitutions_paths, "framework") - pipeline_subs = self._load_file(self.pipeline_substitutions_paths, "pipeline") - - return utility.merge_dicts_recursively(pipeline_subs, framework_subs) - - @cached_property - def tokens(self) -> Dict[str, str]: - """Get merged tokens with additional tokens applied.""" - tokens = self._substitutions_config.get('tokens', {}) - if self.additional_tokens: - tokens = utility.merge_dicts_recursively(tokens, self.additional_tokens) - - # Apply substitutions to token values themselves - return { - k: self._substitute_tokens_in_string(v, tokens) - for k, v in tokens.items() - } - - @cached_property - def prefix_suffix_rules(self) -> Dict[str, Dict[str, str]]: - """Get prefix/suffix rules with tokens substituted.""" - rules = self._substitutions_config.get('prefix_suffix', {}) - - # Apply substitutions to prefix/suffix values - return { - k: { - rule_k: self._substitute_tokens_in_string(rule_v, self.tokens) - for rule_k, rule_v in v.items() - } - for k, v in rules.items() - } - - def substitute_string(self, input_string: str, additional_tokens: Optional[Dict[str, Any]] = None) -> str: - """Substitute tokens in a string. - - Args: - input_string: String containing tokens to replace - additional_tokens: Optional dictionary of additional token replacements. - Dictionary values will be converted to strings, with nested dictionaries being JSON serialized. - - Returns: - str: String with all tokens substituted - """ - if not isinstance(input_string, str): - raise TypeError(f"Expected string input, got {type(input_string)}") - - tokens = self.tokens - if additional_tokens: - tokens = utility.merge_dicts_recursively(self.tokens.copy(), additional_tokens) - - return self._substitute_tokens_in_string(input_string, tokens) - - def substitute_dict(self, data: Any) -> Any: - """ - Substitute tokens and apply prefix/suffix rules to a string or dictionary. - - Args: - data: The data to process (dict, list, or string) - - Returns: - Processed data with tokens references replaced by their values - """ - result = self._substitute_tokens(data) - result = self._apply_prefix_suffix(result) - return result - - def _substitute_tokens_in_string( - self, - value: str, - tokens: Dict[str, str], - pattern: Optional[Pattern] = None - ) -> str: - """Replace tokens in a string using regex substitution.""" - if not isinstance(value, str): - return value - - def replace_token(match): - token = match.group(1) - if token in self.additional_tokens: - return self.additional_tokens[token] - elif token in tokens: - return tokens[token] - return match.group(0) - - return (pattern or self.DEFAULT_TOKEN_PATTERN).sub(replace_token, value) - - def _substitute_tokens(self, data: Any) -> Any: - """Substitute tokens in a string or dictionary""" - if isinstance(data, dict): - return {k: self._substitute_tokens(v) for k, v in data.items()} - elif isinstance(data, list): - return [self._substitute_tokens(item) for item in data] - elif isinstance(data, str): - return self._substitute_tokens_in_string(data, self.tokens, self.DEFAULT_TOKEN_PATTERN) - else: - return data - - def _apply_prefix_suffix(self, data: Any) -> Any: - """Apply prefix/suffix rules to a string or dictionary.""" - def apply_value(key: str, value: Any) -> Any: - if isinstance(value, str) and key in self.prefix_suffix_rules: - rules = self.prefix_suffix_rules[key] - if 'prefix' in rules: - value = rules['prefix'] + value - if 'suffix' in rules: - value = value + rules['suffix'] - return value - - if isinstance(data, dict): - return {k: apply_value(k, v) for k, v in data.items()} - elif isinstance(data, list): - return [self._apply_prefix_suffix(item) for item in data] - else: - return data \ No newline at end of file +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework.substitution_manager import SubstitutionManager +from lakeflow_framework.substitution_manager import SubstitutionManager # noqa: F401 diff --git a/src/utility.py b/src/utility.py index d29665e..e8d746a 100644 --- a/src/utility.py +++ b/src/utility.py @@ -1,535 +1,4 @@ -import concurrent.futures -import importlib.util -import inspect -from functools import reduce -import logging -import os -from typing import Callable, Dict, List - -import json -import jsonschema as js -import yaml - -from pyspark.sql import DataFrame -from pyspark.sql import SparkSession -from pyspark.sql.types import StructType - -from constants import ( - SupportedSpecFormat, - PipelineBundleSuffixesJson, - PipelineBundleSuffixesYaml, -) - - -def get_format_suffixes(file_format: str, suffix_type: str) -> list: - """Get file suffixes based on file format and suffix type. - - This is a centralized utility to retrieve the appropriate file suffixes - for different configuration and specification files based on the format - (JSON or YAML) and the type of file (substitutions, secrets, specs, etc.).""" - suffix_map = { - SupportedSpecFormat.JSON.value: { - "substitutions": PipelineBundleSuffixesJson.SUBSTITUTIONS_FILE_SUFFIX, - "secrets": PipelineBundleSuffixesJson.SECRETS_FILE_SUFFIX, - "main_spec": PipelineBundleSuffixesJson.MAIN_SPEC_FILE_SUFFIX, - "flow_group": PipelineBundleSuffixesJson.FLOW_GROUP_FILE_SUFFIX, - "expectations": PipelineBundleSuffixesJson.EXPECTATIONS_FILE_SUFFIX - }, - SupportedSpecFormat.YAML.value: { - "substitutions": PipelineBundleSuffixesYaml.SUBSTITUTIONS_FILE_SUFFIX, - "secrets": PipelineBundleSuffixesYaml.SECRETS_FILE_SUFFIX, - "main_spec": PipelineBundleSuffixesYaml.MAIN_SPEC_FILE_SUFFIX, - "flow_group": PipelineBundleSuffixesYaml.FLOW_GROUP_FILE_SUFFIX, - "expectations": PipelineBundleSuffixesYaml.EXPECTATIONS_FILE_SUFFIX - } - } - - if file_format not in suffix_map: - valid_formats = list(suffix_map.keys()) - raise ValueError( - f"Invalid file format: '{file_format}'. " - f"Valid formats are: {valid_formats}" - ) - - if suffix_type not in suffix_map[file_format]: - valid_types = list(suffix_map[file_format].keys()) - raise ValueError( - f"Invalid suffix type: '{suffix_type}'. " - f"Valid types are: {valid_types}" - ) - - result = suffix_map[file_format][suffix_type] - - # Always return a list for consistency - if isinstance(result, list): - return result - elif isinstance(result, (tuple, set)): - return list(result) - elif isinstance(result, str): - return [result] - else: - raise TypeError( - f"Invalid suffix type for '{suffix_type}': expected str, tuple, list, or set, " - f"but got {type(result).__name__}" - ) - -class JSONValidator: - """ - A JSON schema validator class. - - Attributes: - schema (dict): The JSON schema loaded from a file. - base_uri (str): The base URI for resolving schema references. - resolver (RefResolver): The JSON schema resolver. - validator (Draft7Validator): The JSON schema validator. - - Methods: - validate(json_data: Dict) -> List: - Validates the provided JSON data against the loaded schema and returns a list of validation errors. - """ - - def __init__(self, schema_path: str): - try: - with open(schema_path, "r", encoding="utf-8") as schema_file: - self.schema = json.load(schema_file) - except Exception as e: - raise ValueError(f"JSON Schema not found: {schema_path}") from e - - # Resolve references - self.base_uri = "file://" + os.path.abspath(os.path.dirname(schema_path)) + "/" - self.resolver = js.RefResolver(base_uri=self.base_uri, referrer=self.schema) - self.validator = js.Draft7Validator(self.schema, resolver=self.resolver) - - def validate(self, json_data: Dict) -> List: - """Validate the provided JSON data against the loaded schema and returns a list of validation errors.""" - return list(self.validator.iter_errors(json_data)) - - -def add_struct_field(struct: StructType, column: Dict): - """Add a field to a StructType schema.""" - return struct.jsonValue()["fields"].append(column) - - -def drop_columns(df: DataFrame, columns_to_drop: List) -> DataFrame: - """Drop columns from a DataFrame.""" - drop_column_list = [] - for column in columns_to_drop: - if column in df.columns: - drop_column_list.append(column) - if drop_column_list: - df = df.drop(*drop_column_list) - return df - - -def load_config_file(file_path: str, file_format: str = "json", fail_on_not_exists: bool = True) -> Dict: - """Load JSON or YAML data from a file.""" - if not os.path.exists(file_path): - if fail_on_not_exists: - raise ValueError(f"Path does not exist: {file_path}") - return {} - - with open(file_path, 'r', encoding='utf-8') as file: - try: - if file_format == "json": - return json.load(file) - elif file_format == "yaml": - return yaml.safe_load(file) - else: - raise ValueError(f"Invalid file format: {file_format}. Only 'json' and 'yaml' are supported.") - except json.JSONDecodeError as e: - raise ValueError( - f"Error loading JSON file '{file_path}': {e.msg} at line {e.lineno}, column {e.colno}" - ) from e - except yaml.YAMLError as e: - raise yaml.YAMLError(f"Error loading YAML file '{file_path}': {e}") - - -def load_config_files( - path: str, - file_format: str = "json", - file_suffix: str | List[str] = None, - recursive: bool = False -) -> Dict: - """Load configuration data from files with a specific suffix.""" - if file_suffix is None: - file_suffix = f".{file_format}" - - data = {} - if not path or path.strip() == "" or not os.path.exists(path): - raise ValueError(f"Path does not exist: {path}") - - file_suffix_list = [file_suffix] if isinstance(file_suffix, str) else file_suffix - - if recursive: - for root, _, filenames in os.walk(path): - for filename in filenames: - if any(filename.endswith(suffix) for suffix in file_suffix_list): - file_path = os.path.join(root, filename) - data[file_path] = load_config_file(file_path, file_format) - else: - for filename in os.listdir(path): - if any(filename.endswith(suffix) for suffix in file_suffix_list): - file_path = os.path.join(path, filename) - data[file_path] = load_config_file(file_path, file_format) - - return data - - -def load_config_file_auto(file_path: str, fail_on_not_exists: bool = True) -> Dict: - """Load JSON or YAML data from a file with automatic format detection. - - The file format is automatically detected based on the file extension: - - .json -> JSON format - - .yaml, .yml -> YAML format - - Args: - file_path: Path to the configuration file - fail_on_not_exists: Whether to raise an error if file doesn't exist - - Returns: - Dict containing the loaded configuration data - - Raises: - ValueError: If file extension is not recognized or path doesn't exist - """ - file_ext = os.path.splitext(file_path)[1].lower() - - if file_ext == '.json': - file_format = 'json' - elif file_ext in ('.yaml', '.yml'): - file_format = 'yaml' - else: - raise ValueError( - f"Unable to detect file format from extension '{file_ext}'. " - f"Supported extensions: .json, .yaml, .yml" - ) - - return load_config_file(file_path, file_format, fail_on_not_exists) - - -# TODO: Legacy wrapper around load_config_file. Remove this in a future release. -def get_json_from_file(file_path: str, fail_on_not_exists: bool = True) -> Dict: - """Load JSON data from a file.""" - return load_config_file(file_path, "json", fail_on_not_exists) - - -# TODO: Legacy wrapper around load_config_files. Remove this in a future release. -def get_json_from_files(path: str, file_suffix: str | List[str] = ".json", recursive: bool = False) -> Dict: - """Load JSON data from files that have a specific suffix.""" - return load_config_files(path, "json", file_suffix, recursive) - - -# TODO: Legacy wrapper around load_config_file. Remove this in a future release. -def get_yaml_from_file(file_path: str, fail_on_not_exists: bool = True) -> Dict: - """Load YAML data from a file.""" - return load_config_file(file_path, "yaml", fail_on_not_exists) - - -# TODO: Legacy wrapper around load_config_files. Remove this in a future release. -def get_yaml_from_files(path: str, file_suffix: str | List[str] = ".yaml", recursive: bool = False) -> Dict: - """Load YAML data from files with a specific suffix.""" - return load_config_files(path, "yaml", file_suffix, recursive) - - -def get_data_from_files_parallel( - path: str, - file_format: str, - file_suffix: str | List[str], - recursive: bool = False, - max_workers: int = 10, - logger=None, -) -> Dict: - """ - Load data from JSON or YAML files that have a specific suffix using parallel processing. - - Args: - path: Directory path to search for files - file_format: File format to load. ["json", "yaml"] - file_suffix: File suffixes to filter by e.g. ".json" or ["_main.json", "_flow.json"] - recursive: Whether to search recursively in subdirectories - max_workers: Maximum number of worker threads for parallel loading - - Returns: - Dict mapping file suffix to a list of file paths to their loaded JSON data - - Example output: - { - ".json": { - "/path/to/file1.json": { - "data": { - "key": "value" - } - }, - "/path/to/file2.json": { - "data": { - "key": "value" - } - } - } - } - - Raises: - ValueError: If the path doesn't exist - """ - def _discover_files(path: str, file_suffix: str | List[str], recursive: bool) -> List[str]: - file_path_list = [] - if recursive: - for root, _, filenames in os.walk(path): - for filename in filenames: - if filename.endswith(file_suffix): - file_path_list.append(os.path.join(root, filename)) - else: - for filename in os.listdir(path): - if filename.endswith(file_suffix): - file_path_list.append(os.path.join(path, filename)) - - return file_path_list - - def load_single_file(file_path): - try: - return file_path, load_config_file(file_path, file_format), None - except Exception as e: - return file_path, None, str(e) - - # Validate path - if not path or path.strip() == "" or not os.path.exists(path): - raise ValueError(f"Path does not exist: {path}") - - file_suffix_list = [file_suffix] if isinstance(file_suffix, str) else file_suffix - file_paths = {} - data = {} - errors = {} - for suffix in file_suffix_list: - file_paths[suffix] = _discover_files(path, suffix, recursive) - - for file_suffix, file_paths in file_paths.items(): - if logger is not None: - logger.debug("Loading %s files...", file_suffix) - logger.debug("File paths: %s", file_paths) - data[file_suffix] = {} - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - future_to_path = { - executor.submit(load_single_file, file_path): file_path - for file_path in file_paths - } - - for future in concurrent.futures.as_completed(future_to_path): - file_path, file_data, error = future.result() - if error: - errors[file_path] = error - if logger is not None: - logger.debug("Failed to load %s: %s", file_path, error) - elif file_data is not None: - data[file_suffix][file_path] = file_data - - if errors: - if logger is not None: - logger.debug("%d files failed to load.", len(errors)) - for file_path, error in errors.items(): - logger.debug("Failed: %s: %s", file_path, error) - - return data - - -def get_pipeline_update_id(spark: SparkSession) -> str: - """ - Get the pipeline update id from the spark conf. - This is only populated post initialisation of the pipeline, by an event hook in DltPipelineBuilder. - - Args: - spark (SparkSession): The Spark session to use for the pipeline. - - Returns: - str: The pipeline update id. - """ - return spark.conf.get("pipeline.pipeline_update_id", None) - - -def get_table_versions(spark, source_view_dict: Dict[str, str]) -> DataFrame: - """Get table versions from a Spark DataFrame.""" - df_list = [] - for view_name, source_table in source_view_dict.items(): - sql = f"DESCRIBE HISTORY {source_table} LIMIT 1" - select_expr = [ - f"'{view_name}' AS viewName", - f"'{source_table}' AS tableName", - "version" - ] - df = spark.sql(sql).selectExpr(select_expr) - df_list.append(df) - return reduce(DataFrame.unionAll, df_list) if len(df_list) > 1 else df_list[0] - - -def load_python_function( - python_function_path: str, - function_name: str, - required_params: List[str] = None -) -> Callable: - """Load and validate a Python function from a file.""" - if required_params is None: - required_params = [] - - spec = importlib.util.spec_from_file_location("module", python_function_path) - if not spec or not spec.loader: - raise ImportError(f"Could not load Python function from {python_function_path}") - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - # Validate function exists - if not hasattr(module, function_name): - raise AttributeError( - f"Python function file '{python_function_path}' must contain a " - f"'{function_name}' function with parameters: {', '.join(required_params)}" - ) - - function = getattr(module, function_name) - if not callable(function): - raise TypeError(f"'{function_name}' in '{python_function_path}' is not callable") - - # Inspect signature - sig = inspect.signature(function) - func_params = list(sig.parameters.keys()) - missing_params = [p for p in required_params if p not in func_params] - if missing_params: - raise ValueError( - f"Function '{function_name}' in '{python_function_path}' is missing " - f"required parameters: {', '.join(missing_params)}" - ) - - return function - - -def load_python_function_from_module( - python_module: str, - required_params: List[str] = None -) -> Callable: - """ - Load and validate a Python function from an extension module. - - The module must be importable via sys.path (typically from extensions/libraries, - which is added to sys.path during pipeline initialization). - - Args: - python_module: Module and function reference in format 'module_name.function_name' - (e.g., 'transforms.customer_source' or 'my_module.sub.get_data') - required_params: List of required parameter names for validation - - Returns: - The callable function from the module - - Raises: - ValueError: If python_module format is invalid - ImportError: If module cannot be imported - AttributeError: If function is not found in module - TypeError: If the attribute is not callable - """ - if required_params is None: - required_params = [] - - # Parse module and function name - if "." not in python_module: - raise ValueError( - f"Invalid pythonModule format: '{python_module}'. " - f"Expected format: 'module_name.function_name' (e.g., 'transforms.get_df')" - ) - - # Split on last dot to get module path and function name - module_path, function_name = python_module.rsplit(".", 1) - - # Import the module - try: - module = importlib.import_module(module_path) - except ModuleNotFoundError as e: - raise ImportError( - f"Could not import module '{module_path}' for pythonModule '{python_module}'. " - f"Ensure the module exists in the extensions directory. Original error: {e}" - ) from e - - # Get the function - if not hasattr(module, function_name): - raise AttributeError( - f"Module '{module_path}' does not contain function '{function_name}'. " - f"Available attributes: {[a for a in dir(module) if not a.startswith('_')]}" - ) - - function = getattr(module, function_name) - if not callable(function): - raise TypeError( - f"'{function_name}' in module '{module_path}' is not callable" - ) - - # Validate required parameters - if required_params: - sig = inspect.signature(function) - func_params = list(sig.parameters.keys()) - missing_params = [p for p in required_params if p not in func_params] - if missing_params: - raise ValueError( - f"Function '{function_name}' in module '{module_path}' is missing " - f"required parameters: {', '.join(missing_params)}" - ) - - return function - - -def list_sub_paths(path: str) -> List[str]: - """List subdirectories in a given directory path.""" - return [x for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))] - - -def merge_dicts(*dicts: Dict) -> Dict: - """Merge dictionaries into a single dictionary.""" - return reduce(lambda a, b: {**a, **b} if b is not None else a, dicts, {}) - - -def merge_dicts_recursively(d1: Dict, d2: Dict) -> Dict: - """Recursively merges two dictionaries. Keys in d1 take precedence over d2.""" - d = d1.copy() - - for key in d2: - if key in d1: - if isinstance(d1[key], dict) and isinstance(d2[key], dict): - d[key] = merge_dicts_recursively(d1[key], d2[key]) - else: - d[key] = d2[key] - - return d - - -def replace_dict_key_value(spec: Dict, target_key: str, new_value: str) -> Dict: - """Replace values of a specific key in a nested dictionary.""" - if isinstance(spec, dict): - for key, value in spec.items(): - if key == target_key: - if spec[key] is not None and spec[key].strip() != "": - spec[key] = f"{new_value}/{spec[key]}" - elif isinstance(value, dict) or isinstance(value, list): - replace_dict_key_value(value, target_key, new_value) - elif isinstance(spec, list): - for item in spec: - replace_dict_key_value(item, target_key, new_value) - return spec - - -def set_logger(logger_name: str, log_level: str = "INFO") -> logging.Logger: - """Here for backwards compatibility. Use resolve_logger instead. Set up and return a logger with a specified name and log level.""" - from logger import create_default_logger - - return create_default_logger(logger_name, log_level) - - -def deep_merge(base: Dict, overlay: Dict) -> Dict: - """Recursively merge *overlay* into *base*, returning a new dict. - - - Dict values are merged recursively; overlay wins on key conflicts. - - Non-dict values (including lists) are replaced wholesale by the overlay value. - - Neither input is mutated. - """ - result = base.copy() - for key, val in overlay.items(): - if key in result and isinstance(result[key], dict) and isinstance(val, dict): - result[key] = deep_merge(result[key], val) - else: - result[key] = val - return result +# Compat shim — kept until v1.0.0. +# Use: from lakeflow_framework import utility or from lakeflow_framework.utility import ... +from lakeflow_framework import utility # noqa: F401 +from lakeflow_framework.utility import * # noqa: F401, F403 diff --git a/tests/test_bundle_loader.py b/tests/test_bundle_loader.py index 197e895..9dcd3f4 100644 --- a/tests/test_bundle_loader.py +++ b/tests/test_bundle_loader.py @@ -12,7 +12,7 @@ import pytest -from bundle_loader import register_bundle_sys_paths, run_init_scripts +from lakeflow_framework.bundle_loader import register_bundle_sys_paths, run_init_scripts logger = logging.getLogger("test_bundle_loader") diff --git a/tests/test_package.py b/tests/test_package.py new file mode 100644 index 0000000..24c9aeb --- /dev/null +++ b/tests/test_package.py @@ -0,0 +1,57 @@ +""" +Smoke tests for the ``lakeflow_framework`` package. + +Verifies: +- ``import lakeflow_framework`` resolves correctly (package is on sys.path via src/). +- ``lakeflow_framework.__version__`` is a non-empty string. +- Core public API symbols are importable from the canonical path. +- Compat shims at ``src/*.py`` continue to re-export the same symbols. +""" +from __future__ import annotations + +import sys + + +def test_import_package(): + import lakeflow_framework # noqa: F401 — just ensure no ImportError + + +def test_version_is_set(): + import lakeflow_framework + assert isinstance(lakeflow_framework.__version__, str) + assert lakeflow_framework.__version__ != "" + assert lakeflow_framework.__version__ != "unknown" + + +def test_core_api_importable(): + from lakeflow_framework import DLTPipelineBuilder + from lakeflow_framework import DataflowSpecBuilder + from lakeflow_framework import DataFlow + from lakeflow_framework import DataflowSpec + from lakeflow_framework import SecretsManager + from lakeflow_framework import SubstitutionManager + from lakeflow_framework.constants import FrameworkPaths, PipelineBundlePaths + + +def test_contrib_importable(): + import lakeflow_framework.contrib # noqa: F401 + + +def test_compat_shim_constants(): + """src/constants.py shim must re-export FrameworkPaths unchanged.""" + from lakeflow_framework.constants import FrameworkPaths as canonical + from constants import FrameworkPaths as shim # type: ignore[import] + assert canonical is shim + + +def test_compat_shim_pipeline_config(): + """src/pipeline_config.py shim must export get_spark.""" + import pipeline_config # type: ignore[import] + assert hasattr(pipeline_config, "get_spark") + + +def test_compat_shim_dlt_pipeline_builder(): + """src/dlt_pipeline_builder.py shim must export DLTPipelineBuilder.""" + from dlt_pipeline_builder import DLTPipelineBuilder as shim # type: ignore[import] + from lakeflow_framework.dlt_pipeline_builder import DLTPipelineBuilder as canonical + assert shim is canonical diff --git a/tests/test_strategy_b_resolver.py b/tests/test_strategy_b_resolver.py new file mode 100644 index 0000000..9ea161e --- /dev/null +++ b/tests/test_strategy_b_resolver.py @@ -0,0 +1,166 @@ +""" +Unit tests for ``load_framework_default_json()`` and ``load_framework_schema()`` +— Strategy B Workspace Files-first resolver. + +Coverage: +- Workspace Files-only: file exists in Workspace Files → loads from there (no package data involved). +- Package-data-only: no framework_path → loads via importlib.resources. +- Both sources: Workspace Files takes priority over package data. +- Sparse local/config overlay: only overridden keys change; unset keys come from base. +- Missing everywhere: FileNotFoundError raised. +- load_framework_schema() helper returns a readable traversable. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from lakeflow_framework.config_resolver import load_framework_default_json, load_framework_schema +from lakeflow_framework.constants import FrameworkPaths + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _write_json(path: Path, data: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data), encoding="utf-8") + + +def _config_dir(framework_path: Path) -> Path: + """Return the on-disk config/default directory under framework_path.""" + return framework_path / FrameworkPaths.CONFIG_PATH.lstrip("./") + + +def _local_dir(framework_path: Path) -> Path: + """Return the on-disk local/config directory under framework_path.""" + return framework_path / FrameworkPaths.LOCAL_CONFIG_PATH.lstrip("./") + + +# --------------------------------------------------------------------------- +# Disk-only tests +# --------------------------------------------------------------------------- + +class TestWorkspaceFilesFirst: + def test_loads_from_workspace_files_when_file_exists(self, tmp_path): + data = {"key": "workspace_value", "nested": {"a": 1}} + _write_json(_config_dir(tmp_path) / "test.json", data) + + result = load_framework_default_json("test.json", str(tmp_path)) + + assert result == data + + def test_workspace_files_takes_priority_over_package_data(self, tmp_path): + """When file is in Workspace Files, importlib.resources must NOT be consulted.""" + in_workspace = {"source": "workspace_files"} + _write_json(_config_dir(tmp_path) / "global.json", in_workspace) + + # If package data were used it would return the real global.json content + result = load_framework_default_json("global.json", str(tmp_path)) + + assert result["source"] == "workspace_files" + + def test_workspace_files_missing_falls_through_to_package_data(self, tmp_path): + """File not in Workspace Files → package data is used (no FileNotFoundError).""" + # tmp_path has no config dir — falls through to importlib.resources + result = load_framework_default_json("global.json", str(tmp_path)) + + # Real package data contains at least "global" / pipeline keys + assert isinstance(result, dict) + assert len(result) > 0 + + +# --------------------------------------------------------------------------- +# Package-data-only test (no framework_path) +# --------------------------------------------------------------------------- + +class TestPackageDataFallback: + def test_no_framework_path_loads_from_package(self): + """When framework_path is None, importlib.resources is used directly.""" + result = load_framework_default_json("global.json") + + assert isinstance(result, dict) + assert len(result) > 0 + + def test_nonexistent_file_raises_file_not_found(self): + with pytest.raises(FileNotFoundError, match="nonexistent_file.json"): + load_framework_default_json("nonexistent_file.json") + + def test_nonexistent_file_with_framework_path_raises(self, tmp_path): + with pytest.raises(FileNotFoundError): + load_framework_default_json("nonexistent_file.json", str(tmp_path)) + + +# --------------------------------------------------------------------------- +# Overlay (local/config) tests +# --------------------------------------------------------------------------- + +class TestLocalConfigOverlay: + def test_overlay_merges_on_top_of_disk_base(self, tmp_path): + base = {"logging": {"level": "INFO"}, "other": "unchanged"} + overlay = {"logging": {"level": "DEBUG"}} # sparse — only changes level + _write_json(_config_dir(tmp_path) / "global.json", base) + _write_json(_local_dir(tmp_path) / "global.json", overlay) + + result = load_framework_default_json("global.json", str(tmp_path)) + + assert result["logging"]["level"] == "DEBUG" + assert result["other"] == "unchanged" + + def test_overlay_merges_on_top_of_package_base(self, tmp_path): + """Overlay applied even when disk base file is absent (falls to package data).""" + overlay = {"_test_override_key": "overlay_value"} + _write_json(_local_dir(tmp_path) / "global.json", overlay) + + result = load_framework_default_json("global.json", str(tmp_path)) + + assert result["_test_override_key"] == "overlay_value" + + def test_no_overlay_file_returns_base_unchanged(self, tmp_path): + base = {"key": "value"} + _write_json(_config_dir(tmp_path) / "global.json", base) + + result = load_framework_default_json("global.json", str(tmp_path)) + + assert result == base + + def test_empty_overlay_does_not_alter_base(self, tmp_path): + base = {"key": "value"} + _write_json(_config_dir(tmp_path) / "global.json", base) + _write_json(_local_dir(tmp_path) / "global.json", {}) + + result = load_framework_default_json("global.json", str(tmp_path)) + + assert result == base + + +# --------------------------------------------------------------------------- +# load_framework_schema() helper +# --------------------------------------------------------------------------- + +class TestLoadFrameworkSchema: + def test_returns_readable_traversable(self): + traversable = load_framework_schema("main.json") + content = traversable.read_text("utf-8") + parsed = json.loads(content) + assert isinstance(parsed, dict) + + def test_all_known_schemas(self): + schemas = [ + "main.json", + "flow_group.json", + "expectations.json", + "secrets.json", + "spec_template.json", + "spec_template_definition.json", + "spec_mapping.json", + ] + for name in schemas: + ref = load_framework_schema(name) + data = json.loads(ref.read_text("utf-8")) + assert isinstance(data, dict), f"{name} did not parse to a dict" diff --git a/tests/test_validate_dataflows.py b/tests/test_validate_dataflows.py index edd905a..47d4e13 100644 --- a/tests/test_validate_dataflows.py +++ b/tests/test_validate_dataflows.py @@ -64,16 +64,16 @@ def test_expanded_form_when_not_a_dict(self): class TestGetSchemaPath: def test_template_form_returns_spec_template_schema(self, tmp_path): result = vd.get_schema_path(tmp_path, vd.SPEC_FORM_TEMPLATE) - assert result == tmp_path / "src" / "schemas" / "spec_template.json" + assert result == tmp_path / "src" / "lakeflow_framework" / "schemas" / "spec_template.json" def test_expanded_form_returns_main_schema(self, tmp_path): result = vd.get_schema_path(tmp_path, vd.SPEC_FORM_EXPANDED) - assert result == tmp_path / "src" / "schemas" / "main.json" + assert result == tmp_path / "src" / "lakeflow_framework" / "schemas" / "main.json" def test_unknown_form_falls_back_to_main_schema(self, tmp_path): # Defensive: unknown form string should not crash; defaults to main.json result = vd.get_schema_path(tmp_path, "some_other_form") - assert result == tmp_path / "src" / "schemas" / "main.json" + assert result == tmp_path / "src" / "lakeflow_framework" / "schemas" / "main.json" # ----------------------------------------------------------------------------- From c58617883aa74876d76b0437f6d7a067192e638b Mon Sep 17 00:00:00 2001 From: rederik76 Date: Sat, 23 May 2026 16:15:13 +1000 Subject: [PATCH 2/9] Uddate description in toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9a0d2d3..446752b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.backends.legacy:build" [project] name = "lakeflow-framework" dynamic = ["version"] -description = "A metadata-driven framework for building Databricks Spark Declarative Pipelines" +description = "Metadata-driven framework for Databricks Spark Declarative Pipelines." readme = "README.md" license = { file = "LICENSE.md" } requires-python = ">=3.10" From 951bef231f0af8c53a89937ddc13b1f919e7433e Mon Sep 17 00:00:00 2001 From: rederik76 <127075662+rederik76@users.noreply.github.com> Date: Sun, 28 Jun 2026 09:57:10 +1000 Subject: [PATCH 3/9] merge main --- src/dataflow/cdc_snapshot.py | 715 ------------------ .../config/override/.gitkeep | 0 .../dataflow/cdc_snapshot.py | 94 ++- src/lakeflow_framework/utility.py | 29 +- 4 files changed, 101 insertions(+), 737 deletions(-) delete mode 100644 src/dataflow/cdc_snapshot.py rename src/{ => lakeflow_framework}/config/override/.gitkeep (100%) diff --git a/src/dataflow/cdc_snapshot.py b/src/dataflow/cdc_snapshot.py deleted file mode 100644 index 8a0392d..0000000 --- a/src/dataflow/cdc_snapshot.py +++ /dev/null @@ -1,715 +0,0 @@ -import bisect -from dataclasses import dataclass, field -from datetime import datetime -import re -from typing import Dict, List, Literal, Optional, Union - -from pyspark import pipelines as dp -from pyspark.sql import DataFrame -from pyspark.sql import functions as F -import pyspark.sql.types as T - -import pipeline_config - -from .dataflow_config import DataFlowConfig -from .sources import SourceDelta, SourceBatchFiles, ReadConfig - - -@dataclass(frozen=True) -class CDCSnapshotTypes: - """Constants for the types of CDC Snapshot.""" - HISTORICAL = "historical" - PERIODIC = "periodic" - - -@dataclass(frozen=True) -class CDCSnapshotSourceTypes: - """Constants for the types of CDC Snapshot source types.""" - FILE = "file" - TABLE = "table" - - -@dataclass(frozen=True) -class CDCSnapshotVersionTypes: - """Constants for the types of CDC Snapshot version types.""" - DATE = "date" - INTEGER = "integer" - LONG = "long" - TIMESTAMP = "timestamp" - - -@dataclass(frozen=True) -class DeduplicateMode: - """Deduplication strategy for CDC snapshot source data. - - off: no deduplication (default). - - full_row: deduplicate based on the full row using dropDuplicates(); deterministic as the full row is deduplicated. - - keys_only: deduplicate based on the keys using dropDuplicates(keys); non-deterministic as it preserves the first row per key(s) without ordering on any other columns.""" - OFF = "off" - FULL_ROW = "full_row" - KEYS_ONLY = "keys_only" - - -@dataclass -class VersionInfo: - """A structure to hold version information with both raw and formatted values.""" - raw_value: Union[str, int, datetime] - version_type: str - datetime_format: Optional[str] = None - micro_second_mask_length: Optional[int] = None - - @property - def formatted_value(self) -> str: - """Get formatted value based on version type and datetime format.""" - if self.version_type == CDCSnapshotVersionTypes.TIMESTAMP: - if isinstance(self.raw_value, datetime): - if self.datetime_format: - if '%f' in self.datetime_format and self.micro_second_mask_length: - truncate_from_right = 6 - self.micro_second_mask_length - return self.raw_value.strftime(self.datetime_format)[:-truncate_from_right] - else: - return self.raw_value.strftime(self.datetime_format) - else: - return self.raw_value.strftime('%Y-%m-%d %H:%M:%S') - else: - return str(self.raw_value) - else: - return str(self.raw_value) - - @property - def sql_formatted_value(self) -> str: - """Get SQL formatted value with appropriate quotes.""" - if self.version_type in [CDCSnapshotVersionTypes.TIMESTAMP, CDCSnapshotVersionTypes.DATE]: - return f"'{self.formatted_value}'" - elif self.version_type in [CDCSnapshotVersionTypes.INTEGER, CDCSnapshotVersionTypes.LONG]: - return f"'{self.formatted_value}'" - else: - raise ValueError(f"Unsupported version type: {self.version_type}") - -@dataclass -class FilePathInfo: - """A structure to hold file path information.""" - full_path: str - filename_with_version_path: str - -@dataclass -class CDCSnapshotFileSource: - """A structure to hold the source configuration for CDC Snapshot.""" - format: str - path: str - readerOptions: Dict = field(default_factory=dict) - filter: Optional[str] = None - versionType: Optional[str] = None - startingVersion: Optional[Union[int, str]] = None - datetimeFormat: Optional[str] = None - microSecondMaskLength: Optional[int] = None - schemaPath: Optional[str] = None - selectExp: Optional[List[str]] = None - recursiveFileLookup: bool = False - deduplicateMode: Optional[ - Literal[ - DeduplicateMode.OFF, - DeduplicateMode.FULL_ROW, - DeduplicateMode.KEYS_ONLY, - ] - ] = DeduplicateMode.OFF - - -@dataclass -class CDCSnapshotTableSource: - """A structure to hold the source configuration for CDC Snapshot.""" - table: str - versionColumn: str - versionType: str - startingVersion: Optional[Union[int, str]] = None - selectExp: Optional[List[str]] = None - deduplicateMode: Optional[ - Literal[ - DeduplicateMode.OFF, - DeduplicateMode.FULL_ROW, - DeduplicateMode.KEYS_ONLY, - ] - ] = DeduplicateMode.OFF - - -@dataclass -class CDCSnapshotSettings: - """CDC Settings for the SDP auto CDC Snapshot API.""" - keys: List[str] - scd_type: str - snapshotType: str - sourceType: str = None - source: Dict = field(default_factory=dict) - track_history_column_list: Optional[List[str]] = None - track_history_except_column_list: Optional[List[str]] = None - sequence_by_data_type: T.DataType = None - - def __post_init__(self): - if self.snapshotType == CDCSnapshotTypes.HISTORICAL and not self.source: - raise ValueError("Source is required for Historical CDC from Snapshot") - - if self.scd_type == "2": - # TODO: implement dynamic sequence by type - self.sequence_by_data_type = T.TimestampType() - - if self.snapshotType == CDCSnapshotTypes.HISTORICAL: - version_type = self.get_source().versionType - if version_type == CDCSnapshotVersionTypes.INTEGER: - self.sequence_by_data_type = T.IntegerType() - - def get_source(self) -> Optional[CDCSnapshotFileSource]: - """Get source configuration for CDC from Snapshot.""" - if self.sourceType == CDCSnapshotSourceTypes.FILE: - return CDCSnapshotFileSource(**self.source) - elif self.sourceType == CDCSnapshotSourceTypes.TABLE: - return CDCSnapshotTableSource(**self.source) - else: - raise ValueError(f"Unsupported source type: {self.sourceType}") - - def is_historical(self) -> bool: - """Is the CDC snapshot type historical.""" - return self.snapshotType == CDCSnapshotTypes.HISTORICAL - - def is_file_source(self) -> bool: - """Is the CDC snapshot source type file.""" - return self.sourceType == CDCSnapshotSourceTypes.FILE - - -class CDCSnapshotFlow: - """A class to create a CDC Snapshot flow.""" - - def __init__(self, settings: CDCSnapshotSettings): - self.settings = settings - self.logger = pipeline_config.get_logger() - - # Core CDC settings - self.keys = settings.keys - self.scd_type = settings.scd_type - self.snapshotType = settings.snapshotType - self.track_history_column_list = settings.track_history_column_list - self.track_history_except_column_list = settings.track_history_except_column_list - self.sequence_by_data_type = settings.sequence_by_data_type - - # Historical snapshot specific settings - self.sourceType = None - self.source = None - if self.snapshotType == CDCSnapshotTypes.HISTORICAL: - self.sourceType = settings.sourceType - self.source = settings.get_source() - - if self.source is None: - raise ValueError("Source configuration is required for historical snapshots") - - # Cached version data - self._available_versions: Optional[List[VersionInfo]] = None - self._sorted_versions: Optional[List[VersionInfo]] = None - self._version_values: Optional[List[Union[int, datetime]]] = None - - @property - def sorted_versions(self) -> List[VersionInfo]: - """Get sorted versions.""" - if self._sorted_versions is None and self._available_versions: - self._sorted_versions = sorted(self._available_versions, key=lambda x: x.raw_value) - return self._sorted_versions or [] - - @property - def version_values(self) -> List[Union[int, datetime]]: - """Get version values.""" - if self._version_values is None: - self._version_values = [v.raw_value for v in self.sorted_versions] - return self._version_values - - def _deduplicate_by_keys(self, df: DataFrame) -> DataFrame: - """Deduplicate by configured keys, retaining first row per key. WARNING: This is non-deterministic.""" - self.logger.warning( - "CDC Snapshot: deduplicateMode=keys_only is non-deterministic as it peserves the first row per key." - ) - return df.dropDuplicates(self.keys) - - def _deduplicate_full_row(self, df: DataFrame) -> DataFrame: - """Deduplicate by full row besides metadata column if present.""" - columns_to_deduplicate_on = [col for col in df.columns if col not in ["_metadata"]] - return df.dropDuplicates(columns_to_deduplicate_on) - - def create( - self, - dataflow_config: DataFlowConfig, - target_table: str, - source_view_name: Optional[str] = None, - target_config_flags: Optional[List[str]] = None, - flow_name: Optional[str] = None # TODO: Add flow name - ) -> None: - """Create CDC from snapshot flow. - - Args: - dataflow_config: DataFlow configuration - target_table: Name of the target table - source_view_name: Name of the source view - flow_name: Optional name for the flow - """ - self.logger.debug(f"CDC From Snapshot: {self.source}") - - try: - if self.snapshotType == CDCSnapshotTypes.PERIODIC: - self._apply_periodic_changes(target_table, source_view_name) - elif self.snapshotType == CDCSnapshotTypes.HISTORICAL: - self._apply_historical_changes(target_table, dataflow_config, target_config_flags) - else: - raise ValueError(f"Unsupported snapshot type: {self.snapshotType}") - except Exception as e: - self.logger.error(f"Failed to create CDC snapshot flow: {e}") - raise - - def _apply_periodic_changes(self, target_table: str, source_view_name: str) -> None: - """Apply periodic changes from snapshot.""" - dp.create_auto_cdc_from_snapshot_flow( - target=target_table, - source=source_view_name, - keys=self.keys, - stored_as_scd_type=self.scd_type, - track_history_column_list=self.track_history_column_list, - track_history_except_column_list=self.track_history_except_column_list - ) - - def _apply_historical_changes(self, target_table: str, dataflow_config: DataFlowConfig, target_config_flags: Optional[List[str]] = None): - """Apply historical changes from snapshot.""" - dp.create_auto_cdc_from_snapshot_flow( - target=target_table, - snapshot_and_version=lambda version: self._next_snapshot_and_version(version, dataflow_config, target_config_flags), - keys=self.keys, - stored_as_scd_type=self.scd_type, - track_history_column_list=self.track_history_column_list, - track_history_except_column_list=self.track_history_except_column_list - ) - - def _next_snapshot_and_version(self, latest_snapshot_version, dataflow_config: DataFlowConfig, target_config_flags: Optional[List[str]] = None): - """Get the next snapshot and version.""" - try: - if self._available_versions is None: - self._available_versions = self._get_available_versions(latest_snapshot_version) - - if not self._available_versions: - self.logger.warning("CDC Snapshot: No valid versions found") - return None - - version_info = self._get_next_version(latest_snapshot_version) - if version_info is None: - self.logger.debug("CDC Snapshot: Retrieving next version was None") - return None - - self.logger.info(f"CDC Snapshot: Reading file version: {version_info.formatted_value}") - df = self._read_snapshot_dataframe(version_info, dataflow_config, target_config_flags) - if df is None or df.isEmpty(): - self.logger.debug("CDC Snapshot: Retrieving snapshot dataframe was None or empty") - return None - - self.logger.info(f"CDC Snapshot: Returning dataframe with version: {version_info.formatted_value}. Raw version: {version_info.raw_value}.") - return (df, version_info.raw_value) - - except Exception as e: - self.logger.error(f"CDC Snapshot: Error processing snapshots: {e}", exc_info=True) - raise - - - def _get_available_versions(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> List[VersionInfo]: - """Get list of available versions from source.""" - if self.sourceType == CDCSnapshotSourceTypes.FILE: - return self._get_available_file_versions(latest_snapshot_version) - elif self.sourceType == CDCSnapshotSourceTypes.TABLE: - return self._get_available_table_versions(latest_snapshot_version) - else: - raise ValueError(f"Unsupported source type: {self.sourceType}") - - def _list_files(self, path: str, recursive: bool = True) -> List: - """List files in a directory, attempting dbutils.fs.ls() first. - - Falls back to Spark binaryFile if dbutils is unavailable (e.g., blocked - by Serverless Restricted Access / SEG Py4JSecurityException). - - Args: - path: Directory path to list files from. - recursive: If True, list files recursively. - - Returns: - List of objects with a .path attribute per discovered file. - """ - try: - dbutils = pipeline_config.get_dbutils() - all_files = [] - for f in dbutils.fs.ls(path): - all_files.append(f) - # Recursively list files in subdirectories unless the path ends with .parquet - if recursive and f.isDir() and not f.path.rstrip("/").endswith(".parquet"): - all_files.extend(self._list_files(f.path, recursive=True)) - return all_files - except Exception as e: - self.logger.warning( - f"CDC Snapshot: dbutils.fs.ls() failed at '{path}'. " - f"Falling back to Spark binaryFile for file discovery." - ) - - # Fallback: Spark binaryFile read is SEG-compatible and needs no dbutils. - self.logger.debug("CDC Snapshot: Falling back to Spark binaryFile for file discovery") - - class _FileInfo: - __slots__ = ("path", "name", "size", "modificationTime") - - # binaryFile returns full URIs (e.g. dbfs:/Volumes/...). Normalise to - # match the caller's path scheme so split-by-segment indices stay aligned. - _prefix = path.startswith("dbfs:") - - def __init__(self, row) -> None: - raw = row.path - if self._prefix: - self.path = raw if raw.startswith("dbfs:") else f"dbfs:{raw}" - else: - self.path = raw[5:] if raw.startswith("dbfs:") else raw - self.name = self.path.rstrip("/").rsplit("/", 1)[-1] - self.size = row.length - self.modificationTime = row.modificationTime - - spark = pipeline_config.get_spark() - rows = ( - spark.read - .format("binaryFile") - .option("recursiveFileLookup", str(recursive).lower()) - .load(path) - .select("path", "modificationTime", "length") - .collect() - ) - - # binaryFile only returns leaf files, not directories. For parquet "files" that - # are actually directories (e.g. customer.parquet/part-00000.parquet), truncate - # the path at the .parquet directory level and deduplicate so each snapshot is - # counted once. - def _truncate_at_parquet_dir(p: str) -> str: - segments = p.split("/") - for i, seg in enumerate(segments[:-1]): # skip the last segment - if seg.endswith(".parquet"): - return "/".join(segments[: i + 1]) - return p - - seen_paths = set() - files: List[_FileInfo] = [] - for row in rows: - fi = _FileInfo(row) - fi.path = _truncate_at_parquet_dir(fi.path) - fi.name = fi.path.rstrip("/").rsplit("/", 1)[-1] - if fi.path not in seen_paths: - seen_paths.add(fi.path) - files.append(fi) - - self.logger.debug( - f"CDC Snapshot: Spark binaryFile fallback listed {len(files)} file(s) under '{path}'" - ) - - return files - - def _path_to_regex_pattern(self, path: str) -> str: - """Convert path to normalized regex pattern with named groups. - - Curly-brace syntax is converted to named capture groups: - - {version} -> (?P.+) (single capture group for version) - - {fragment} -> (?P.*?) (single capture group for fragment) - - If path already contains regex named groups (?P, - it is returned as-is (already a regex pattern). - """ - if re.search(r'\(\?P', path): - return path - if '{version}' not in path and '{fragment}' not in path: - return path - escaped = re.escape(path) - normalized = ( - escaped - .replace(r'\{version\}', r'(?P.+)') - .replace(r'\{fragment\}', r'(?P.*?)') - ) - return normalized - - def _get_dynamic_path_index(self, path: str) -> int: - """Return index of first path segment that contains version or fragment (curly or regex).""" - path_parts = path.split('/') - for idx, part in enumerate(path_parts): - if '{version}' in part or '{fragment}' in part: - return idx - if '(?P' in part: - return idx - raise ValueError(f"No version or fragment placeholder found in path: {path}") - - def _get_version_string_from_match(self, match: re.Match) -> Optional[str]: - """From a regex match, concatenate all groups named version_* (sorted) to form version string.""" - groupdict = match.groupdict() - version_keys = [k for k in groupdict if k.startswith('version_')] - if not version_keys: - return None - return ''.join(groupdict[k] or '' for k in version_keys) - - def _has_fragment_group(self, pattern: str) -> bool: - """Return True if pattern has a fragment capture group.""" - if '{fragment}' in pattern: - return True - return bool(re.search(r'\(\?P', pattern)) - - def _get_available_file_versions(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> List[VersionInfo]: - """Get list of available versions from file path (supports regex and {version}/{fragment} syntax).""" - path = self.source.path - pattern = self._path_to_regex_pattern(path) - dynamic_idx = self._get_dynamic_path_index(path) - path_parts = path.split('/') - parent_dir = '/'.join(path_parts[:dynamic_idx]) or '.' - # Anchor pattern so we only match full path (e.g. exclude customer.parquet/_SUCCESS) - file_pattern_regex = '/'.join(pattern.split('/')[dynamic_idx:]) + r'/?$' - - self.logger.debug(f"CDC Snapshot: Listing files in {parent_dir} with pattern {file_pattern_regex}") - - # List files using the configured recursive file lookup option - recursive_file_lookup = self.source.recursiveFileLookup - self.logger.debug(f"CDC Snapshot: Using recursive file lookup: {recursive_file_lookup}") - files_list = self._list_files(parent_dir, recursive=recursive_file_lookup) - files_with_path_info = [FilePathInfo(full_path=f.path, filename_with_version_path='/'.join(f.path.split('/')[dynamic_idx:])) for f in files_list] - - self.logger.debug(f"CDC Snapshot: Found {len(files_with_path_info)} files") - - # Extract version from filename and filter by latest_snapshot_version if provided - available_versions = [] - for file in files_with_path_info: - self.logger.debug(f"CDC Snapshot: Processing file: {file.filename_with_version_path}") - try: - version_info = self._extract_version_from_filename(file.filename_with_version_path, file_pattern_regex) - if version_info is None: - continue - - self.logger.debug(f"CDC Snapshot: Extracted version from filename: {version_info.formatted_value}. Raw version: {version_info.raw_value}") - - if latest_snapshot_version is None and self.source.startingVersion is not None and version_info.raw_value < self.source.startingVersion: - continue - - if latest_snapshot_version is not None and version_info.raw_value <= latest_snapshot_version: - continue - - # Dedupe by version (multiple fragments can share same version) - if any(v.raw_value == version_info.raw_value for v in available_versions): - continue - available_versions.append(version_info) - self.logger.debug(f"CDC Snapshot: Added version {version_info.formatted_value} to available versions") - - except ValueError as e: - self.logger.warning(f"CDC Snapshot: Skipping file '{file.filename_with_version_path}' - {e}") - continue - - return available_versions - - def _get_available_table_versions(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> List[VersionInfo]: - """Get list of available versions from table.""" - spark = pipeline_config.get_spark() - table_name = self.source.table - - self.logger.info(f"CDC Snapshot: Getting versions from table: {table_name}") - try: - df = spark.table(table_name) - except Exception as e: - self.logger.error(f"CDC Snapshot: Error getting versions from table: {e}") - raise - - # Get the version column - version_column = self.source.versionColumn - - # Check if the version column is a valid data type - valid_data_types = ["timestamp", "date", "integer","long"] - version_column_type = df.schema[version_column].dataType.typeName() - if version_column_type not in valid_data_types: - raise ValueError(f"Version column: {version_column}, type: {version_column_type}, is not a valid data type: {valid_data_types}") - if version_column_type != self.source.versionType: - raise ValueError(f"Version column: {version_column}, type: {version_column_type}, does not match specified version type: {self.source.versionType}") - - # Get the version values and filter by latest_snapshot_version if provided - if latest_snapshot_version is not None: - latest_version_info = VersionInfo( - raw_value=latest_snapshot_version, - version_type=self.source.versionType) - version_df = df.select(version_column).where(f"{version_column} > {latest_version_info.sql_formatted_value}").distinct() - else: - version_df = df.select(version_column).distinct() - - available_versions = [] - for row in version_df.collect(): - version = row[version_column] - - if version is None: - continue - - if self.source.startingVersion is not None and version < self.source.startingVersion: - self.logger.debug(f"CDC Snapshot: Skipping version {version} because it is less than the starting version {self.source.startingVersion}") - continue - - if latest_snapshot_version is not None and version <= latest_snapshot_version: - self.logger.debug(f"CDC Snapshot: Skipping version {version} because it is less than or equal to the latest snapshot version {latest_snapshot_version}") - continue - - version_info = VersionInfo( - raw_value=version, - version_type=self.source.versionType, - datetime_format=None - ) - - available_versions.append(version_info) - - self.logger.debug(f"CDC Snapshot: Found {len(available_versions)} available versions") - - return available_versions - - def _extract_version_from_filename(self, filename: str, file_pattern: str) -> Optional[VersionInfo]: - """Extract version from filename using pattern (regex or curly-brace; normalized to regex). - - Version is taken from all named groups starting with version_, concatenated in name order. - Curly-brace {version} is converted to (?P.+); {fragment} to (?P.*?). - """ - regex_pattern = self._path_to_regex_pattern(file_pattern) - match = re.match(regex_pattern, filename) - - if not match: - self.logger.debug(f"CDC Snapshot: No match for filename: {filename}") - self.logger.debug(f"CDC Snapshot: Regex pattern: {regex_pattern}") - return None - - version_str = self._get_version_string_from_match(match) - - if not version_str: - self.logger.debug(f"CDC Snapshot: No version_* groups in pattern for filename: {filename}") - return None - - self.logger.debug(f"CDC Snapshot: Version string match found: {version_str}") - - try: - if self.source.versionType == CDCSnapshotVersionTypes.TIMESTAMP: - raw_value = datetime.strptime(version_str, self.source.datetimeFormat) - else: - raw_value = int(version_str) - - return VersionInfo( - raw_value=raw_value, - version_type=self.source.versionType, - datetime_format=self.source.datetimeFormat if self.source.versionType == CDCSnapshotVersionTypes.TIMESTAMP else None, - micro_second_mask_length=self.source.microSecondMaskLength \ - if self.source.versionType == CDCSnapshotVersionTypes.TIMESTAMP and self.source.microSecondMaskLength else None - ) - except (ValueError, TypeError) as e: - self.logger.error(f"CDC Snapshot: Failed to parse version '{version_str}': {e}") - raise - - def _get_next_version(self, latest_snapshot_version: Optional[Union[int, datetime]]) -> Optional[VersionInfo]: - """Get the next version to process.""" - # If no previous version exists yet - if latest_snapshot_version is None: - if not self.sorted_versions: - return None - version_info = self.sorted_versions[0] - self.logger.debug(f"CDC Snapshot: Using initial version: {version_info.formatted_value}") - return version_info - - # If a previous version exists, - # use bisect to find the first version greater than latest_snapshot_version - index = bisect.bisect_right(self.version_values, latest_snapshot_version) - if index < len(self.sorted_versions): - version_info = self.sorted_versions[index] - self.logger.debug(f"CDC Snapshot: Using next version: {version_info.formatted_value}") - return version_info - else: - self.logger.debug("CDC Snapshot: No more versions available") - return None - - def _read_snapshot_dataframe(self, version_info: VersionInfo, dataflow_config: DataFlowConfig, target_config_flags: Optional[List[str]] = None) -> Optional[DataFrame]: - """Read snapshot data into dataframe.""" - read_config = ReadConfig( - features=dataflow_config.features, - mode="batch", - target_config_flags=target_config_flags - ) - - if self.sourceType == CDCSnapshotSourceTypes.FILE: - path = self.source.path - pattern = self._path_to_regex_pattern(path) - dynamic_idx = self._get_dynamic_path_index(path) - path_parts = path.split('/') - parent_dir = '/'.join(path_parts[:dynamic_idx]) or '.' - # Anchor pattern so we only match full path (e.g. exclude customer.parquet/_SUCCESS) - file_pattern_regex = '/'.join(pattern.split('/')[dynamic_idx:]) + r'/?$' - regex_pattern = self._path_to_regex_pattern(file_pattern_regex) - - recursive_file_lookup = self.source.recursiveFileLookup - files_list = self._list_files(parent_dir, recursive=recursive_file_lookup) - - target_version = version_info.formatted_value - files_to_read: List[str] = [] - - for f in files_list: - full_path = f.path - relative_path = '/'.join(full_path.split('/')[dynamic_idx:]) - match = re.match(regex_pattern, relative_path) - if not match: - continue - version_str = self._get_version_string_from_match(match) - if version_str is None or version_str != target_version: - continue - files_to_read.append(full_path) - - if not files_to_read: - self.logger.warning(f"CDC Snapshot: No files found for version {target_version}") - return None - - self.logger.info(f"CDC Snapshot: Reading {len(files_to_read)} file(s) for version {target_version}") - schema_path = self.source.schemaPath - select_exp = self.source.selectExp - df = None - for file_path in sorted(files_to_read): - self.logger.debug(f"CDC Snapshot: Reading file: {file_path}") - file_df = SourceBatchFiles( - path=file_path, - format=self.source.format, - readerOptions=self.source.readerOptions, - schemaPath=schema_path, - selectExp=select_exp - ).read_source(read_config) - if df is None: - df = file_df - else: - df = df.union(file_df) - - # Apply filter if specified - if self.source.filter: - df = df.where(self.source.filter.replace("{version}", version_info.formatted_value)) - - elif self.sourceType == CDCSnapshotSourceTypes.TABLE: - table_parts = self.source.table.split(".") - if not 1 < len(table_parts) < 4: - raise ValueError(f"Invalid table name format: {self.source.table}. Accepted formats are: schema.table & database.schema.table") - - if len(table_parts) == 3: - database = f"{table_parts[0]}.{table_parts[1]}" - else: - # set to the specified target catalog by default - _pipeline_details = pipeline_config.get_pipeline_details() - database = f"{_pipeline_details.pipeline_catalog}.{table_parts[0]}" - - table = table_parts[-1] - - select_exp = self.source.selectExp - where_clause = [ - f"{self.source.versionColumn} = {version_info.sql_formatted_value}"] - - self.logger.info(f"CDC Snapshot: Reading table: {database}.{table} with where clause: {where_clause}") - df = SourceDelta( - database=database, - table=table, - whereClause=where_clause, - selectExp=select_exp - ).read_source(read_config) - - if self.source.deduplicateMode == DeduplicateMode.KEYS_ONLY: - df = self._deduplicate_by_keys(df) - self.logger.debug("CDC Snapshot: Applied deduplication by keys") - elif self.source.deduplicateMode == DeduplicateMode.FULL_ROW: - df = self._deduplicate_full_row(df) - self.logger.debug("CDC Snapshot: Applied full-row deduplication") - - return df diff --git a/src/config/override/.gitkeep b/src/lakeflow_framework/config/override/.gitkeep similarity index 100% rename from src/config/override/.gitkeep rename to src/lakeflow_framework/config/override/.gitkeep diff --git a/src/lakeflow_framework/dataflow/cdc_snapshot.py b/src/lakeflow_framework/dataflow/cdc_snapshot.py index 1625ad7..a0cdfe1 100644 --- a/src/lakeflow_framework/dataflow/cdc_snapshot.py +++ b/src/lakeflow_framework/dataflow/cdc_snapshot.py @@ -9,7 +9,7 @@ from pyspark.sql import functions as F import pyspark.sql.types as T -import lakeflow_framework.pipeline_config as pipeline_config +import pipeline_config from .dataflow_config import DataFlowConfig from .sources import SourceDelta, SourceBatchFiles, ReadConfig @@ -306,7 +306,7 @@ def _next_snapshot_and_version(self, latest_snapshot_version, dataflow_config: D return (df, version_info.raw_value) except Exception as e: - self.logger.error(f"CDC Snapshot: Error processing snapshots: {e}") + self.logger.error(f"CDC Snapshot: Error processing snapshots: {e}", exc_info=True) raise @@ -318,27 +318,91 @@ def _get_available_versions(self, latest_snapshot_version: Optional[Union[int, d return self._get_available_table_versions(latest_snapshot_version) else: raise ValueError(f"Unsupported source type: {self.sourceType}") + + def _list_files(self, path: str, recursive: bool = True) -> List: + """List files in a directory, attempting dbutils.fs.ls() first. - def _list_files(self, path, recursive=True): - """List files in a directory, with optional recursive file lookup. + Falls back to Spark binaryFile if dbutils is unavailable (e.g., blocked + by Serverless Restricted Access / SEG Py4JSecurityException). Args: - path: Directory path to list files from - recursive: If True, list files recursively. If False, list only files in the immediate directory. + path: Directory path to list files from. + recursive: If True, list files recursively. Returns: - List of file objects from dbutils.fs.ls() + List of objects with a .path attribute per discovered file. """ - dbutils = pipeline_config.get_dbutils() - all_files = [] + try: + dbutils = pipeline_config.get_dbutils() + all_files = [] + for f in dbutils.fs.ls(path): + all_files.append(f) + # Recursively list files in subdirectories unless the path ends with .parquet + if recursive and f.isDir() and not f.path.rstrip("/").endswith(".parquet"): + all_files.extend(self._list_files(f.path, recursive=True)) + return all_files + except Exception as e: + self.logger.warning( + f"CDC Snapshot: dbutils.fs.ls() failed at '{path}'. " + f"Falling back to Spark binaryFile for file discovery." + ) - for f in dbutils.fs().ls(path): - all_files.append(f) + # Fallback: Spark binaryFile read is SEG-compatible and needs no dbutils. + self.logger.debug("CDC Snapshot: Falling back to Spark binaryFile for file discovery") + + class _FileInfo: + __slots__ = ("path", "name", "size", "modificationTime") - if recursive and f.isDir(): - all_files.extend(self._list_files(f.path, recursive=True)) + # binaryFile returns full URIs (e.g. dbfs:/Volumes/...). Normalise to + # match the caller's path scheme so split-by-segment indices stay aligned. + _prefix = path.startswith("dbfs:") + + def __init__(self, row) -> None: + raw = row.path + if self._prefix: + self.path = raw if raw.startswith("dbfs:") else f"dbfs:{raw}" + else: + self.path = raw[5:] if raw.startswith("dbfs:") else raw + self.name = self.path.rstrip("/").rsplit("/", 1)[-1] + self.size = row.length + self.modificationTime = row.modificationTime + + spark = pipeline_config.get_spark() + rows = ( + spark.read + .format("binaryFile") + .option("recursiveFileLookup", str(recursive).lower()) + .load(path) + .select("path", "modificationTime", "length") + .collect() + ) + + # binaryFile only returns leaf files, not directories. For parquet "files" that + # are actually directories (e.g. customer.parquet/part-00000.parquet), truncate + # the path at the .parquet directory level and deduplicate so each snapshot is + # counted once. + def _truncate_at_parquet_dir(p: str) -> str: + segments = p.split("/") + for i, seg in enumerate(segments[:-1]): # skip the last segment + if seg.endswith(".parquet"): + return "/".join(segments[: i + 1]) + return p + + seen_paths = set() + files: List[_FileInfo] = [] + for row in rows: + fi = _FileInfo(row) + fi.path = _truncate_at_parquet_dir(fi.path) + fi.name = fi.path.rstrip("/").rsplit("/", 1)[-1] + if fi.path not in seen_paths: + seen_paths.add(fi.path) + files.append(fi) + + self.logger.debug( + f"CDC Snapshot: Spark binaryFile fallback listed {len(files)} file(s) under '{path}'" + ) - return all_files + return files def _path_to_regex_pattern(self, path: str) -> str: """Convert path to normalized regex pattern with named groups. @@ -648,4 +712,4 @@ def _read_snapshot_dataframe(self, version_info: VersionInfo, dataflow_config: D df = self._deduplicate_full_row(df) self.logger.debug("CDC Snapshot: Applied full-row deduplication") - return df + return df \ No newline at end of file diff --git a/src/lakeflow_framework/utility.py b/src/lakeflow_framework/utility.py index 961c4de..bfce03a 100644 --- a/src/lakeflow_framework/utility.py +++ b/src/lakeflow_framework/utility.py @@ -4,6 +4,7 @@ from functools import reduce import logging import os +import threading from typing import Callable, Dict, List import json @@ -14,7 +15,7 @@ from pyspark.sql import SparkSession from pyspark.sql.types import StructType -from lakeflow_framework.constants import ( +from constants import ( SupportedSpecFormat, PipelineBundleSuffixesJson, PipelineBundleSuffixesYaml, @@ -80,12 +81,18 @@ class JSONValidator: Attributes: schema (dict): The JSON schema loaded from a file. base_uri (str): The base URI for resolving schema references. - resolver (RefResolver): The JSON schema resolver. - validator (Draft7Validator): The JSON schema validator. Methods: validate(json_data: Dict) -> List: Validates the provided JSON data against the loaded schema and returns a list of validation errors. + + Thread safety: + A ``Draft7Validator`` and ``RefResolver`` are created once per OS thread (via + ``threading.local``) and reused on that thread. Different threads never share a + resolver, so ``RefResolver`` internal state is not mutated concurrently. The root + schema dict is loaded once in ``__init__``; referenced schema files (e.g. + ``definitions_main.json``) are read from disk at most once per thread and then + cached on that resolver's ``_remote_cache``. """ def __init__(self, schema_path: str): @@ -95,14 +102,21 @@ def __init__(self, schema_path: str): except Exception as e: raise ValueError(f"JSON Schema not found: {schema_path}") from e - # Resolve references self.base_uri = "file://" + os.path.abspath(os.path.dirname(schema_path)) + "/" - self.resolver = js.RefResolver(base_uri=self.base_uri, referrer=self.schema) - self.validator = js.Draft7Validator(self.schema, resolver=self.resolver) + self._thread_local = threading.local() + + def _thread_validator(self) -> js.Draft7Validator: + """Return a validator for the current thread, creating it on first use.""" + validator = getattr(self._thread_local, "validator", None) + if validator is None: + resolver = js.RefResolver(base_uri=self.base_uri, referrer=self.schema) + validator = js.Draft7Validator(self.schema, resolver=resolver) + self._thread_local.validator = validator + return validator def validate(self, json_data: Dict) -> List: """Validate the provided JSON data against the loaded schema and returns a list of validation errors.""" - return list(self.validator.iter_errors(json_data)) + return list(self._thread_validator().iter_errors(json_data)) def add_struct_field(struct: StructType, column: Dict): @@ -533,3 +547,4 @@ def deep_merge(base: Dict, overlay: Dict) -> Dict: else: result[key] = val return result + \ No newline at end of file From 478416aa124bc99f717e8f08886a8e49a015a266 Mon Sep 17 00:00:00 2001 From: rederik76 <127075662+rederik76@users.noreply.github.com> Date: Fri, 10 Jul 2026 14:16:15 +1000 Subject: [PATCH 4/9] Update docs and READMEs for lakeflow_framework package paths. Align operational metadata, templates, logging, config, validator, and test docs with the packaged layout; extend the spelling wordlist; rephrase historized to history-tracked in samples README. --- docs/source/contributor_dev_steps.rst | 2 +- docs/source/feature_framework_configuration.rst | 2 +- docs/source/feature_logging.rst | 8 ++++---- docs/source/feature_operational_metadata.rst | 2 +- docs/source/feature_templates.rst | 2 +- docs/source/spelling_wordlist.txt | 7 +++++++ samples/README.md | 6 +++--- scripts/README.md | 4 ++-- src/local/config/README.md | 6 +++--- tests/README.md | 6 +++--- 10 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/source/contributor_dev_steps.rst b/docs/source/contributor_dev_steps.rst index c1c7f02..356e33a 100644 --- a/docs/source/contributor_dev_steps.rst +++ b/docs/source/contributor_dev_steps.rst @@ -43,7 +43,7 @@ Development Process pytest tests/ -m "not integration and not spark" - See ``tests/README.md`` for layout, fixtures, markers, and conventions. - - Optional coverage: add ``--cov=src --cov-report=term-missing``. + - Optional coverage: add ``--cov=lakeflow_framework --cov-report=term-missing``. - CI runs the same pytest command on every pull request (``.github/workflows/ci.yml``). 3. Integration Testing / Samples diff --git a/docs/source/feature_framework_configuration.rst b/docs/source/feature_framework_configuration.rst index 5da476b..efb34e4 100644 --- a/docs/source/feature_framework_configuration.rst +++ b/docs/source/feature_framework_configuration.rst @@ -126,7 +126,7 @@ Example — change one global setting without touching the rest of ``global.json } Save this as ``src/local/config/global.json``. All other keys from -``src/config/default/global.json`` are kept unchanged. +``src/lakeflow_framework/config/default/global.json`` are kept unchanged. For **directory-based config** (e.g. ``dataflow_spec_mapping/``), place the entire override directory in ``src/local/config/`` — the local directory takes diff --git a/docs/source/feature_logging.rst b/docs/source/feature_logging.rst index 63202a7..1ecc2bf 100644 --- a/docs/source/feature_logging.rst +++ b/docs/source/feature_logging.rst @@ -18,7 +18,7 @@ You can optionally plug in a **custom logger** via a dedicated ``logger.json`` c .. admonition:: Setting Precedence :class: warning - * Framework code and extensions should use the shared logger helper ``pipeline_config.get_logger()`` rather than creating their own loggers. + * Framework code and extensions should use the shared logger helper ``get_logger()`` from ``lakeflow_framework.pipeline_config`` rather than creating their own loggers. Overview -------- @@ -460,14 +460,14 @@ Using the Logger in Code .. code-block:: python - from pipeline_config import get_logger + from lakeflow_framework.pipeline_config import get_logger logger = get_logger() logger.info("Processing batch %s", batch_id) -``utility.set_logger`` remains for backward compatibility and delegates to the framework default logger factory; new code should use ``pipeline_config.get_logger()``. +``lakeflow_framework.utility.set_logger`` remains for backward compatibility and delegates to the framework default logger factory; new code should use ``get_logger()`` from ``lakeflow_framework.pipeline_config``. -Custom loggers that support async flushing may require an explicit ``close()`` at the end of the pipeline. Consider a **post_init** hook under ``extensions/post_init/`` that calls ``close()`` on the primary logger when exposed. +Custom loggers that support async flushing may require an explicit ``close()`` at the end of the pipeline. Consider a **post-init** hook under ``src/init/post/`` (pipeline bundle) or ``src/local/init/post/`` (framework bundle) that calls ``close()`` on the primary logger when exposed. Troubleshooting ^^^^^^^^^^^^^^^^ diff --git a/docs/source/feature_operational_metadata.rst b/docs/source/feature_operational_metadata.rst index 5a2ca6a..ce1c117 100644 --- a/docs/source/feature_operational_metadata.rst +++ b/docs/source/feature_operational_metadata.rst @@ -29,7 +29,7 @@ Configuration ------------- | **Scope: Global** -| In the Framework bundle, operational metadata columns are defined in JSON configuration files at Lakehouse layer level (e.g. bronze, silver, gold). The configuration files are locate at and must be named as follows: ``src/config/default/operational_metadata_.json`` +| In the Framework bundle, operational metadata columns are defined in JSON configuration files at Lakehouse layer level (e.g. bronze, silver, gold). The configuration files are locate at and must be named as follows: ``src/lakeflow_framework/config/default/operational_metadata_.json`` .. admonition:: Layer Config :class: note diff --git a/docs/source/feature_templates.rst b/docs/source/feature_templates.rst index 4eac55b..4d11e8f 100644 --- a/docs/source/feature_templates.rst +++ b/docs/source/feature_templates.rst @@ -421,7 +421,7 @@ Validation Each expanded spec is validated using the existing schema validators to ensure correctness. -Template usage specs are validated against the schema at ``src/schemas/spec_template.json``: +Template usage specs are validated against the schema at ``src/lakeflow_framework/schemas/spec_template.json``: - ``template``: Required string (template name without .json extension) - ``params``: Required array with at least one parameter object diff --git a/docs/source/spelling_wordlist.txt b/docs/source/spelling_wordlist.txt index e15f068..8e05c6a 100644 --- a/docs/source/spelling_wordlist.txt +++ b/docs/source/spelling_wordlist.txt @@ -4,6 +4,7 @@ Artifactory async autocomplete backfill +backend behaviour cardinality cdc @@ -14,8 +15,10 @@ cli Cmd config configs +contrib css Ctrl +customisation customisations dataFlowId Dataframe @@ -27,6 +30,7 @@ deduplicate dedupes deduplication deployable +deps Del Demultiplexing dependant @@ -47,6 +51,7 @@ initialisation Inmon IntelliSense instantiation +integrations json keystore kwargs @@ -82,6 +87,7 @@ rst Schemas schemas sdist +semver Serverless sinkOptions sourceDetails @@ -95,6 +101,7 @@ stdout Subquery subdirectory subqueries +subpackage tableMigrationDetails tableProperties targetFormat diff --git a/samples/README.md b/samples/README.md index b4782df..63d18dc 100644 --- a/samples/README.md +++ b/samples/README.md @@ -7,7 +7,7 @@ The Framework comes with extensive samples that demonstrate the use of the frame | **`feature-samples`** | Demonstrates every framework feature in isolation using a single `{namespace}_feature` schema. The simplest entry point. | | **`pattern-samples`** | End-to-end medallion architecture patterns (bronze → silver → gold) across multiple schemas. Includes multi-source streaming, stream-static joins, CDC from snapshot sources, and gold-layer materialized views. | | **`yaml_sample`** | Demonstrates that data flow specs can be written in YAML format instead of JSON. Contains YAML equivalents of basic specs. | -| **`tpch_sample`** | The most comprehensive end-to-end reference, built on the TPC-H dataset in the UC samples catalog. Covers multi-source schema-on-read bronze, conformed/historized silver (SCD2 + SCD1 + append-only facts with DQ quarantine), and a governed gold star schema with surrogate keys, point-in-time joins, pre-aggregated MVs, and UC metric views. Uses template specs and a three-run incremental simulation. See its [README](tpch_sample/README.md). | +| **`tpch_sample`** | The most comprehensive end-to-end reference, built on the TPC-H dataset in the UC samples catalog. Covers multi-source schema-on-read bronze, conformed/history-tracked silver (SCD2 + SCD1 + append-only facts with DQ quarantine), and a governed gold star schema with surrogate keys, point-in-time joins, pre-aggregated MVs, and UC metric views. Uses template specs and a three-run incremental simulation. See its [README](tpch_sample/README.md). | ## Deploying the Samples @@ -146,12 +146,12 @@ Parameters: ## TPC-H Sample -The TPC-H sample is the most comprehensive end-to-end example in the framework. Built on the TPC-H schema in the Databricks UC samples catalog, it turns a realistic, sample dataset into a fully streaming medallion data warehouse — from multi-source raw ingestion through conformed, historized silver to a governed gold star schema with value-add aggregations. +The TPC-H sample is the most comprehensive end-to-end example in the framework. Built on the TPC-H schema in the Databricks UC samples catalog, it turns a realistic, sample dataset into a fully streaming medallion data warehouse — from multi-source raw ingestion through conformed, history-tracked silver to a governed gold star schema with value-add aggregations. It demonstrates a wide range of framework capabilities together, including: * **Schema-on-read bronze** that infers and evolves the schema, ingesting Parquet from eight simulated source systems via Auto Loader. -* **Conformed, historized silver** with SCD2 dimensions, SCD1 reference data, append-only facts, and data-quality expectations with quarantine. +* **Conformed, history-tracked silver** with SCD2 dimensions, SCD1 reference data, append-only facts, and data-quality expectations with quarantine. * **Governed gold star schema** with surrogate keys and point-in-time (as-of) joins, plus two metrics approaches side by side: pre-aggregated materialized views and UC metric views. * **Template specs** that collapse repetitive bronze and silver flows into reusable templates. * **A three-run incremental simulation** covering SCD changes, fact growth, a backdated out-of-order correction, and ongoing quarantine. diff --git a/scripts/README.md b/scripts/README.md index 7063777..af4acd1 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -27,8 +27,8 @@ python scripts/validate_dataflows.py -v ``` **What it validates:** -- Searches for all `*_main.json` files in `dataflows/**/dataflowspec/` directories -- Validates against `src/schemas/main.json` (which routes to appropriate sub-schemas) +- Searches for all `*_main.json` files under `dataflows/` (including `dataflowspec/` subdirectories and flat `dataflows/*_main.json` layouts) +- Validates against `src/lakeflow_framework/schemas/main.json` (which routes to appropriate sub-schemas) - Reports validation errors with clear messages **Version Mapping (enabled by default):** diff --git a/src/local/config/README.md b/src/local/config/README.md index 3d24aa1..dcda333 100644 --- a/src/local/config/README.md +++ b/src/local/config/README.md @@ -4,7 +4,7 @@ Fork-safe sparse override directory for framework configuration files. ## How it works -Files placed here are **deep-merged on top of their `src/config/default/` +Files placed here are **deep-merged on top of their `src/lakeflow_framework/config/default/` counterparts** at runtime. You only need to include the keys you want to change — all other keys retain their default values. @@ -25,7 +25,7 @@ key improvements: ## Supported files -Any file present in `src/config/default/` can be partially overridden here +Any file present in `src/lakeflow_framework/config/default/` can be partially overridden here using the same filename. Common candidates: | File | Purpose | @@ -47,7 +47,7 @@ default. To migrate: 1. Identify which keys in your `config/override/` files differ from - `config/default/`. + `lakeflow_framework/config/default/`. 2. Create sparse files here containing only those differing keys. 3. Delete or empty `config/override/`. diff --git a/tests/README.md b/tests/README.md index b76d850..d86d2d6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -14,7 +14,7 @@ pytest tests/ -m "not integration and not spark" pytest tests/ -m integration # Optional coverage report -pytest tests/ -m "not integration and not spark" --cov=src --cov-report=term-missing +pytest tests/ -m "not integration and not spark" --cov=lakeflow_framework --cov-report=term-missing ``` Use **Python 3.12** for development; newer Python versions may break PySpark-related dependencies. @@ -32,7 +32,7 @@ tests/ ├── unit/ # Fast, isolated module tests └── integration/ # Sample-dependent tests (require samples/ checkout) ├── conftest.py # validate_bundle(), bundle paths, feature-samples fixtures - ├── test_validate_dataflows.py # Script helpers + all specs per JSON bundle + ├── test_validate_dataflows.py # Per-bundle sample validation + CLI subprocess └── test_feature_samples_spec_builder.py # DataflowSpecBuilder on feature-samples ``` @@ -50,7 +50,7 @@ Configure in `pytest.ini`. CI runs: `-m "not integration and not spark"`. 1. **Use `pipeline_context`** — do not call `initialize_core()` inline in test modules. 2. **Prefer `tests/fixtures/`** — unit tests should not depend on `samples/` paths. -3. **Mirror `src/`** — `tests/unit/test_.py` maps to `src/.py`. +3. **Mirror the package** — `tests/unit/test_.py` maps to `src/lakeflow_framework/.py`; use `lakeflow_framework.*` imports (not compat shims). 4. **One concern per test** — name tests `test__when_`. 5. **No DLT mocks** — tests that need `@dp.table` belong in samples/E2E, not unit tests. From d08301d4d5730ababda9775ba95f11f839b7e96c Mon Sep 17 00:00:00 2001 From: rederik76 <127075662+rederik76@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:52:58 +1000 Subject: [PATCH 5/9] Document package refactor: ADR renumbering, import conventions, and README updates. Renumber package-refactor ADRs to 0009/0010 to avoid conflicting with the existing scripted-docs ADR-0007. Add contributor import conventions guide, expand tests README for the layered suite, and align root README with a deploy-first quick start, src/ package layout, and a separate contributor section. --- README.md | 63 ++++++++--- ....md => 0009-lakeflow-framework-package.md} | 6 +- ...=> 0010-strategy-b-disk-first-resolver.md} | 4 +- docs/source/contributor.rst | 1 + docs/source/contributor_dev_steps.rst | 2 + docs/source/contributor_imports.rst | 107 ++++++++++++++++++ docs/source/spelling_wordlist.txt | 3 + tests/README.md | 42 +++++-- 8 files changed, 197 insertions(+), 31 deletions(-) rename docs/decisions/{0007-lakeflow-framework-package.md => 0009-lakeflow-framework-package.md} (96%) rename docs/decisions/{0008-strategy-b-disk-first-resolver.md => 0010-strategy-b-disk-first-resolver.md} (98%) create mode 100644 docs/source/contributor_imports.rst diff --git a/README.md b/README.md index 78da480..40672ee 100644 --- a/README.md +++ b/README.md @@ -23,33 +23,41 @@ The framework supports centralized and domain-oriented operating models, and acc - Support for batch and streaming pipelines across Bronze/Silver/Gold, aligned to your chosen modelling pattern - Flexible for centralized and domain-oriented operating models +## Prerequisites + +- Access to a Databricks workspace +- Databricks CLI installed and authenticated (`databricks auth login` for your workspace, or a configured CLI profile) +- Familiarity with Databricks Lakeflow Spark Declarative Pipelines concepts + ## Quick start +Deploy the framework: + ```bash git clone https://github.com/databricks-solutions/lakeflow_framework.git cd lakeflow_framework -pip install -r requirements-dev.txt +databricks bundle deploy -t dev ``` -Then: - -1. Open the hosted docs: https://databricks-solutions.github.io/lakeflow_framework/ -2. Deploy the framework using the `Deploy Framework` guide -3. Deploy samples from `samples/` using the documentation walkthroughs -4. Build your first pipeline bundle using the `Build a Pipeline Bundle` guide +Deploy samples (requires the framework above): see [samples/README.md](samples/README.md) for bundle descriptions and deploy scripts (`deploy.sh`, `deploy_feature_samples.sh`, `deploy_tpch.sh`, and others). -## Prerequisites +```bash +cd samples +./deploy.sh +``` -- Access to a Databricks workspace -- Databricks CLI installed and configured -- Python environment with project dependencies installed -- Familiarity with Databricks Lakeflow Spark Declarative Pipelines concepts +Full deployment steps, configuration options, and walkthroughs are in the public docs [Getting Started](https://databricks-solutions.github.io/lakeflow_framework/current/getting_started.html) section. ## Repository structure -- `docs/` - Sphinx documentation and versioned docs build tooling -- `samples/` - example framework and pipeline bundles -- `src/` - framework source code and runtime components +- `docs/` — Sphinx documentation and versioned docs build tooling +- `samples/` — example framework and pipeline bundles +- `src/` — framework bundle root deployed to the workspace (`framework.sourcePath` in DAB) + - `lakeflow_framework/` — canonical Python package: runtime code, bundled default config (`config/default/`), and JSON schemas (`schemas/`). Prefer `from lakeflow_framework...` imports in new code. + - `*.py` at `src/` root — backward-compatibility shims for legacy bare imports (e.g. `from constants import ...`); removed at v1.0.0 + - `local/` — customer-owned sparse config and extensions; never overwritten by upstream upgrades ([src/local/README.md](src/local/README.md)) + +See [Import conventions](https://databricks-solutions.github.io/lakeflow_framework/contributor_imports.html) ## Version compatibility @@ -69,10 +77,33 @@ The framework is actively maintained. Databricks support does not cover this rep Please refer to the [documentation](https://databricks-solutions.github.io/lakeflow_framework/) for further details and an explanation of the samples. The documentation needs to be deployed as HTML or Markdown within your org before it can be used. +### Local development quick start (contributors) + +Clone the repository, then set up a Python 3.12 environment. This is a minimal local quick start — full contributor guidance (VS Code setup, lockfiles, deployment, and PR workflow) is in the documentation under **Framework Development & Contributors**. + +```bash +git clone https://github.com/databricks-solutions/lakeflow_framework.git +cd lakeflow_framework +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install --require-hashes --no-deps -r requirements-dev.lock +``` + +For editable installs and IDE auto-complete: `pip install -e ".[contrib]"` (see [Development Environment Setup](https://databricks-solutions.github.io/lakeflow_framework/contributor_dev_env.html)). + +Run unit tests: + +```bash +pytest tests/ -m "not integration and not spark" +``` + +See also `tests/README.md` and `docs/source/contributor_imports.rst` in the repository. + ### Local docs development (optional) +Requires dev dependencies from `requirements-dev.lock`: + ```bash -pip install -r requirements-docs.txt make -C docs html ``` diff --git a/docs/decisions/0007-lakeflow-framework-package.md b/docs/decisions/0009-lakeflow-framework-package.md similarity index 96% rename from docs/decisions/0007-lakeflow-framework-package.md rename to docs/decisions/0009-lakeflow-framework-package.md index a61bf4a..12d4715 100644 --- a/docs/decisions/0007-lakeflow-framework-package.md +++ b/docs/decisions/0009-lakeflow-framework-package.md @@ -1,4 +1,4 @@ -# ADR-0007: `lakeflow_framework` pip-installable package +# ADR-0009: `lakeflow_framework` pip-installable package **Date:** 2026-05-23 **Status:** Accepted @@ -107,7 +107,7 @@ deploy) converge on it. `src/lakeflow_framework/contrib/` is introduced as a scaffold with an empty `__init__.py` and a support-policy `README.rst`. No modules land in this PR. -See `docs/decisions/0007-*` (this file) for the publish model and +See `docs/decisions/0009-*` (this file) for the publish model and `docs/source/contributor_contrib.rst` for the contributor guide. ### Optional extras @@ -137,7 +137,7 @@ module is added. risk with customer code. - Existing flat-deploy customers are unaffected: the `src/` shims preserve backward compatibility, and `framework.sourcePath` + disk-first resolution - (ADR-0008) means behaviour is identical to pre-v0.16.0. + (ADR-0010) means behaviour is identical to pre-v0.16.0. - Editable installs (`pip install -e ".[contrib]"`) are supported for local development; the `src/` layout ensures the installed package and the source tree are the same directory. diff --git a/docs/decisions/0008-strategy-b-disk-first-resolver.md b/docs/decisions/0010-strategy-b-disk-first-resolver.md similarity index 98% rename from docs/decisions/0008-strategy-b-disk-first-resolver.md rename to docs/decisions/0010-strategy-b-disk-first-resolver.md index ba9aa46..af9f04e 100644 --- a/docs/decisions/0008-strategy-b-disk-first-resolver.md +++ b/docs/decisions/0010-strategy-b-disk-first-resolver.md @@ -1,4 +1,4 @@ -# ADR-0008: Strategy B — Workspace Files-first config and schema resolution +# ADR-0010: Strategy B — Workspace Files-first config and schema resolution **Date:** 2026-05-23 **Status:** Accepted @@ -8,7 +8,7 @@ ## Context -With the introduction of the `lakeflow_framework` Python package (ADR-0007), +With the introduction of the `lakeflow_framework` Python package (ADR-0009), default config files (`config/default/`) and JSON schemas (`schemas/`) are bundled inside the wheel via `importlib.resources`. Previously they lived only in Workspace Files at `{framework_path}/lakeflow_framework/config/default/`. diff --git a/docs/source/contributor.rst b/docs/source/contributor.rst index 885c173..ae05233 100644 --- a/docs/source/contributor.rst +++ b/docs/source/contributor.rst @@ -6,6 +6,7 @@ Framework Development & Contributors :caption: Contents: contributor_dev_env + contributor_imports contributor_dev_git contributor_dev_steps contributor_dev_docs diff --git a/docs/source/contributor_dev_steps.rst b/docs/source/contributor_dev_steps.rst index 356e33a..1048d96 100644 --- a/docs/source/contributor_dev_steps.rst +++ b/docs/source/contributor_dev_steps.rst @@ -25,6 +25,8 @@ Development Process ------------------- 1. Local Development + - Follow :doc:`contributor_imports` for ``lakeflow_framework.*`` vs compat + shim import rules. - Follow coding standards and style guides - Ensure the yapf extension is installed and enabled in VS Code (refer to step 2 of :doc:`contributor_dev_env`) - Use yapf to format your python code (right click and select 'Format Document With' then select yapf) diff --git a/docs/source/contributor_imports.rst b/docs/source/contributor_imports.rst new file mode 100644 index 0000000..72b6efd --- /dev/null +++ b/docs/source/contributor_imports.rst @@ -0,0 +1,107 @@ +Import conventions +################## + +Framework code, tests, and scripts in this repository use the +``lakeflow_framework`` Python package namespace. Flat ``src/`` import paths +exist only as backward-compatibility shims for existing pipeline bundles. + +Canonical imports +----------------- + +Use absolute imports from the package: + +.. code-block:: python + + from lakeflow_framework.constants import FrameworkPaths + from lakeflow_framework.bundle_loader import BundleLoader + from lakeflow_framework.dataflow import Dataflow + +Implementation lives under ``src/lakeflow_framework/``. That tree holds the +core modules, ``dataflow/``, ``dataflow_spec_builder/``, bundled default config +(``config/default/``), JSON schemas (``schemas/``), and the ``contrib/`` +subpackage. + +Compat shims (do not use in new code) +-------------------------------------- + +The old flat-deploy layout placed modules directly under ``src/`` (for example +``from constants import FrameworkPaths``). Since v0.16.0, matching +``src/*.py`` files are thin re-export shims: + +.. code-block:: python + + # src/constants.py — compat shim, remove at v1.0.0 + from lakeflow_framework.constants import FrameworkPaths # noqa: F401 + +Shims cover the top-level modules that used to live as bare ``src/*.py`` files +(``constants``, ``bundle_loader``, ``logger``, and similar). They do **not** +duplicate ``dataflow/`` or ``dataflow_spec_builder/`` — those subpackages exist +only under ``src/lakeflow_framework/``. + +Existing customer pipeline notebooks and bundles may keep bare imports until +v1.0.0. **Contributors must not add new bare imports** in framework source, +tests, or repository scripts. + +Rules for contributors +---------------------- + +.. list-table:: + :header-rows: 1 + :widths: 28 72 + + * - Context + - Import rule + * - Code under ``src/lakeflow_framework/`` + - Always ``from lakeflow_framework. import ...``. No bare imports + and no relative imports across package boundaries. + * - Unit and integration tests (``tests/``) + - Always ``lakeflow_framework.*``. Tests exercise the installed package, + not the shim layer. + * - Repository scripts (``scripts/``) + - Prefer ``lakeflow_framework.*`` where the script imports framework + modules. + * - Sample pipeline bundles (``samples/``) + - May use either style during the shim deprecation window; prefer + ``lakeflow_framework.*`` in new sample code. + * - ``src/local/`` + - Customer-owned overlay — not part of the published package. See + ``src/local/README.md``. + +Local development +----------------- + +For IDE resolution and ``pytest``, install the package in editable mode (see +:doc:`contributor_dev_env`): + +.. code-block:: bash + + pip install -e ".[contrib]" + +This puts ``lakeflow_framework`` on ``sys.path`` from ``src/lakeflow_framework/`` +via ``pyproject.toml``. You do not need to add ``src/`` manually when running +tests locally after an editable install. + +Deprecation timeline +-------------------- + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Version + - Change + * - v0.16.0 + - ``lakeflow_framework`` package introduced; bare ``src/`` imports still + work via shims. + * - v1.0.0 + - Compat shims at ``src/*.py`` removed. + +Further reading +--------------- + +- ``docs/decisions/0009-lakeflow-framework-package.md`` — package layout, + wheel distribution, and shim policy (ADR-0009). +- ``docs/decisions/0010-strategy-b-disk-first-resolver.md`` — how bundled + config and schemas resolve against workspace files (ADR-0010). +- :doc:`deploy_wheel` — wheel install and shim guidance for deployers. +- :doc:`contributor_contrib` — import rules specific to ``contrib/`` modules. diff --git a/docs/source/spelling_wordlist.txt b/docs/source/spelling_wordlist.txt index 8e05c6a..479fbc6 100644 --- a/docs/source/spelling_wordlist.txt +++ b/docs/source/spelling_wordlist.txt @@ -12,6 +12,7 @@ cdcSettings cdcSnapshotSettings changelog cli +compat Cmd config configs @@ -31,6 +32,7 @@ dedupes deduplication deployable deps +deployers Del Demultiplexing dependant @@ -102,6 +104,7 @@ Subquery subdirectory subqueries subpackage +subpackages tableMigrationDetails tableProperties targetFormat diff --git a/tests/README.md b/tests/README.md index d86d2d6..b8aa719 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,7 +4,7 @@ Fast unit tests and slower integration checks for the Lakeflow Framework. ## Quick start -From the repository root (after `pip install --require-hashes --no-deps -r requirements-dev.lock`): +From the repository root (after `pip install --require-hashes --no-deps -r requirements-dev.lock` or `pip install -e ".[contrib]"`): ```bash # Unit tests (default local/CI command) @@ -19,23 +19,44 @@ pytest tests/ -m "not integration and not spark" --cov=lakeflow_framework --cov- Use **Python 3.12** for development; newer Python versions may break PySpark-related dependencies. +## Test architecture + +The suite is layered so local runs and CI stay fast by default: + +| Layer | Location | Marker | Depends on | +|-------|----------|--------|------------| +| **Unit** | `tests/unit/` | (none) | `tests/fixtures/` only; mocks via `pipeline_context` | +| **Integration** | `tests/integration/` | `integration` | Full `samples/` checkout | + +**Import policy:** all tests use `lakeflow_framework.*` imports. The compat shim layer at `src/*.py` is not exercised — see `docs/source/contributor_imports.rst`. + +**Post-merge layout (v0.16+):** root-level `tests/test_bundle_loader.py` and `tests/test_validate_dataflows.py` were removed. Their coverage now lives under `tests/unit/` (isolated module and script-helper tests) and `tests/integration/` (sample-bundle validation and CLI subprocess tests). + ## Layout ``` tests/ -├── conftest.py # Shared fixtures (pipeline_context, bundle trees) -├── helpers.py # make_tree() for temp directory layouts -├── fixtures/ # Minimal JSON/YAML — not full samples/ +├── conftest.py # Shared fixtures (pipeline_context, bundle trees) +├── helpers.py # make_tree() for temp directory layouts +├── fixtures/ # Minimal JSON/YAML — not full samples/ │ ├── specs/ │ ├── bundles/ │ └── golden/ -├── unit/ # Fast, isolated module tests -└── integration/ # Sample-dependent tests (require samples/ checkout) - ├── conftest.py # validate_bundle(), bundle paths, feature-samples fixtures - ├── test_validate_dataflows.py # Per-bundle sample validation + CLI subprocess - └── test_feature_samples_spec_builder.py # DataflowSpecBuilder on feature-samples +├── unit/ # Fast, isolated module tests +│ └── dataflow/ # mirrors src/lakeflow_framework/dataflow/ +└── integration/ # Sample-dependent tests (require samples/ checkout) ``` +### Key fixtures (`conftest.py`) + +| Fixture | Purpose | +|---------|---------| +| `pipeline_context` | Bootstrap `pipeline_config` singletons with mocks; always use instead of inline `initialize_core()` | +| `framework_package_path` | Path to `src/lakeflow_framework/` (canonical package tree) | +| `framework_src_path` | Path to `src/` (`framework.sourcePath` in flat deploy) | +| `minimal_framework_tree` | Temp tree with default config copied from the package | +| `minimal_bundle_tree` / `dataflow_bundle_tree` / `template_bundle_tree` | Minimal pipeline bundle layouts under `tmp_path` | + ## Markers | Marker | Purpose | @@ -50,9 +71,10 @@ Configure in `pytest.ini`. CI runs: `-m "not integration and not spark"`. 1. **Use `pipeline_context`** — do not call `initialize_core()` inline in test modules. 2. **Prefer `tests/fixtures/`** — unit tests should not depend on `samples/` paths. -3. **Mirror the package** — `tests/unit/test_.py` maps to `src/lakeflow_framework/.py`; use `lakeflow_framework.*` imports (not compat shims). +3. **Mirror the package** — `tests/unit/test_.py` maps to `src/lakeflow_framework/.py`; `tests/unit/dataflow/` mirrors `src/lakeflow_framework/dataflow/`. Use `lakeflow_framework.*` imports (not compat shims). See `docs/source/contributor_imports.rst` for the full policy. 4. **One concern per test** — name tests `test__when_`. 5. **No DLT mocks** — tests that need `@dp.table` belong in samples/E2E, not unit tests. +6. **Script helpers vs integration** — pure functions from `scripts/validate_dataflows.py` belong in `test_validate_dataflows_helpers.py`; subprocess runs against `samples/` belong in `tests/integration/`. ## CI From 3bb3d5d4178a3d3750b1723562d9b61d101554c6 Mon Sep 17 00:00:00 2001 From: rederik76 <127075662+rederik76@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:36:24 +1000 Subject: [PATCH 6/9] Fix ADR numbering: package refactor is 0008, Strategy B is 0009. Renumber from 0009/0010 to fill the gap after ADR-0007 and update cross-references in the ADRs and contributor import conventions doc. --- ...work-package.md => 0008-lakeflow-framework-package.md} | 6 +++--- ...resolver.md => 0009-strategy-b-disk-first-resolver.md} | 4 ++-- docs/source/contributor_imports.rst | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) rename docs/decisions/{0009-lakeflow-framework-package.md => 0008-lakeflow-framework-package.md} (96%) rename docs/decisions/{0010-strategy-b-disk-first-resolver.md => 0009-strategy-b-disk-first-resolver.md} (98%) diff --git a/docs/decisions/0009-lakeflow-framework-package.md b/docs/decisions/0008-lakeflow-framework-package.md similarity index 96% rename from docs/decisions/0009-lakeflow-framework-package.md rename to docs/decisions/0008-lakeflow-framework-package.md index 12d4715..f5cff94 100644 --- a/docs/decisions/0009-lakeflow-framework-package.md +++ b/docs/decisions/0008-lakeflow-framework-package.md @@ -1,4 +1,4 @@ -# ADR-0009: `lakeflow_framework` pip-installable package +# ADR-0008: `lakeflow_framework` pip-installable package **Date:** 2026-05-23 **Status:** Accepted @@ -107,7 +107,7 @@ deploy) converge on it. `src/lakeflow_framework/contrib/` is introduced as a scaffold with an empty `__init__.py` and a support-policy `README.rst`. No modules land in this PR. -See `docs/decisions/0009-*` (this file) for the publish model and +See `docs/decisions/0008-*` (this file) for the publish model and `docs/source/contributor_contrib.rst` for the contributor guide. ### Optional extras @@ -137,7 +137,7 @@ module is added. risk with customer code. - Existing flat-deploy customers are unaffected: the `src/` shims preserve backward compatibility, and `framework.sourcePath` + disk-first resolution - (ADR-0010) means behaviour is identical to pre-v0.16.0. + (ADR-0009) means behaviour is identical to pre-v0.16.0. - Editable installs (`pip install -e ".[contrib]"`) are supported for local development; the `src/` layout ensures the installed package and the source tree are the same directory. diff --git a/docs/decisions/0010-strategy-b-disk-first-resolver.md b/docs/decisions/0009-strategy-b-disk-first-resolver.md similarity index 98% rename from docs/decisions/0010-strategy-b-disk-first-resolver.md rename to docs/decisions/0009-strategy-b-disk-first-resolver.md index af9f04e..99d16dd 100644 --- a/docs/decisions/0010-strategy-b-disk-first-resolver.md +++ b/docs/decisions/0009-strategy-b-disk-first-resolver.md @@ -1,4 +1,4 @@ -# ADR-0010: Strategy B — Workspace Files-first config and schema resolution +# ADR-0009: Strategy B — Workspace Files-first config and schema resolution **Date:** 2026-05-23 **Status:** Accepted @@ -8,7 +8,7 @@ ## Context -With the introduction of the `lakeflow_framework` Python package (ADR-0009), +With the introduction of the `lakeflow_framework` Python package (ADR-0008), default config files (`config/default/`) and JSON schemas (`schemas/`) are bundled inside the wheel via `importlib.resources`. Previously they lived only in Workspace Files at `{framework_path}/lakeflow_framework/config/default/`. diff --git a/docs/source/contributor_imports.rst b/docs/source/contributor_imports.rst index 72b6efd..6382217 100644 --- a/docs/source/contributor_imports.rst +++ b/docs/source/contributor_imports.rst @@ -99,9 +99,9 @@ Deprecation timeline Further reading --------------- -- ``docs/decisions/0009-lakeflow-framework-package.md`` — package layout, - wheel distribution, and shim policy (ADR-0009). -- ``docs/decisions/0010-strategy-b-disk-first-resolver.md`` — how bundled - config and schemas resolve against workspace files (ADR-0010). +- ``docs/decisions/0008-lakeflow-framework-package.md`` — package layout, + wheel distribution, and shim policy (ADR-0008). +- ``docs/decisions/0009-strategy-b-disk-first-resolver.md`` — how bundled + config and schemas resolve against workspace files (ADR-0009). - :doc:`deploy_wheel` — wheel install and shim guidance for deployers. - :doc:`contributor_contrib` — import rules specific to ``contrib/`` modules. From 1082bf6c4012b37e50d1a0205172bd86a981136d Mon Sep 17 00:00:00 2001 From: rederik76 <127075662+rederik76@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:57:46 +1000 Subject: [PATCH 7/9] bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6d3d55f..8c20c52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.19.0 \ No newline at end of file +v0.20.0 \ No newline at end of file From f6e1ba7056f4b28766e5f5b013219e2c00cd03f0 Mon Sep 17 00:00:00 2001 From: rederik76 <127075662+rederik76@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:08:23 +1000 Subject: [PATCH 8/9] Fix docs wranings. --- docs/source/contributor_contrib.rst | 2 +- samples/README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/contributor_contrib.rst b/docs/source/contributor_contrib.rst index 7461628..c381123 100644 --- a/docs/source/contributor_contrib.rst +++ b/docs/source/contributor_contrib.rst @@ -43,7 +43,7 @@ Understanding both modes is important when writing or using a contrib module. because ``src/`` is. However, any *external* dependencies declared by the contrib module still need to be installed separately — they are not bundled with the flat deploy. Install them as cluster libraries or - via ``src/local/libraries/`` (see :doc:`/source/feature_python_extensions`). + via ``src/local/libraries/`` (see :doc:`feature_python_extensions`). * - **Wheel + local overlay** - Wheel installed, plus ``framework.sourcePath`` set for ``src/local/config/`` sparse overrides. diff --git a/samples/README.md b/samples/README.md index 63d18dc..e44d878 100644 --- a/samples/README.md +++ b/samples/README.md @@ -7,7 +7,7 @@ The Framework comes with extensive samples that demonstrate the use of the frame | **`feature-samples`** | Demonstrates every framework feature in isolation using a single `{namespace}_feature` schema. The simplest entry point. | | **`pattern-samples`** | End-to-end medallion architecture patterns (bronze → silver → gold) across multiple schemas. Includes multi-source streaming, stream-static joins, CDC from snapshot sources, and gold-layer materialized views. | | **`yaml_sample`** | Demonstrates that data flow specs can be written in YAML format instead of JSON. Contains YAML equivalents of basic specs. | -| **`tpch_sample`** | The most comprehensive end-to-end reference, built on the TPC-H dataset in the UC samples catalog. Covers multi-source schema-on-read bronze, conformed/history-tracked silver (SCD2 + SCD1 + append-only facts with DQ quarantine), and a governed gold star schema with surrogate keys, point-in-time joins, pre-aggregated MVs, and UC metric views. Uses template specs and a three-run incremental simulation. See its [README](tpch_sample/README.md). | +| **`tpch_sample`** | The most comprehensive end-to-end reference, built on the TPC-H dataset in the UC samples catalog. Covers multi-source schema-on-read bronze, conformed/history-tracked silver (SCD2 + SCD1 + append-only facts with DQ quarantine), and a governed gold star schema with surrogate keys, point-in-time joins, pre-aggregated MVs, and UC metric views. Uses template specs and a three-run incremental simulation. See its README.md | ## Deploying the Samples @@ -21,7 +21,7 @@ The samples can be deployed using the scripts located in the `samples` directory ### Prerequisites * Databricks CLI installed and configured -* Lakeflow framework already deployed to your workspace (see [Deploying the Framework](../docs/source/deploy_framework.rst)) +* Lakeflow framework already deployed to your workspace (see {doc}`deploy_framework`) ### Interactive Deployment @@ -156,4 +156,4 @@ It demonstrates a wide range of framework capabilities together, including: * **Template specs** that collapse repetitive bronze and silver flows into reusable templates. * **A three-run incremental simulation** covering SCD changes, fact growth, a backdated out-of-order correction, and ongoing quarantine. -To deploy it, use the `deploy_tpch.sh` script following the same methods described above. See the sample's own [README](tpch_sample/README.md) for the full walkthrough, design choices, and demo flow. +To deploy it, use the `deploy_tpch.sh` script following the same methods described above. See the sample's own See its README.md for the full walkthrough, design choices, and demo flow. From b0540745c324fff5bcd4eddbd86c6873c1b77b75 Mon Sep 17 00:00:00 2001 From: rederik76 <127075662+rederik76@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:36:41 +1000 Subject: [PATCH 9/9] Align package refactor docs to v0.20.0 and fix utility import. Update ADRs and user-facing docs from v0.16.0 to v0.20.0, rename ADR-0009 to workspace-files-first, and use consistent Strategy B terminology in config_resolver. Replace the last bare constants import in utility.py so wheel-only installs no longer depend on flat src/ shims. --- docs/decisions/0008-lakeflow-framework-package.md | 8 ++++---- ... => 0009-strategy-b-workspace-files-first-resolver.md} | 6 +++--- docs/source/contributor_imports.rst | 6 +++--- docs/source/deploy_wheel.rst | 4 ++-- docs/source/feature_framework_configuration.rst | 2 +- docs/source/feature_versioning_framework.rst | 2 +- src/lakeflow_framework/config_resolver.py | 4 ++-- src/lakeflow_framework/utility.py | 2 +- tests/README.md | 1 - 9 files changed, 17 insertions(+), 18 deletions(-) rename docs/decisions/{0009-strategy-b-disk-first-resolver.md => 0009-strategy-b-workspace-files-first-resolver.md} (98%) diff --git a/docs/decisions/0008-lakeflow-framework-package.md b/docs/decisions/0008-lakeflow-framework-package.md index f5cff94..1fcb79c 100644 --- a/docs/decisions/0008-lakeflow-framework-package.md +++ b/docs/decisions/0008-lakeflow-framework-package.md @@ -2,7 +2,7 @@ **Date:** 2026-05-23 **Status:** Accepted -**PR:** refactor/lakeflow-framework-package (v0.16.0) +**PR:** refactor/lakeflow-framework-package (v0.20.0) --- @@ -125,7 +125,7 @@ module is added. | Version | Action | |---------|--------| -| v0.16.0 | `src/lakeflow_framework/` package introduced; bare `src/` imports still work via shims | +| v0.20.0 | `src/lakeflow_framework/` package introduced; bare `src/` imports still work via shims | | v1.0.0 | Compat shims at old flat `src/*.py` paths removed | ## Consequences @@ -136,8 +136,8 @@ module is added. - The `lakeflow_framework.*` import namespace is globally unique; no shadow-import risk with customer code. - Existing flat-deploy customers are unaffected: the `src/` shims preserve - backward compatibility, and `framework.sourcePath` + disk-first resolution - (ADR-0009) means behaviour is identical to pre-v0.16.0. + backward compatibility, and `framework.sourcePath` + Workspace Files-first + resolution (ADR-0009) means behaviour is identical to pre-v0.20.0. - Editable installs (`pip install -e ".[contrib]"`) are supported for local development; the `src/` layout ensures the installed package and the source tree are the same directory. diff --git a/docs/decisions/0009-strategy-b-disk-first-resolver.md b/docs/decisions/0009-strategy-b-workspace-files-first-resolver.md similarity index 98% rename from docs/decisions/0009-strategy-b-disk-first-resolver.md rename to docs/decisions/0009-strategy-b-workspace-files-first-resolver.md index 99d16dd..dbd8512 100644 --- a/docs/decisions/0009-strategy-b-disk-first-resolver.md +++ b/docs/decisions/0009-strategy-b-workspace-files-first-resolver.md @@ -2,7 +2,7 @@ **Date:** 2026-05-23 **Status:** Accepted -**PR:** refactor/lakeflow-framework-package (v0.16.0) +**PR:** refactor/lakeflow-framework-package (v0.20.0) --- @@ -92,7 +92,7 @@ similar validators. - **Zero-change upgrade for flat-deploy customers.** Their `framework.sourcePath` points to the deployed `src/` directory in Workspace Files; those files are found at step 1 and the wheel is never consulted. Behaviour is identical to - pre-v0.16.0. + pre-v0.20.0. - **Explicit wins.** Setting `framework.sourcePath` is a deliberate act. Workspace Files-first honours that intent; package-first would silently override it. - **Wheel-install customers get clean defaults from the package.** They do not @@ -108,7 +108,7 @@ similar validators. config and schema reads; call sites that previously used `os.path.join(framework_path, FrameworkPaths.CONFIG_PATH, name)` are migrated to it. -- Flat-deploy customers see **no behaviour change** on upgrade to v0.16.0. +- Flat-deploy customers see **no behaviour change** on upgrade to v0.20.0. - Wheel-install customers get defaults from the wheel with no `framework.sourcePath` required. - The `src/local/config/` sparse overlay (ADR-0006) continues to work in all diff --git a/docs/source/contributor_imports.rst b/docs/source/contributor_imports.rst index 6382217..02a0f2d 100644 --- a/docs/source/contributor_imports.rst +++ b/docs/source/contributor_imports.rst @@ -25,7 +25,7 @@ Compat shims (do not use in new code) -------------------------------------- The old flat-deploy layout placed modules directly under ``src/`` (for example -``from constants import FrameworkPaths``). Since v0.16.0, matching +``from constants import FrameworkPaths``). Since v0.20.0, matching ``src/*.py`` files are thin re-export shims: .. code-block:: python @@ -90,7 +90,7 @@ Deprecation timeline * - Version - Change - * - v0.16.0 + * - v0.20.0 - ``lakeflow_framework`` package introduced; bare ``src/`` imports still work via shims. * - v1.0.0 @@ -101,7 +101,7 @@ Further reading - ``docs/decisions/0008-lakeflow-framework-package.md`` — package layout, wheel distribution, and shim policy (ADR-0008). -- ``docs/decisions/0009-strategy-b-disk-first-resolver.md`` — how bundled +- ``docs/decisions/0009-strategy-b-workspace-files-first-resolver.md`` — how bundled config and schemas resolve against workspace files (ADR-0009). - :doc:`deploy_wheel` — wheel install and shim guidance for deployers. - :doc:`contributor_contrib` — import rules specific to ``contrib/`` modules. diff --git a/docs/source/deploy_wheel.rst b/docs/source/deploy_wheel.rst index fc1d163..33a7d92 100644 --- a/docs/source/deploy_wheel.rst +++ b/docs/source/deploy_wheel.rst @@ -5,9 +5,9 @@ Installing the Framework as a Wheel :header-rows: 0 * - **Available From:** - - v0.16.0 + - v0.20.0 -From v0.16.0 the Lakeflow Framework ships as a proper Python package +From v0.20.0 the Lakeflow Framework ships as a proper Python package (``lakeflow-framework``) with a ``pyproject.toml``. For a comparison of all deployment modes, see :doc:`deploy_framework_bundle`. diff --git a/docs/source/feature_framework_configuration.rst b/docs/source/feature_framework_configuration.rst index efb34e4..fa22541 100644 --- a/docs/source/feature_framework_configuration.rst +++ b/docs/source/feature_framework_configuration.rst @@ -19,7 +19,7 @@ want to change are needed. .. note:: - **Default config location (v0.16.0+)** + **Default config location (v0.20.0+)** Framework default files now live inside the installed package at ``src/lakeflow_framework/config/default/`` rather than at the top-level diff --git a/docs/source/feature_versioning_framework.rst b/docs/source/feature_versioning_framework.rst index 3266bca..f4b00d2 100644 --- a/docs/source/feature_versioning_framework.rst +++ b/docs/source/feature_versioning_framework.rst @@ -75,7 +75,7 @@ Set this path using the ``BUNDLE_VAR_framework_source_path`` environment variabl modified directly in the Databricks workspace, this is not recommended for production environments. .. note:: - **Wheel-deploy mode (v0.16.0+)** + **Wheel-deploy mode (v0.20.0+)** When using ``pip install lakeflow-framework``, ``framework.sourcePath`` is optional. Default configs and schemas are bundled inside the wheel and diff --git a/src/lakeflow_framework/config_resolver.py b/src/lakeflow_framework/config_resolver.py index 55677bc..4a44650 100644 --- a/src/lakeflow_framework/config_resolver.py +++ b/src/lakeflow_framework/config_resolver.py @@ -38,7 +38,7 @@ def load_framework_default_json( name: str, framework_path: Optional[str] = None, ) -> Dict: - """Load a framework default JSON config file using Strategy B (disk-first). + """Load a framework default JSON config file using Strategy B (Workspace Files-first). Resolution order: @@ -65,7 +65,7 @@ def load_framework_default_json( """ from lakeflow_framework.utility import deep_merge - # 1. Disk-first: explicit framework_path takes priority over package data. + # 1. Workspace Files-first: explicit framework_path takes priority over package data. base: Optional[Dict] = None file_path: Optional[str] = None if framework_path: diff --git a/src/lakeflow_framework/utility.py b/src/lakeflow_framework/utility.py index bfce03a..3b18988 100644 --- a/src/lakeflow_framework/utility.py +++ b/src/lakeflow_framework/utility.py @@ -15,7 +15,7 @@ from pyspark.sql import SparkSession from pyspark.sql.types import StructType -from constants import ( +from lakeflow_framework.constants import ( SupportedSpecFormat, PipelineBundleSuffixesJson, PipelineBundleSuffixesYaml, diff --git a/tests/README.md b/tests/README.md index b8aa719..927505d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -30,7 +30,6 @@ The suite is layered so local runs and CI stay fast by default: **Import policy:** all tests use `lakeflow_framework.*` imports. The compat shim layer at `src/*.py` is not exercised — see `docs/source/contributor_imports.rst`. -**Post-merge layout (v0.16+):** root-level `tests/test_bundle_loader.py` and `tests/test_validate_dataflows.py` were removed. Their coverage now lives under `tests/unit/` (isolated module and script-helper tests) and `tests/integration/` (sample-bundle validation and CLI subprocess tests). ## Layout