Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ uv pip install .

Add *airflow-dbt-python* to your `requirements.txt` file and edit your Airflow environment to use this new `requirements.txt` file, or upload it as a plugin.

Read the [documentation](https://airflow-dbt-python.readthedocs.io/en/latest/getting_started.html#installing-in-mwaa) for more a more detailed AWS MWAA installation breakdown.
Read the [documentation](https://airflow-dbt-python.readthedocs.io/en/latest/getting_started/#installing-in-mwaa) for more a more detailed AWS MWAA installation breakdown.

## In GCP Cloud Composer

Expand Down Expand Up @@ -198,7 +198,7 @@ with DAG(
) as dag:
dbt_test = DbtTestOperator(
task_id="dbt_test",
selector_name="pre-run-tests",
selector="pre-run-tests",
)

dbt_seed = DbtSeedOperator(
Expand All @@ -221,7 +221,7 @@ More examples can be found in the [`examples/`](examples/) directory and the [do

# Development

See the [development documentation](https://airflow-dbt-python.readthedocs.io/en/latest/development.html) for a more in-depth dive into setting up a development environment, running the test-suite, and general commentary on working on *airflow-dbt-python*.
See the [development documentation](https://airflow-dbt-python.readthedocs.io/en/latest/development/) for a more in-depth dive into setting up a development environment, running the test-suite, and general commentary on working on *airflow-dbt-python*.

## Testing

Expand Down
26 changes: 17 additions & 9 deletions airflow_dbt_python/operators/dbt.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,14 +400,15 @@ def __init__(
full_refresh: Optional[bool] = None,
models: Optional[list[str]] = None,
select: Optional[list[str]] = None,
selector_name: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
exclude: Optional[list[str]] = None,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.full_refresh = full_refresh
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name
self.select = select or models

@property
Expand All @@ -432,6 +433,7 @@ def __init__(
select: Optional[list[str]] = None,
show: Optional[bool] = None,
exclude: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
**kwargs,
) -> None:
Expand All @@ -440,7 +442,7 @@ def __init__(
self.select = select
self.show = show
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name

@property
def command(self) -> str:
Expand All @@ -465,6 +467,7 @@ def __init__(
models: Optional[list[str]] = None,
select: Optional[list[str]] = None,
exclude: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
indirect_selection: Optional[str] = None,
**kwargs,
Expand All @@ -473,7 +476,7 @@ def __init__(
self.singular = singular
self.generic = generic
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name
self.select = select or models
self.indirect_selection = indirect_selection

Expand All @@ -500,6 +503,7 @@ def __init__(
models: Optional[list[str]] = None,
select: Optional[list[str]] = None,
exclude: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
upload_dbt_project: bool = True,
**kwargs,
Expand All @@ -508,7 +512,7 @@ def __init__(
self.parse_only = parse_only
self.full_refresh = full_refresh
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name
self.select = select or models
self.upload_dbt_project = upload_dbt_project

Expand Down Expand Up @@ -614,13 +618,14 @@ def __init__(
self,
select: Optional[list[str]] = None,
exclude: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.select = select
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name

@property
def command(self) -> str:
Expand All @@ -644,6 +649,7 @@ def __init__(
resource_types: Optional[list[str]] = None,
select: Optional[list[str]] = None,
exclude: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
dbt_output: Optional[Output] = None,
output_keys: Optional[list[str]] = None,
Expand All @@ -654,7 +660,7 @@ def __init__(
self.resource_types = resource_types
self.select = select
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name
self.dbt_output = dbt_output
self.output_keys = output_keys
self.indirect_selection = indirect_selection
Expand Down Expand Up @@ -727,14 +733,15 @@ def __init__(
select: Optional[list[str]] = None,
dbt_output: Optional[Union[str, Path]] = None,
exclude: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
upload_dbt_project: bool = True,
**kwargs,
) -> None:
super().__init__(**kwargs)
self.select = select
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name
self.dbt_output = dbt_output
self.upload_dbt_project = upload_dbt_project

Expand All @@ -759,6 +766,7 @@ def __init__(
full_refresh: Optional[bool] = None,
select: Optional[list[str]] = None,
exclude: Optional[list[str]] = None,
selector: Optional[str] = None,
selector_name: Optional[str] = None,
singular: Optional[bool] = None,
generic: Optional[bool] = None,
Expand All @@ -770,7 +778,7 @@ def __init__(
self.full_refresh = full_refresh
self.select = select
self.exclude = exclude
self.selector_name = selector_name
self.selector = selector or selector_name
self.singular = singular
self.generic = generic
self.show = show
Expand Down
4 changes: 3 additions & 1 deletion airflow_dbt_python/utils/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -518,11 +518,13 @@ class SelectionConfig(BaseConfig):

exclude: Optional[list[str]] = None
select: Optional[list[str]] = None
selector_name: Optional[list[str]] = None
selector: Optional[str] = None
state: Optional[Union[Path, str]] = None
defer_state: Optional[Union[Path, str]] = None

# Kept for compatibility with dbt versions < 1.5
selector_name: Optional[str] = None

# Kept for compatibility with dbt versions < 0.21
models: Optional[list[str]] = None

Expand Down
16 changes: 13 additions & 3 deletions docs/development.rst
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,17 @@ Create a local virtual environment (if you don't want *uv* to manage it), and en

.. code-block:: shell

uv sync --dev --extra airflow-providers --extra postgres
uv sync --all-extras --dev

The *dev* dependency group includes development tools for code formatting, type checking, and testing.

The additional extras, *airflow-providers* and *postgres* install dependencies required for testing. If testing a specific Airflow version, the extras may be omitted, see the following section with more details.
The additional extras install dependencies required for testing. If testing a specific Airflow version, the extras may be omitted, see the following section with more details.

*airflow-dbt-python* does not have a build system (yet!) so it needs to be installed separately:

.. code-block:: shell

uv pip install -e .

Support for different versions of *Airflow*
-------------------------------------------
Expand Down Expand Up @@ -96,7 +102,11 @@ The ``AIRFLOW_HOME`` environment variable has to be set to the same value used w

The files ``airflow.cfg`` and ``airflow.db`` created as part of initializing the database can be safely deleted once not needed anymore.

Finally, some unit tests require Airflow provider packages. These are all provided by the *airflow-providers* extra.
Afterwards, ensure default connections are created by running:

.. code-block:: shell

uv run airflow connections create-default-connections

Running tests
^^^^^^^^^^^^^
Expand Down
2 changes: 1 addition & 1 deletion examples/readme_example_dbt_dag.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
) as dag:
dbt_test = DbtTestOperator(
task_id="dbt_test",
selector_name="pre-run-tests",
selector="pre-run-tests",
)

dbt_seed = DbtSeedOperator(
Expand Down
17 changes: 17 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,23 @@ def model_files(dbt_project_dir):
return paths


@pytest.fixture(scope="session")
def selector_file(model_files, dbt_project_dir):
"""Create a selector file."""
p = dbt_project_dir / "selectors.yml"
last_model = [str(m.stem) for m in model_files][-1]

p.write_text(f"""\
selectors:
- name: first_model
definition:
method: fqn
value: "{last_model}"
""")

return p


@pytest.fixture(scope="session")
def sources_file(model_files, database):
"""Create test source file."""
Expand Down
4 changes: 2 additions & 2 deletions tests/operators/test_dbt_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def test_dbt_build_mocked_all_args():
fail_fast=True,
threads=3,
exclude=["/path/to/model/to/exclude.sql"],
selector_name=["a-selector"],
selector="a-selector",
state="/path/to/state/",
singular=True,
generic=True,
Expand All @@ -62,7 +62,7 @@ def test_dbt_build_mocked_all_args():
"test_type:generic",
]
assert config.exclude == ["/path/to/model/to/exclude.sql"]
assert config.selector_name == ["a-selector"]
assert config.selector == "a-selector"
assert config.state == Path("/path/to/state/")
assert config.singular is True
assert config.generic is True
Expand Down
8 changes: 5 additions & 3 deletions tests/operators/test_dbt_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def test_dbt_compile_mocked_all_args():
models=["/path/to/model1.sql", "/path/to/model2.sql"],
threads=2,
exclude=["/path/to/data/to/exclude.sql"],
selector_name=["a-selector"],
selector="a-selector",
state="/path/to/state/",
)

Expand All @@ -49,7 +49,7 @@ def test_dbt_compile_mocked_all_args():
"/path/to/model2.sql",
]
assert config.exclude == ["/path/to/data/to/exclude.sql"]
assert config.selector_name == ["a-selector"]
assert config.selector == "a-selector"
assert config.state == Path("/path/to/state/")


Expand Down Expand Up @@ -198,9 +198,11 @@ def test_dbt_compile_uses_correct_argument_according_to_version():
models=["/path/to/model1.sql", "/path/to/model2.sql"],
threads=2,
exclude=["/path/to/data/to/exclude.sql"],
selector_name=["a-selector"],
selector_name="a-selector",
state="/path/to/state/",
)

assert op.select == ["/path/to/model1.sql", "/path/to/model2.sql"]
assert getattr(op, "models", None) is None
assert op.selector == "a-selector"
assert getattr(op, "selector_name", None) is None
40 changes: 37 additions & 3 deletions tests/operators/test_dbt_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def test_dbt_run_mocked_all_args():
fail_fast=True,
threads=3,
exclude=["/path/to/model/to/exclude.sql"],
selector_name=["a-selector"],
selector="a-selector",
state="/path/to/state/",
)
assert op.command == "run"
Expand All @@ -54,7 +54,7 @@ def test_dbt_run_mocked_all_args():
assert config.threads == 3
assert config.select == ["/path/to/model.sql", "+/another/model.sql+2"]
assert config.exclude == ["/path/to/model/to/exclude.sql"]
assert config.selector_name == ["a-selector"]
assert config.selector == "a-selector"
assert config.state == Path("/path/to/state/")


Expand Down Expand Up @@ -136,6 +136,38 @@ def test_dbt_run_models(profiles_file, dbt_project_file, model_files, logs_dir):
)


def test_dbt_run_models_with_first_model_selector(
profiles_file, dbt_project_file, model_files, selector_file, logs_dir
):
"""Test execution of DbtRunOperator with a single model selector."""
op = DbtRunOperator(
task_id="dbt_task",
project_dir=dbt_project_file.parent,
profiles_dir=profiles_file.parent,
selector="first_model",
do_xcom_push=True,
log_path=logs_dir,
log_level_file="info",
debug=True,
)

execution_results = op.execute({})
run_result = execution_results["results"][0]

assert run_result["status"] == RunStatus.Success

log_file = logs_dir / "dbt.log"
assert log_file.exists()

with open(log_file) as f:
logs = f.read()

assert (
"OK created view model public.model_4" in logs
or "OK created sql view model public.model_4" in logs
)


def test_dbt_run_models_with_env_vars(
profiles_file_with_env, dbt_project_file, model_files, logs_dir, database
):
Expand Down Expand Up @@ -334,12 +366,14 @@ def test_dbt_run_uses_correct_argument_according_to_version():
fail_fast=True,
threads=3,
exclude=["/path/to/model/to/exclude.sql"],
selector_name=["a-selector"],
selector_name="a-selector",
state="/path/to/state/",
)

assert op.select == ["/path/to/model.sql", "+/another/model.sql+2"]
assert getattr(op, "models", None) is None
assert op.selector == "a-selector"
assert getattr(op, "selector_name", None) is None


def test_dbt_run_models_with_airflow_connection(
Expand Down
4 changes: 2 additions & 2 deletions tests/operators/test_dbt_seed.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def test_dbt_seed_mocked_all_args():
show=True,
threads=2,
exclude=["/path/to/data/to/exclude.csv"],
selector_name=["a-selector"],
selector="a-selector",
state="/path/to/state/",
)
assert op.command == "seed"
Expand All @@ -54,7 +54,7 @@ def test_dbt_seed_mocked_all_args():
assert config.select == ["/path/to/data.csv"]
assert config.show is True
assert config.exclude == ["/path/to/data/to/exclude.csv"]
assert config.selector_name == ["a-selector"]
assert config.selector == "a-selector"
assert config.state == Path("/path/to/state/")


Expand Down
Loading
Loading