Chore: Deprecate Airflow integration (#4180)

Author: izeigerman

.circleci/continue_config.yml

@@ -143,39 +143,6 @@ jobs:
           name: Run tests
           command: npm --prefix web/client run test
 
-  airflow_docker_tests:
-    machine:
-      image: ubuntu-2204:2022.10.2
-      docker_layer_caching: true
-    resource_class: large
-    environment:
-      PYTEST_XDIST_AUTO_NUM_WORKERS: 8
-      SQLMESH__DISABLE_ANONYMIZED_ANALYTICS: "1"
-    steps:
-      - checkout
-      - run:
-          name: Install envsubst
-          command: sudo apt-get update && sudo apt-get install gettext-base
-      - run:
-          name: Setup python env
-          command: |
-            pip3 install --upgrade pip
-            pip3 install ruamel.yaml==0.16.0
-            python3 --version
-      - run:
-          name: Run Airflow slow tests
-          command: make airflow-docker-test-with-env
-          no_output_timeout: 15m
-      - run:
-          name: Collect Airflow logs
-          command: |
-            tar -czf ./airflow_logs.tgz -C ./examples/airflow/logs .
-            mkdir -p /tmp/airflow_logs
-            cp ./airflow_logs.tgz /tmp/airflow_logs/
-          when: on_fail
-      - store_artifacts:
-          path: /tmp/airflow_logs
-
   trigger_private_tests:
       docker:
         - image: cimg/python:3.12.0
@@ -281,13 +248,6 @@ workflows:
                 - "3.10"
                 - "3.11"
                 - "3.12"
-      - airflow_docker_tests:
-          requires:
-            - style_and_cicd_tests
-          filters:
-            branches:
-              only:
-                - main
       - engine_tests_docker:
           name: engine_<< matrix.engine >>
           matrix:

.gitignore

@@ -138,15 +138,6 @@ dmypy.json
 *~
 *#
 
-# Airflow example
-examples/airflow/Dockerfile
-examples/airflow/docker-compose.yaml
-examples/airflow/airflow.sh
-examples/airflow/.env
-examples/airflow/logs
-examples/airflow/plugins
-examples/airflow/warehouse
-
 *.duckdb
 *.duckdb.wal
 

Makefile

@@ -19,7 +19,7 @@ ui-style:
 	SKIP=ruff,ruff-format,mypy pre-commit run --all-files
 
 doc-test:
-	PYTEST_PLUGINS=tests.common_fixtures python -m pytest --doctest-modules sqlmesh/core sqlmesh/utils
+	python -m pytest --doctest-modules sqlmesh/core sqlmesh/utils
 
 package:
 	pip3 install build && python3 -m build
@@ -33,24 +33,6 @@ package-tests:
 publish-tests: package-tests
 	pip3 install twine && python3 -m twine upload -r tobiko-private tests/dist/*
 
-airflow-init:
-	export AIRFLOW_ENGINE_OPERATOR=spark && make -C ./examples/airflow init
-
-airflow-run:
-	make -C ./examples/airflow run
-
-airflow-stop:
-	make -C ./examples/airflow stop
-
-airflow-clean:
-	make -C ./examples/airflow clean
-
-airflow-psql:
-	make -C ./examples/airflow psql
-
-airflow-spark-sql:
-	make -C ./examples/airflow spark-sql
-
 docs-serve:
 	mkdocs serve
 
@@ -91,27 +73,10 @@ cicd-test:
 	pytest -n auto -m "fast or slow" --junitxml=test-results/junit-cicd.xml && pytest -m "isolated"
 
 core-fast-test:
-	pytest -n auto -m "fast and not web and not github and not dbt and not airflow and not jupyter"
+	pytest -n auto -m "fast and not web and not github and not dbt and not jupyter"
 
 core-slow-test:
-	pytest -n auto -m "(fast or slow) and not web and not github and not dbt and not airflow and not jupyter"
-
-airflow-fast-test:
-	pytest -n auto -m "fast and airflow"
-
-airflow-test:
-	pytest -n auto -m "(fast or slow) and airflow"
-
-airflow-local-test:
-	export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@localhost/airflow && \
-		pytest -n 1 -m "docker and airflow"
-
-airflow-docker-test:
-	make -C ./examples/airflow docker-test
-
-airflow-local-test-with-env: install-dev airflow-clean airflow-init airflow-run airflow-local-test airflow-stop
-
-airflow-docker-test-with-env: install-dev airflow-clean airflow-init airflow-run airflow-docker-test airflow-stop
+	pytest -n auto -m "(fast or slow) and not web and not github and not dbt and not jupyter"
 
 engine-slow-test:
 	pytest -n auto -m "(fast or slow) and engine"

docs/comparisons.md

@@ -37,7 +37,6 @@ SQLMesh aims to be dbt format-compatible. Importing existing dbt projects with m
 | `Virtual Data Environments`       | ❌ | [✅](../concepts/environments)
 | `Open-source CI/CD bot`           | ❌ | [✅](../integrations/github)
 | `Data consistency enforcement`    | ❌ | ✅
-| `Native Airflow integration`      | ❌ | [✅](../integrations/airflow)
 | Interfaces
 | `CLI`                             | ✅ | [✅](../reference/cli)
 | `Paid UI`                         | ✅ | ❌

docs/concepts/models/sql_models.md

@@ -281,7 +281,7 @@ JOIN countries
 
 SQLMesh will detect that the model depends on both `employees` and `countries`. When executing this model, it will ensure that `employees` and `countries` are executed first.
 
-External dependencies not defined in SQLMesh are also supported. SQLMesh can either depend on them implicitly through the order in which they are executed, or through signals if you are using [Airflow](../../integrations/airflow.md).
+External dependencies not defined in SQLMesh are also supported. SQLMesh can either depend on them implicitly through the order in which they are executed, or through [signals](../../guides/signals.md).
 
 Although automatic dependency detection works most of the time, there may be specific cases for which you want to define dependencies manually. You can do so in the `MODEL` DDL with the [dependencies property](./overview.md#properties).
 

docs/concepts/overview.md

@@ -68,6 +68,4 @@ SQLMesh automatically runs audits when you apply a `plan` to an environment, or
 ## Infrastructure and orchestration
 Every company's data infrastructure is different. SQLMesh is flexible with regard to which engines and orchestration frameworks you use — its only requirement is access to the target SQL/analytics engine.
 
-SQLMesh keeps track of model versions and processed data intervals using your existing infrastructure. If SQLMesh is configured without an external orchestrator (such as Airflow), it automatically creates a `sqlmesh` schema in your data warehouse for its internal metadata.
-
-If SQLMesh is configured with Airflow, then it will store all its metadata in the Airflow database. Read more about how [SQLMesh integrates with Airflow](../integrations/airflow.md).
+SQLMesh keeps track of model versions and processed data intervals using your existing infrastructure. SQLMesh it automatically creates a `sqlmesh` schema in your data warehouse for its internal metadata.

docs/development.md

@@ -25,10 +25,6 @@ Run more comprehensive tests that run on each commit:
 ```bash
 make slow-test
 ```
-Run Airflow tests that will run when PR is merged to main:
-```bash
-make airflow-docker-test-with-env
-```
 Install docs dependencies:
 ```bash
 make install-doc

docs/faq/faq.md

@@ -167,14 +167,17 @@
 ## Scheduling
 
 ??? question "How do I run SQLMesh models on a schedule?"
-    You can run SQLMesh models using the [built-in scheduler](../guides/scheduling.md#built-in-scheduler) or with the native [Airflow integration](../integrations/airflow.md).
+    You can run SQLMesh models using the [built-in scheduler](../guides/scheduling.md#built-in-scheduler) or using [Tobiko Cloud](../cloud/features/scheduler/scheduler.md)
 
     Both approaches use each model's `cron` parameter to determine when the model should run - see the [question about `cron` above](#cron-question) for more information.
 
     The built-in scheduler works by executing the command `sqlmesh run`. A sensible approach to running on your project on a schedule is to use Linux’s `cron` tool to execute `sqlmesh run` on a cadence at least as frequent as your briefest SQLMesh model `cron` parameter. For example, if your most frequent model’s `cron` is hour, the `cron` tool should execute `sqlmesh run` at least every hour.
 
 ??? question "How do I use SQLMesh with Airflow?"
-    SQLMesh has first-class support for Airflow - learn more [here](../integrations/airflow.md).
+    Tobiko Cloud offers first-class support for Airflow - learn more [here](../cloud/features/scheduler/airflow.md)
+
+??? question "How do I use SQLMesh with Dagster?"
+    Tobiko Cloud offers first-class support for Dagster - learn more [here](../cloud/features/scheduler/dagster.md)
 
 ## Warnings and Errors
 

docs/guides/configuration.md

@@ -501,7 +501,7 @@ These pages describe the connection configuration options for each execution eng
 
 Configuration for the state backend connection if different from the data warehouse connection.
 
-The data warehouse connection is used to store SQLMesh state if the `state_connection` key is not specified, unless the configuration uses an Airflow or Google Cloud Composer scheduler. If using one of those schedulers, the scheduler's database is used (not the data warehouse) unless an [Airflow Connection has been configured](../integrations/airflow.md#state-connection).
+The data warehouse connection is used to store SQLMesh state if the `state_connection` key is not specified.
 
 Unlike data transformations, storing state information requires database transactions. Data warehouses aren’t optimized for executing transactions, and storing state information in them can slow down your project or produce corrupted data due to simultaneous writes to the same table. Therefore, production SQLMesh deployments should use a dedicated state connection.
 
@@ -675,7 +675,7 @@ Configuration for a connection used to run unit tests. An in-memory DuckDB datab
 
 ### Scheduler
 
-Identifies which scheduler backend to use. The scheduler backend is used both for storing metadata and for executing [plans](../concepts/plans.md). By default, the scheduler type is set to `builtin`, which uses the existing SQL engine to store metadata. Use the `airflow` type integrate with Airflow.
+Identifies which scheduler backend to use. The scheduler backend is used both for storing metadata and for executing [plans](../concepts/plans.md). By default, the scheduler type is set to `builtin`, which uses the existing SQL engine to store metadata. 
 
 These options are in the [scheduler](../reference/configuration.md#scheduler) section of the configuration reference page.
 
@@ -716,89 +716,6 @@ Example configuration:
 
 No additional configuration options are supported by this scheduler type.
 
-#### Airflow
-
-Example configuration:
-
-=== "YAML"
-
-    ```yaml linenums="1"
-    gateways:
-      my_gateway:
-        scheduler:
-          type: airflow
-          airflow_url: <airflow_url>
-          username: <username>
-          password: <password>
-    ```
-
-=== "Python"
-
-    An Airflow scheduler is specified with an `AirflowSchedulerConfig` object.
-
-    ```python linenums="1"
-    from sqlmesh.core.config import (
-        Config,
-        ModelDefaultsConfig,
-        GatewayConfig,
-        AirflowSchedulerConfig,
-    )
-
-    config = Config(
-        model_defaults=ModelDefaultsConfig(dialect=<dialect>),
-        gateways={
-            "my_gateway": GatewayConfig(
-                scheduler=AirflowSchedulerConfig(
-                    airflow_url=<airflow_url>,
-                    username=<username>,
-                    password=<password>,
-                ),
-            ),
-        }
-    )
-    ```
-
-See [Airflow Integration Guide](../integrations/airflow.md) for information about how to integrate Airflow with SQLMesh. See the [configuration reference page](../reference/configuration.md#airflow) for a list of all parameters.
-
-#### Cloud Composer
-
-The Google Cloud Composer scheduler type shares the same configuration options as the `airflow` type, except for `username` and `password`. Cloud Composer relies on `gcloud` authentication, so the `username` and `password` options are not required.
-
-Example configuration:
-
-=== "YAML"
-
-    ```yaml linenums="1"
-    gateways:
-      my_gateway:
-        scheduler:
-          type: cloud_composer
-          airflow_url: <airflow_url>
-    ```
-
-=== "Python"
-
-    An Google Cloud Composer scheduler is specified with an `CloudComposerSchedulerConfig` object.
-
-    ```python linenums="1"
-    from sqlmesh.core.config import (
-        Config,
-        ModelDefaultsConfig,
-        GatewayConfig,
-        CloudComposerSchedulerConfig,
-    )
-
-    config = Config(
-        model_defaults=ModelDefaultsConfig(dialect=<dialect>),
-        gateways={
-            "my_gateway": GatewayConfig(
-                scheduler=CloudComposerSchedulerConfig(
-                    airflow_url=<airflow_url>,
-                ),
-            ),
-        }
-    )
-    ```
 
 ### Gateway/connection defaults
 

docs/guides/connections.md

@@ -2,8 +2,6 @@
 
 ## Overview
 
-**Note:** The following guide only applies when using the built-in scheduler. Connections are configured differently when using an external scheduler such as Airflow. See the [Scheduling guide](scheduling.md) for more details.
-
 In order to deploy models and to apply changes to them, you must configure a connection to your Data Warehouse and, optionally, connection to the database where the SQLMesh state is stored. This can be done in either the `config.yaml` file in your project folder, or the one in `~/.sqlmesh`.
 
 Each connection is configured as part of a gateway which has a unique name associated with it. The gateway name can be used to select a specific combination of connection settings  when using the CLI. For example:
@@ -23,7 +21,7 @@ sqlmesh --gateway local_db plan
 
 ## State connection
 
-By default, the data warehouse connection is also used to store the SQLMesh state, unless the configuration uses an Airflow or Google Cloud Composer scheduler. If using one of those schedulers, the state connection defaults to the scheduler's database.
+By default, the data warehouse connection is also used to store the SQLMesh state.
 
 The state connection can be changed by providing different connection settings in the `state_connection` key of the gateway configuration: