Merge branch 'master' into 1944-add-orchestrator-job-data-alerts

Author: arbiv

.github/workflows/test-warehouse.yml

@@ -26,16 +26,6 @@ on:
         type: string
         required: false
         description: dbt's version to test with
-      should-run-tests:
-        type: boolean
-        required: false
-        default: true
-        description: Whether to run E2E tests
-      clear-tests:
-        type: boolean
-        required: false
-        default: true
-        description: Whether to clean test environment
       generate-data:
         type: boolean
         required: false
@@ -47,10 +37,6 @@ on:
       warehouse-type:
         type: string
         required: true
-      should-run-tests:
-        type: boolean
-        required: false
-        default: true
       elementary-ref:
         type: string
         required: false
@@ -60,24 +46,37 @@ on:
       dbt-version:
         type: string
         required: false
-      clear-tests:
-        type: boolean
-        required: false
-        default: true
       generate-data:
         type: boolean
         required: false
         default: false
 
 env:
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
-  DBT_PKG_INTEG_TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests/deprecated_tests
-  ELMENTARY_INTERNAL_DBT_PKG_DIR: ${{ github.workspace }}/elementary/elementary/monitor/dbt_project
+  ELEMENTARY_DBT_PACKAGE_PATH: ${{ github.workspace }}/dbt-data-reliability
+  CLI_INTERNAL_DBT_PKG_DIR: ${{ github.workspace }}/elementary/elementary/monitor/dbt_project
+  E2E_DBT_PROJECT_DIR: ${{ github.workspace }}/elementary/tests/e2e_dbt_project
 
 jobs:
+  # PRs from forks require approval, specifically with the "pull_request_target" event as it contains repo secrets.
+  check-if-requires-approval:
+    runs-on: ubuntu-latest
+    outputs:
+      requires_approval: ${{ steps.set-output.outputs.requires_approval }}
+    steps:
+      - name: Set requires approval output
+        id: set-output
+        run: |
+          if [[ "${{ github.event_name }}" =~ ^pull_request && "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]]; then
+            echo "requires_approval=true" >> $GITHUB_OUTPUT
+          else
+            echo "requires_approval=false" >> $GITHUB_OUTPUT
+          fi
+
   test:
     runs-on: ubuntu-latest
-    environment: elementary_test_env # This is a github environment (not to be confused with env vars)
+    needs: [check-if-requires-approval]
+    environment: ${{ (needs.check-if-requires-approval.outputs.requires_approval == 'true' && 'elementary_test_env') || '' }}
     defaults:
       run:
         working-directory: elementary
@@ -101,12 +100,12 @@ jobs:
 
       - name: Start Postgres
         if: inputs.warehouse-type == 'postgres'
-        working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
         run: docker compose up -d postgres
 
       # - name: Start Clickhouse
       #   if: inputs.warehouse-type == 'clickhouse'
-      #   working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
+      #   working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
       #   run: docker compose up -d clickhouse
 
       - name: Setup Python
@@ -122,8 +121,7 @@ jobs:
         run: >
           pip install
           "dbt-core${{ inputs.dbt-version && format('=={0}', inputs.dbt-version) }}"
-          # TODO: remove the <1.10.2 once we have a fix for https://github.com/elementary-data/elementary/issues/1931
-          "dbt-${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks<1.10.2,') || inputs.warehouse-type }}${{ inputs.dbt-version && format('~={0}', inputs.dbt-version) }}"
+          "dbt-${{ (inputs.warehouse-type == 'databricks_catalog' && 'databricks') || inputs.warehouse-type }}${{ inputs.dbt-version && format('~={0}', inputs.dbt-version) }}"
 
       - name: Install Elementary
         run: |
@@ -136,7 +134,7 @@ jobs:
         run: |
           mkdir -p ~/.dbt
           DBT_VERSION=$(pip show dbt-core | grep -i version | awk '{print $2}' | sed 's/\.//g')
-          UNDERSCORED_REF_NAME=$(echo "dbt_${DBT_VERSION}_${BRANCH_NAME}" | head -c 32 | sed "s/-/_/g")
+          UNDERSCORED_REF_NAME=$(echo "${{ inputs.warehouse-type }}_dbt_${DBT_VERSION}_${BRANCH_NAME}" | awk '{print tolower($0)}' | head -c 40 | sed "s/[-\/]/_/g")
           echo "$PROFILES_YML" | base64 -d | sed "s/<SCHEMA_NAME>/py_$UNDERSCORED_REF_NAME/g" > ~/.dbt/profiles.yml
 
       - name: Run Python package unit tests
@@ -154,12 +152,46 @@ jobs:
           rm -rf "$DBT_PKGS_PATH/elementary"
           ln -vs "$GITHUB_WORKSPACE/dbt-data-reliability" "$DBT_PKGS_PATH/elementary"
 
-      - name: Run dbt package integration tests
-        if: github.event_name != 'workflow_dispatch' || inputs.should-run-tests
-        working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
+      - name: Run deps for E2E dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        env:
+          ELEMENTARY_DBT_PACKAGE_PATH: ${{ env.ELEMENTARY_DBT_PACKAGE_PATH }}
         run: |
           dbt deps
-          python run_e2e_tests.py -t "${{ inputs.warehouse-type }}" -g "${{ inputs.warehouse-type == 'postgres' || inputs.generate-data }}" --clear-tests "${{ inputs.clear-tests }}"
+
+      - name: Seed e2e dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        if: inputs.warehouse-type == 'postgres' || inputs.generate-data
+        run: |
+          python generate_data.py
+          dbt seed -f --target "${{ inputs.warehouse-type }}"
+
+      - name: Run e2e dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        run: |
+          dbt run --target "${{ inputs.warehouse-type }}" || true
+
+          # Validate run_results.json: only error_model should be non-success
+          jq -e '
+            [.results[] | select(.status != "success") | .unique_id]
+            | length == 1 and .[0] == "model.elementary_integration_tests.error_model"
+          ' target/run_results.json > /dev/null
+          jq_exit=$?
+
+          if [ $jq_exit -eq 0 ]; then
+            echo "✅ Validation passed: only error_model failed."
+          else
+            echo "❌ Validation failed. Unexpected failures:"
+            jq '[.results[] | select(.status != "success") | .unique_id] | join(", ")' target/run_results.json
+          fi
+
+          exit $jq_exit
+
+      - name: Test e2e dbt project
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        continue-on-error: true
+        run: |
+          dbt test --target "${{ inputs.warehouse-type }}"
 
       - name: Run help
         run: edr --help
@@ -168,15 +200,15 @@ jobs:
         env:
           SLACK_WEBHOOK: ${{ secrets.CI_SLACK_WEBHOOK }}
         run: >
-          edr monitor 
+          edr monitor
           -t "${{ inputs.warehouse-type }}"
           --group-by table
-          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          --project-dir "${{ env.E2E_DBT_PROJECT_DIR }}"
           --project-profile-target "${{ inputs.warehouse-type }}"
           --slack-webhook "$SLACK_WEBHOOK"
 
       - name: Validate alerts statuses were updated
-        working-directory: ${{ env.ELMENTARY_INTERNAL_DBT_PKG_DIR }}
+        working-directory: ${{ env.CLI_INTERNAL_DBT_PKG_DIR }}
         run: |
           dbt deps
           dbt run-operation validate_alert_statuses_are_updated -t "${{ inputs.warehouse-type }}"
@@ -185,13 +217,19 @@ jobs:
         run: >
           edr monitor report
           -t "${{ inputs.warehouse-type }}"
-          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          --project-dir "${{ env.E2E_DBT_PROJECT_DIR }}"
           --project-profile-target "${{ inputs.warehouse-type }}"
 
+      - name: Set report artifact name
+        id: set_report_artifact_name
+        run: |
+          ARTIFACT_NAME=$(echo "report_${{ inputs.warehouse-type }}_${BRANCH_NAME}_dbt_${{ inputs.dbt-version || '' }}.html" | awk '{print tolower($0)}' | sed 's#[":/\\<>|*?-]#_#g')
+          echo "artifact_name=$ARTIFACT_NAME" >> "$GITHUB_OUTPUT"
+
       - name: Upload report artifact
         uses: actions/upload-artifact@v4
         with:
-          name: report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}_dbt_${{ inputs.dbt-version }}.html
+          name: ${{ steps.set_report_artifact_name.outputs.artifact_name }}
           path: elementary/edr_target/elementary_report.html
 
       - name: Write GCS keyfile
@@ -206,9 +244,9 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AZURE_CONNECTION_STRING: ${{ secrets.AZURE_CONNECTION_STRING }}
         run: >
-          edr monitor send-report 
+          edr monitor send-report
           -t "${{ inputs.warehouse-type }}"
-          --project-dir "${{ env.DBT_PKG_INTEG_TESTS_DIR }}"
+          --project-dir "${{ env.E2E_DBT_PROJECT_DIR }}"
           --project-profile-target "${{ inputs.warehouse-type }}"
           --slack-file-name "report_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}.html"
           --slack-token "$SLACK_TOKEN"
@@ -223,13 +261,18 @@ jobs:
           --azure-container-name reports
           --update-bucket-website true
 
+      - name: Set artifact name
+        id: set_artifact_name
+        run: |
+          ARTIFACT_NAME=$(echo "edr_${{ inputs.warehouse-type }}_${BRANCH_NAME}_dbt_${{ inputs.dbt-version || '' }}.log" | awk '{print tolower($0)}' | sed 's#[":/\\<>|*?-]#_#g')
+          echo "artifact_name=$ARTIFACT_NAME" >> "$GITHUB_OUTPUT"
+
       - name: Upload edr log
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
-          name: edr_${{ inputs.warehouse-type }}_${{ env.BRANCH_NAME }}_dbt_${{ inputs.dbt-version }}.log
+          name: ${{ steps.set_artifact_name.outputs.artifact_name }}
           path: elementary/edr_target/edr.log
 
       - name: Run Python package e2e tests
-        if: github.event_name != 'workflow_dispatch' || inputs.should-run-tests
         run: pytest -vv tests/e2e --warehouse-type ${{ inputs.warehouse-type }}

.gitignore

@@ -96,4 +96,4 @@ venv/
 
 # elementary outputs
 edr_target/
-tests/tests_with_db/dbt_project/dbt_packages/
+**/dbt_packages/

README.md

@@ -101,6 +101,10 @@ For additional information and help, you can use one of these channels:
 <a href="https://github.com/erikzaadi"><img src="https://avatars.githubusercontent.com/u/77775?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/dapollak"><img src="https://avatars.githubusercontent.com/u/13542197?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/Adamgo23"><img src="https://avatars.githubusercontent.com/u/125807478?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/MikaKerman"><img src="https://avatars.githubusercontent.com/u/24632225?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/GuyEshdat"><img src="https://avatars.githubusercontent.com/u/81393741?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/michael-myaskovsky"><img src="https://avatars.githubusercontent.com/u/203525071?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/arbiv"><img src="https://avatars.githubusercontent.com/u/12132333?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/RoiTabach"><img src="https://avatars.githubusercontent.com/u/25003091?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/hahnbeelee"><img src="https://avatars.githubusercontent.com/u/55263191?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/seanglynn-thrive"><img src="https://avatars.githubusercontent.com/u/93200565?v=4" width="50" height="50" alt=""/></a>
@@ -199,6 +203,12 @@ For additional information and help, you can use one of these channels:
 <a href="https://github.com/nkmr-jp"><img src="https://avatars.githubusercontent.com/u/8490118?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/dbt-markwan"><img src="https://avatars.githubusercontent.com/u/114556261?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/pedro-klein-ext-bayer"><img src="https://avatars.githubusercontent.com/u/181362556?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/ClementSicard"><img src="https://avatars.githubusercontent.com/u/33360172?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/nickozilla"><img src="https://avatars.githubusercontent.com/u/14976256?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/Lawiss"><img src="https://avatars.githubusercontent.com/u/30115537?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/abhipalsingh"><img src="https://avatars.githubusercontent.com/u/57302403?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/pushrbx"><img src="https://avatars.githubusercontent.com/u/6832715?v=4" width="50" height="50" alt=""/></a>
+
 
 
 <!-- markdownlint-restore -->

docs/_snippets/quickstart-package-install.mdx

@@ -39,7 +39,7 @@ Some packages we recommend you check out: [dbt_utils](https://github.com/dbt-lab
     ```yml packages.yml
     packages:
       - package: elementary-data/elementary
-        version: 0.19.2
+        version: 0.20.0
         ## Docs: https://docs.elementary-data.com
     ```
   </Step>

elementary/clients/dbt/api_dbt_runner.py

@@ -57,7 +57,9 @@ def collect_dbt_command_logs(event):
                 logs=[DbtLog.from_log_line(log) for log in dbt_logs],
             )
 
-        return APIDbtCommandResult(success=res.success, output=output, result_obj=res)
+        return APIDbtCommandResult(
+            success=res.success, output=output, stderr=None, result_obj=res
+        )
 
     def _parse_ls_command_result(
         self, select: Optional[str], result: DbtCommandResult

elementary/clients/dbt/command_line_dbt_runner.py

@@ -25,6 +25,7 @@
 class DbtCommandResult:
     success: bool
     output: Optional[str]
+    stderr: Optional[str]
 
 
 class CommandLineDbtRunner(BaseDbtRunner):
@@ -190,22 +191,28 @@ def run_operation(
         log_pattern = (
             RAW_EDR_LOGS_PATTERN if return_raw_edr_logs else MACRO_RESULT_PATTERN
         )
-        if capture_output and result.output is not None:
-            for log in parse_dbt_output(result.output):
-                if log_errors and log.level == "error":
-                    logger.error(log.msg)
-                    continue
-
-                if log.msg:
-                    match = log_pattern.match(log.msg)
-                    if match:
-                        run_operation_results.append(match.group(1))
+        if capture_output:
+            if result.output is not None:
+                for log in parse_dbt_output(result.output):
+                    if log_errors and log.level == "error":
+                        logger.error(log.msg)
+                        continue
+
+                    if log.msg:
+                        match = log_pattern.match(log.msg)
+                        if match:
+                            run_operation_results.append(match.group(1))
+
+            if result.stderr is not None and log_errors:
+                for log in parse_dbt_output(result.stderr):
+                    if log.level == "error":
+                        logger.error(log.msg)
+                        continue
 
         return run_operation_results
 
     def run(
         self,
-        models: Optional[str] = None,
         select: Optional[str] = None,
         selector: Optional[str] = None,
         full_refresh: bool = False,
@@ -216,8 +223,6 @@ def run(
         command_args = ["run"]
         if full_refresh:
             command_args.append("--full-refresh")
-        if models:
-            command_args.extend(["-m", models])
         if select:
             command_args.extend(["-s", select])
         if selector:

elementary/clients/dbt/dbt_fusion_runner.py

@@ -0,0 +1,14 @@
+import os
+
+from elementary.clients.dbt.subprocess_dbt_runner import SubprocessDbtRunner
+
+DBT_FUSION_PATH = os.getenv("DBT_FUSION_PATH", "~/.local/bin/dbt")
+
+
+class DbtFusionRunner(SubprocessDbtRunner):
+    def _get_dbt_command_name(self) -> str:
+        return os.path.expanduser(DBT_FUSION_PATH)
+
+    def _run_deps_if_needed(self):
+        # Currently we don't support auto-updating deps for dbt fusion
+        return