Merge remote-tracking branch 'origin/master' into devin/CORE-93-1763039329

devin-ai-integration[bot] · devin-ai-integration[bot] · commit b4de5122f7db · 2025-11-20T13:13:26.000Z
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 name: "elementary"
 version: "0.20.1"
 
-require-dbt-version: [">=1.0.0", "<2.0.0"]
+require-dbt-version: [">=1.0.0", "<3.0.0"]
 
 config-version: 2
 profile: "elementary"
diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt
@@ -3,4 +3,6 @@ pytest-xdist
 pytest-parametrization
 pytest-html
 filelock
-urllib3==2.0.6
+# urllib3>=2.2.2 fixes CVE-2023-45803 and CVE-2024-37891
+# Upper bound <3.0.0 prevents breaking changes from future major versions
+urllib3>=2.2.2,<3.0.0
diff --git a/integration_tests/tests/test_column_anomalies.py b/integration_tests/tests/test_column_anomalies.py
@@ -476,3 +476,105 @@ def test_anomalous_boolean_column_anomalies(test_id: str, dbt_project: DbtProjec
         "count_true",
         "count_false",
     }
+
+
+# Anomalies currently not supported on ClickHouse
+@pytest.mark.skip_targets(["clickhouse"])
+def test_col_anom_excl_detect_train(test_id: str, dbt_project: DbtProject):
+    """
+    Test the exclude_detection_period_from_training flag functionality for column anomalies.
+
+    Scenario:
+    - 30 days of normal data with low null count (0-2 nulls per day)
+    - 7 days of anomalous data with high null count (20 nulls per day) in detection period
+    - Without exclusion: anomaly gets included in training baseline, test passes (misses anomaly)
+    - With exclusion: anomaly excluded from training, test fails (detects anomaly)
+    """
+    utc_today = datetime.utcnow().date()
+
+    # Generate 30 days of normal data with variance in null count (8, 10, 12 pattern)
+    normal_pattern = [8, 10, 12]
+    normal_data = []
+    for i in range(30):
+        date = utc_today - timedelta(days=37 - i)
+        null_count = normal_pattern[i % 3]
+        normal_data.extend(
+            [
+                {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), "superhero": superhero}
+                for superhero in ["Superman", "Batman", "Wonder Woman", "Flash"] * 10
+            ]
+        )
+        normal_data.extend(
+            [
+                {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), "superhero": None}
+                for _ in range(null_count)
+            ]
+        )
+
+    # Generate 7 days of anomalous data (20 nulls per day) - 100% increase from mean
+    anomalous_data = []
+    for i in range(7):
+        date = utc_today - timedelta(days=7 - i)
+        anomalous_data.extend(
+            [
+                {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), "superhero": superhero}
+                for superhero in ["Superman", "Batman", "Wonder Woman", "Flash"] * 10
+            ]
+        )
+        anomalous_data.extend(
+            [
+                {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT), "superhero": None}
+                for _ in range(20)
+            ]
+        )
+
+    all_data = normal_data + anomalous_data
+
+    # Test 1: WITHOUT exclusion (should pass - misses the anomaly because it's included in training)
+    test_args_without_exclusion = {
+        "timestamp_column": TIMESTAMP_COLUMN,
+        "column_anomalies": ["null_count"],
+        "time_bucket": {"period": "day", "count": 1},
+        "training_period": {"period": "day", "count": 30},
+        "detection_period": {"period": "day", "count": 7},
+        "min_training_set_size": 5,
+        "anomaly_sensitivity": 5,
+        "anomaly_direction": "spike",
+        "exclude_detection_period_from_training": False,
+    }
+
+    test_result_without_exclusion = dbt_project.test(
+        test_id + "_f",
+        DBT_TEST_NAME,
+        test_args_without_exclusion,
+        data=all_data,
+        test_column="superhero",
+        test_vars={"force_metrics_backfill": True},
+    )
+
+    # This should PASS because the anomaly is included in training, making it part of the baseline
+    assert test_result_without_exclusion["status"] == "pass", (
+        "Expected PASS when exclude_detection_period_from_training=False "
+        "(detection data included in training baseline)"
+    )
+
+    # Test 2: WITH exclusion (should fail - detects the anomaly because it's excluded from training)
+    test_args_with_exclusion = {
+        **test_args_without_exclusion,
+        "exclude_detection_period_from_training": True,
+    }
+
+    test_result_with_exclusion = dbt_project.test(
+        test_id + "_t",
+        DBT_TEST_NAME,
+        test_args_with_exclusion,
+        data=all_data,
+        test_column="superhero",
+        test_vars={"force_metrics_backfill": True},
+    )
+
+    # This should FAIL because the anomaly is excluded from training, so it's detected as anomalous
+    assert test_result_with_exclusion["status"] == "fail", (
+        "Expected FAIL when exclude_detection_period_from_training=True "
+        "(detection data excluded from training baseline, anomaly detected)"
+    )
diff --git a/integration_tests/tests/test_event_freshness_anomalies.py b/integration_tests/tests/test_event_freshness_anomalies.py
@@ -88,3 +88,105 @@ def test_slower_rate_event_freshness(test_id: str, dbt_project: DbtProject):
         test_vars={"custom_run_started_at": test_started_at.isoformat()},
     )
     assert result["status"] == "fail"
+
+
+# Anomalies currently not supported on ClickHouse
+@pytest.mark.skip_targets(["clickhouse"])
+def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject):
+    """
+    Test the exclude_detection_period_from_training flag functionality for event freshness anomalies.
+
+    Scenario:
+    - 7 days of normal data (5 minute lag between event and update) - training period
+    - 7 days of anomalous data (5 hour lag) - detection period
+    - Without exclusion: anomaly gets included in training baseline, test passes (misses anomaly)
+    - With exclusion: anomaly excluded from training, test fails (detects anomaly)
+
+    """
+    utc_now = datetime.utcnow()
+    test_started_at = (utc_now + timedelta(days=1)).replace(
+        hour=0, minute=0, second=0, microsecond=0
+    )
+
+    # Generate 7 days of normal data with varying lag (2-8 minutes) to ensure training_stddev > 0
+    training_lags_minutes = [2, 3, 4, 5, 6, 7, 8]
+    normal_data = []
+    for i in range(7):
+        event_date = test_started_at - timedelta(days=14 - i)
+        event_time = event_date.replace(hour=12, minute=0, second=0, microsecond=0)
+        update_time = event_time + timedelta(minutes=training_lags_minutes[i])
+        normal_data.append(
+            {
+                EVENT_TIMESTAMP_COLUMN: event_time.strftime(DATE_FORMAT),
+                UPDATE_TIMESTAMP_COLUMN: update_time.strftime(DATE_FORMAT),
+            }
+        )
+
+    # Generate 7 days of anomalous data with 5-hour lag (detection period)
+    anomalous_data = []
+    for i in range(7):
+        event_date = test_started_at - timedelta(days=7 - i)
+        event_time = event_date.replace(hour=12, minute=0, second=0, microsecond=0)
+        update_time = event_time + timedelta(hours=5)
+        anomalous_data.append(
+            {
+                EVENT_TIMESTAMP_COLUMN: event_time.strftime(DATE_FORMAT),
+                UPDATE_TIMESTAMP_COLUMN: update_time.strftime(DATE_FORMAT),
+            }
+        )
+
+    all_data = normal_data + anomalous_data
+
+    # Test 1: WITHOUT exclusion (should pass - misses the anomaly because it's included in training)
+    test_args_without_exclusion = {
+        "event_timestamp_column": EVENT_TIMESTAMP_COLUMN,
+        "update_timestamp_column": UPDATE_TIMESTAMP_COLUMN,
+        "days_back": 14,  # Scoring window: 14 days to include both training and detection
+        "backfill_days": 7,  # Detection period: last 7 days (days 7-1 before test_started_at)
+        "time_bucket": {
+            "period": "day",
+            "count": 1,
+        },  # Daily buckets to avoid boundary issues
+        "sensitivity": 3,
+        "anomaly_direction": "spike",  # Explicit direction since we're testing increased lag
+        "min_training_set_size": 5,  # Explicit minimum to avoid threshold issues
+        # exclude_detection_period_from_training is not set (defaults to False/None)
+    }
+
+    test_result_without_exclusion = dbt_project.test(
+        test_id + "_without_exclusion",
+        TEST_NAME,
+        test_args_without_exclusion,
+        data=all_data,
+        test_vars={
+            "custom_run_started_at": test_started_at.isoformat(),
+            "force_metrics_backfill": True,
+        },
+    )
+
+    # This should PASS because the anomaly is included in training, making it part of the baseline
+    assert (
+        test_result_without_exclusion["status"] == "pass"
+    ), "Test should pass when anomaly is included in training"
+
+    # Test 2: WITH exclusion (should fail - detects the anomaly because it's excluded from training)
+    test_args_with_exclusion = {
+        **test_args_without_exclusion,
+        "exclude_detection_period_from_training": True,
+    }
+
+    test_result_with_exclusion = dbt_project.test(
+        test_id + "_with_exclusion",
+        TEST_NAME,
+        test_args_with_exclusion,
+        data=all_data,
+        test_vars={
+            "custom_run_started_at": test_started_at.isoformat(),
+            "force_metrics_backfill": True,
+        },
+    )
+
+    # This should FAIL because the anomaly is excluded from training, so it's detected as anomalous
+    assert (
+        test_result_with_exclusion["status"] == "fail"
+    ), "Test should fail when anomaly is excluded from training"
diff --git a/integration_tests/tests/test_freshness_anomalies.py b/integration_tests/tests/test_freshness_anomalies.py
@@ -233,3 +233,88 @@ def test_first_metric_null(test_id, dbt_project: DbtProject):
             materialization="incremental",
         )
         assert result["status"] == "pass"
+
+
+@pytest.mark.skip_targets(["clickhouse"])
+def test_exclude_detection_from_training(test_id: str, dbt_project: DbtProject):
+    """
+    Test exclude_detection_period_from_training flag for freshness anomalies.
+
+    Data: 7 days normal (frequent updates, days -14 to -8) + 7 days anomalous (1 update/day, days -7 to -1)
+    Without exclusion: anomalous data in training baseline → test passes
+    With exclusion: anomalous data excluded from training → test fails
+    """
+    utc_now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
+
+    # Generate 7 days of normal data with frequent updates (every 2 hours) from day -14 to day -8
+    normal_data = [
+        {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT)}
+        for date in generate_dates(
+            base_date=utc_now - timedelta(days=8),
+            step=timedelta(hours=2),
+            days_back=7,
+        )
+    ]
+
+    # Generate 7 days of anomalous data (only 1 update per day at noon) from day -7 to day -1
+    anomalous_data = [
+        {TIMESTAMP_COLUMN: date.strftime(DATE_FORMAT)}
+        for date in generate_dates(
+            base_date=(utc_now - timedelta(days=1)).replace(hour=12, minute=0),
+            step=timedelta(hours=24),
+            days_back=7,
+        )
+    ]
+
+    all_data = normal_data + anomalous_data
+
+    # Test 1: WITHOUT exclusion (should pass - training includes detection window with anomalous pattern)
+    test_args_without_exclusion = {
+        "timestamp_column": TIMESTAMP_COLUMN,
+        "training_period": {"period": "day", "count": 14},
+        "detection_period": {"period": "day", "count": 7},
+        "time_bucket": {"period": "day", "count": 1},
+        "days_back": 20,
+        "backfill_days": 0,
+        "sensitivity": 3,
+        "min_training_set_size": 3,
+        "anomaly_direction": "spike",
+        "ignore_small_changes": {
+            "spike_failure_percent_threshold": 0,
+            "drop_failure_percent_threshold": 0,
+        },
+    }
+
+    detection_end = utc_now
+
+    test_result_without_exclusion = dbt_project.test(
+        test_id + "_without_exclusion",
+        TEST_NAME,
+        test_args_without_exclusion,
+        data=all_data,
+        test_vars={"custom_run_started_at": detection_end.isoformat()},
+    )
+
+    # This should PASS because the anomaly is included in training, making it part of the baseline
+    assert (
+        test_result_without_exclusion["status"] == "pass"
+    ), "Test should pass when anomaly is included in training"
+
+    # Test 2: WITH exclusion (should fail - detects the anomaly because it's excluded from training)
+    test_args_with_exclusion = {
+        **test_args_without_exclusion,
+        "exclude_detection_period_from_training": True,
+    }
+
+    test_result_with_exclusion = dbt_project.test(
+        test_id + "_with_exclusion",
+        TEST_NAME,
+        test_args_with_exclusion,
+        data=all_data,
+        test_vars={"custom_run_started_at": detection_end.isoformat()},
+    )
+
+    # This should FAIL because the anomaly is excluded from training, so it's detected as anomalous
+    assert (
+        test_result_with_exclusion["status"] == "fail"
+    ), "Test should fail when anomaly is excluded from training"
diff --git a/macros/edr/tests/test_event_freshness_anomalies.sql b/macros/edr/tests/test_event_freshness_anomalies.sql
@@ -1,4 +1,4 @@
-{% test event_freshness_anomalies(model, event_timestamp_column, update_timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period) %}
+{% test event_freshness_anomalies(model, event_timestamp_column, update_timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period, exclude_detection_period_from_training=false) %}
   {{ config(tags = ['elementary-tests']) }}
   {% if execute and elementary.is_test_command() and elementary.is_elementary_enabled() %}
     {% set model_relation = elementary.get_model_relation_for_test(model, elementary.get_test_model()) %}
@@ -32,7 +32,8 @@
       detection_delay=detection_delay,
       anomaly_exclude_metrics=anomaly_exclude_metrics,
       detection_period=detection_period,
-      training_period=training_period
+      training_period=training_period,
+      exclude_detection_period_from_training=exclude_detection_period_from_training
     )
   }}
 {% endtest %}
diff --git a/macros/edr/tests/test_freshness_anomalies.sql b/macros/edr/tests/test_freshness_anomalies.sql
@@ -1,4 +1,4 @@
-{% test freshness_anomalies(model, timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period) %}
+{% test freshness_anomalies(model, timestamp_column, where_expression, anomaly_sensitivity, anomaly_direction, min_training_set_size, time_bucket, days_back, backfill_days, seasonality, sensitivity, ignore_small_changes, detection_delay, anomaly_exclude_metrics, detection_period, training_period, exclude_detection_period_from_training=false) %}
   {{ config(tags = ['elementary-tests']) }}
   {{ elementary.test_table_anomalies(
       model=model,
@@ -18,7 +18,8 @@
       detection_delay=detection_delay,
       anomaly_exclude_metrics=anomaly_exclude_metrics,
       detection_period=detection_period,
-      training_period=training_period
+      training_period=training_period,
+      exclude_detection_period_from_training=exclude_detection_period_from_training
     )
   }}
 {% endtest %}
diff --git a/macros/utils/cross_db_utils/get_query_settings.sql b/macros/utils/cross_db_utils/get_query_settings.sql
@@ -0,0 +1,12 @@
+{% macro get_query_settings() %}
+  {% do return(adapter.dispatch("get_query_settings", "elementary")()) %}
+{% endmacro %}
+
+{% macro default__get_query_settings() %}
+  {% do return("") %}
+{% endmacro %}
+
+{% macro clickhouse__get_query_settings() %}
+  {% do return(adapter.get_model_query_settings(model)) %}
+{% endmacro %}
+
diff --git a/macros/utils/table_operations/delete_and_insert.sql b/macros/utils/table_operations/delete_and_insert.sql
@@ -61,14 +61,15 @@
         {% set delete_query %}
             alter table {{ relation }} delete where
             {{ delete_column_key }} is null
-            or {{ delete_column_key }} in (select {{ delete_column_key }} from {{ delete_relation }});
+            or {{ delete_column_key }} in (select {{ delete_column_key }} from {{ delete_relation }})
+            {{ adapter.get_model_query_settings(model) }};
         {% endset %}
         {% do queries.append(delete_query) %}
     {% endif %}
 
     {% if insert_relation %}
         {% set insert_query %}
-            insert into {{ relation }} select * from {{ insert_relation }};
+            insert into {{ relation }} {{ adapter.get_model_query_settings(model) }} select * from {{ insert_relation }};
         {% endset %}
         {% do queries.append(insert_query) %}
     {% endif %}
diff --git a/macros/utils/table_operations/insert_rows.sql b/macros/utils/table_operations/insert_rows.sql
@@ -63,7 +63,9 @@
        insert into {{ table_relation }}
          ({%- for column in columns -%}
            {{- elementary.escape_reserved_keywords(column.name) -}} {{- "," if not loop.last else "" -}}
-         {%- endfor -%}) values
+         {%- endfor -%})
+         {{ elementary.get_query_settings() }}
+         values
     {% endset %}
     {% do elementary.end_duration_measure_context('base_query_calc') %}
 
diff --git a/macros/utils/table_operations/make_temp_relation.sql b/macros/utils/table_operations/make_temp_relation.sql
