APP-1123: add disable_samples to skip test_result_rows query

Thread a new disable_samples parameter from ReportAPI.get_report_data
through TestsAPI -> TestsFetcher -> the get_test_results dbt macro.
When disable_samples=true, the macro skips the get_result_rows_agate
query against test_result_rows entirely instead of relying on the
caller to clear sample_data after the SQL has already run.

This is consumed by elementary-internal's cloud report generation,
where results_sample has zero consumers (522 GB / 249.6M rows scanned
per cycle for nothing). The default remains false so OSS users and
other callers are unaffected.

Co-Authored-By: mika@elementary-data.com <mika.kerman@gmail.com>

Author: devin-ai-integration[bot]

elementary/monitor/api/report/report.py

@@ -86,6 +86,7 @@ def get_report_data(
                 days_back=days_back,
                 invocations_per_test=test_runs_amount,
                 disable_passed_test_metrics=disable_passed_test_metrics,
+                disable_samples=disable_samples,
             )
             source_freshnesses_api = SourceFreshnessesAPI(
                 dbt_runner=self.dbt_runner,

elementary/monitor/api/tests/tests.py

@@ -46,25 +46,29 @@ def __init__(
         days_back: int = 7,
         invocations_per_test: int = 720,
         disable_passed_test_metrics: bool = False,
+        disable_samples: bool = False,
     ):
         super().__init__(dbt_runner)
         self.tests_fetcher = TestsFetcher(dbt_runner=self.dbt_runner)
         self.test_results_db_rows = self._get_test_results_db_rows(
             days_back=days_back,
             invocations_per_test=invocations_per_test,
             disable_passed_test_metrics=disable_passed_test_metrics,
+            disable_samples=disable_samples,
         )
 
     def _get_test_results_db_rows(
         self,
         days_back: Optional[int] = 7,
         invocations_per_test: int = 720,
         disable_passed_test_metrics: bool = False,
+        disable_samples: bool = False,
     ) -> List[TestResultDBRowSchema]:
         return self.tests_fetcher.get_all_test_results_db_rows(
             days_back=days_back,
             invocations_per_test=invocations_per_test,
             disable_passed_test_metrics=disable_passed_test_metrics,
+            disable_samples=disable_samples,
         )
 
     def get_test_results_summary(

elementary/monitor/dbt_project/macros/get_test_results.sql

@@ -1,5 +1,5 @@
-{%- macro get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%}
-    {{ return(adapter.dispatch('get_test_results', 'elementary_cli')(days_back, invocations_per_test, disable_passed_test_metrics)) }}
+{%- macro get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false, disable_samples = false) -%}
+    {{ return(adapter.dispatch('get_test_results', 'elementary_cli')(days_back, invocations_per_test, disable_passed_test_metrics, disable_samples)) }}
 {%- endmacro -%}
 
 {#
@@ -40,7 +40,7 @@
     {% do return(test_results) %}
 {%- endmacro -%}
 
-{%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%}
+{%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false, disable_samples = false) -%}
     {% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass']  %}
     {% set select_test_results %}
         with test_results as (
@@ -111,15 +111,19 @@
     {% endset %}
 
     {% set test_results_agate = elementary.run_query(test_results_agate_sql) %}
-    {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %}
+    {% if not disable_samples %}
+        {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %}
+    {% else %}
+        {% set test_result_rows_agate = {} %}
+    {% endif %}
     {% if not elementary.has_temp_table_support() %}
         {% do elementary.fully_drop_relation(ordered_test_results_relation) %}
     {% endif %}
 
     {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %}
 {%- endmacro -%}
 
-{%- macro fabric__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%}
+{%- macro fabric__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false, disable_samples = false) -%}
     {#
         T-SQL does not allow nested CTEs (WITH inside WITH).
         current_tests_run_results_query already starts with WITH, so we
@@ -165,7 +169,11 @@
     {% endset %}
 
     {% set test_results_agate = elementary.run_query(test_results_agate_sql) %}
-    {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %}
+    {% if not disable_samples %}
+        {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %}
+    {% else %}
+        {% set test_result_rows_agate = {} %}
+    {% endif %}
 
     {# Clean up intermediate tables #}
     {% do elementary.fully_drop_relation(base_relation) %}
@@ -174,7 +182,7 @@
     {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %}
 {%- endmacro -%}
 
-{%- macro clickhouse__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%}
+{%- macro clickhouse__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false, disable_samples = false) -%}
     {% do elementary.run_query('drop table if exists ordered_test_results') %}
     {% set create_table_query %}
     CREATE TABLE ordered_test_results (
@@ -302,7 +310,11 @@
     {% endset %}
 
     {% set test_results_agate = elementary.run_query(test_results_agate_sql) %}
-    {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %}
+    {% if not disable_samples %}
+        {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %}
+    {% else %}
+        {% set test_result_rows_agate = {} %}
+    {% endif %}
     {% if not elementary.has_temp_table_support() %}
         {% do elementary.fully_drop_relation(ordered_test_results_relation) %}
     {% endif %}

elementary/monitor/fetchers/tests/tests.py

@@ -22,13 +22,15 @@ def get_all_test_results_db_rows(
         days_back: Optional[int] = 7,
         invocations_per_test: int = 720,
         disable_passed_test_metrics: bool = False,
+        disable_samples: bool = False,
     ) -> List[TestResultDBRowSchema]:
         run_operation_response = self.dbt_runner.run_operation(
             macro_name="elementary_cli.get_test_results",
             macro_args=dict(
                 days_back=days_back,
                 invocations_per_test=invocations_per_test,
                 disable_passed_test_metrics=disable_passed_test_metrics,
+                disable_samples=disable_samples,
             ),
         )
         test_results = (