perf: optimize CI by skipping redundant dbt test artifacts upload and adding schema cleanup

- Add --vars to dbt test step to disable artifact autoupload (already done during dbt run)
  and skip per-table temp table cleanup (schema will be dropped at end)
- Add drop_test_schemas macro and CI step to drop both main and elementary schemas after tests
- This significantly reduces on-run-end hook time, especially for Athena where each query
  has high overhead (~3-5s) and the hook was issuing 100+ individual queries

Co-Authored-By: Itamar Hartstein <haritamar@gmail.com>

Author: devin-ai-integration[bot]

.github/workflows/test-warehouse.yml

@@ -192,7 +192,7 @@ jobs:
         working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
         continue-on-error: true
         run: |
-          dbt test --target "${{ inputs.warehouse-type }}"
+          dbt test --target "${{ inputs.warehouse-type }}" --vars '{disable_dbt_artifacts_autoupload: true, clean_elementary_temp_tables: false}'
 
       - name: Run help
         run: edr --help
@@ -277,3 +277,10 @@ jobs:
 
       - name: Run Python package e2e tests
         run: pytest -vv tests/e2e --warehouse-type ${{ inputs.warehouse-type }}
+
+      - name: Drop test schemas
+        if: always()
+        working-directory: ${{ env.E2E_DBT_PROJECT_DIR }}
+        continue-on-error: true
+        run: |
+          dbt run-operation elementary_tests.drop_test_schemas --target "${{ inputs.warehouse-type }}"

tests/e2e_dbt_project/macros/system/drop_test_schemas.sql

@@ -0,0 +1,35 @@
+{% macro drop_test_schemas() %}
+    {# Drop both the main test schema and the elementary schema used by the CLI.
+       The schema names are derived from the profile's target schema. #}
+    {% set main_schema = target.schema %}
+    {% set elementary_schema = main_schema ~ '_elementary' %}
+
+    {% do elementary_tests.edr_drop_schema(elementary_schema) %}
+    {% do elementary_tests.edr_drop_schema(main_schema) %}
+    {% do log("Dropped schemas: " ~ main_schema ~ ", " ~ elementary_schema, info=true) %}
+{% endmacro %}
+
+{% macro edr_drop_schema(schema_name) %}
+    {% do return(adapter.dispatch('edr_drop_schema', 'elementary_tests')(schema_name)) %}
+{% endmacro %}
+
+{% macro default__edr_drop_schema(schema_name) %}
+    {% set schema_relation = api.Relation.create(database=target.database, schema=schema_name) %}
+    {% do dbt.drop_schema(schema_relation) %}
+    {% do adapter.commit() %}
+{% endmacro %}
+
+{% macro bigquery__edr_drop_schema(schema_name) %}
+    {% set schema_relation = api.Relation.create(database=target.database, schema=schema_name) %}
+    {% do dbt.drop_schema(schema_relation) %}
+{% endmacro %}
+
+{% macro clickhouse__edr_drop_schema(schema_name) %}
+    {% do run_query("DROP DATABASE IF EXISTS " ~ schema_name) %}
+    {% do adapter.commit() %}
+{% endmacro %}
+
+{% macro athena__edr_drop_schema(schema_name) %}
+    {% set schema_relation = api.Relation.create(database=target.database, schema=schema_name) %}
+    {% do dbt.drop_schema(schema_relation) %}
+{% endmacro %}