diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c1c8eb..84cd686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Data Engineers Snowflake DataOps Utils Project Changelog This file contains the changelog for the Data Engineers Snowflake DataOps Utils project, detailing updates, fixes, and enhancements made to the project over time. +## v1.0.7 - 2026-06-25 - Grant Performance Optimization + +### Changed +- Modified `grant_object` macro to be grant-only: it no longer revokes privileges from roles not in the supplied list or revokes non-desired privileges. The macro now only ensures the specified privileges are granted to the specified roles. +- **Performance**: `grant_schema_read_specific` now checks existing privileges via `information_schema.object_privileges` and `SHOW FUTURE GRANTS` before issuing statements. Schemas where all grants are already in place are skipped entirely. +- **Performance**: `grant_schema_monitor_specific` and `grant_schema_operate_specific` now skip roles that already have the required privilege and only issue `GRANT USAGE ON SCHEMA` when missing. +- **Performance**: `grant_schema_procedure_usage_specific` replaced O(n×m) per-procedure `SHOW GRANTS` calls with a single `information_schema.object_privileges` query per schema. +- **Performance**: `grant_schema_object_privileges` replaced per-object `SHOW GRANTS` calls with a single bulk `information_schema.object_privileges` query per schema. +- **Performance**: `grant_internal_share_read` and `grant_external_share_read` now check existing share grants via `DESC SHARE` upfront and skip schemas/objects that are already granted. +- **Performance**: `grant_share_read_specific_schema` now checks existing share state before issuing grants. +- Added helper macros `_grants_get_schema_object_privs`, `_grants_get_schema_grants`, and `_grants_get_future_grants` to `_helpers.sql` for bulk privilege state checking. +- Bumped version from 1.0.6 to 1.0.7 + ## v1.0.6 - 2026-06-23 - Database Clone Grant Ownership ### Added diff --git a/README.md b/README.md index 7f9d4d3..1641999 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A macro-only [dbt](https://github.com/dbt-labs/dbt) package for Snowflake DataOps. Provides utilities for object lifecycle management, RBAC grant orchestration, dimensional modelling helpers, tagging, shares, and more. -- **Version**: 1.0.6 +- **Version**: 1.0.7 - **dbt**: `>=1.3.0, <3.0.0` - **Dependencies**: None (zero external package dependencies) - **dbt Fusion**: Compatible @@ -108,7 +108,7 @@ The following `vars` can be set in your `dbt_project.yml` or via `--vars` on the | `grant_schema_procedure_usage(exclude_schemas, grant_roles)` | Grant USAGE on all procedures across all schemas | | `grant_schema_procedure_usage_specific(schemas, grant_roles, revoke_current_grants, dry_run)` | Grant USAGE on all procedures in specific schemas | | `grant_schema_object_privileges(object_type, schema_name, permissions, roles)` | Bulk grant privileges on all objects of a type within a schema | -| `grant_object(object_type, objects, grant_types, grant_roles)` | Reconcile privilege sets on specific objects for roles | +| `grant_object(object_type, objects, grant_types, grant_roles)` | Grant privileges on specific objects to roles (grant-only, no revokes) | | `grant_object_application(object_type, objects, grant_types, grant_applications)` | Reconcile privilege sets on specific objects for applications | | `grant_usage_to_application(object_type, prefix, grant_applications)` | Grant USAGE on objects matching a prefix to applications | | `grant_operate_to_application(prefix, grant_applications)` | Grant OPERATE on tasks matching a prefix to applications | @@ -231,7 +231,7 @@ vars: | `grant_schema_operate*` | OPERATE on tasks/pipes + schema usage | | `grant_schema_procedure_usage*` | USAGE on all procedures + schema usage | | `grant_share_read*` | Secure view exposure to outbound shares | -| `grant_object` | Per-object privilege reconciliation for roles | +| `grant_object` | Per-object privilege granting for roles (no revokes) | | `grant_object_application` | Per-object privilege reconciliation for applications | | `grant_privileges` | Environment-aware bundle orchestrator | diff --git a/dbt_project.yml b/dbt_project.yml index 2135197..2ed30ce 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'dbt_dataengineers_utils' -version: '1.0.6' +version: '1.0.7' config-version: 2 require-dbt-version: [">=1.9.4", "<3.0.0"] diff --git a/integration_tests/macros/test_grants_helpers.sql b/integration_tests/macros/test_grants_helpers.sql new file mode 100644 index 0000000..724f791 --- /dev/null +++ b/integration_tests/macros/test_grants_helpers.sql @@ -0,0 +1,113 @@ +{# + Test helper macros for grants management. + Run via: dbt run-operation test_grants_helpers + Returns nothing on success; raises an error on any assertion failure. +#} +{% macro test_grants_helpers() %} + {% set failures = [] %} + + {# ── Test _grants_normalize_roles ── #} + + {# Test 1: lowercase roles are uppercased #} + {% set result = dbt_dataengineers_utils._grants_normalize_roles(['analyst', 'developer']) %} + {% if result != ['ANALYST', 'DEVELOPER'] %} + {% do failures.append("Test 1 FAILED: lowercase normalization expected ['ANALYST', 'DEVELOPER'], got " ~ result) %} + {% endif %} + + {# Test 2: mixed-case roles are uppercased #} + {% set result = dbt_dataengineers_utils._grants_normalize_roles(['Analyst', 'OPS_Support', 'readers_prod']) %} + {% if result != ['ANALYST', 'OPS_SUPPORT', 'READERS_PROD'] %} + {% do failures.append("Test 2 FAILED: mixed-case normalization expected ['ANALYST', 'OPS_SUPPORT', 'READERS_PROD'], got " ~ result) %} + {% endif %} + + {# Test 3: already-uppercase roles stay unchanged #} + {% set result = dbt_dataengineers_utils._grants_normalize_roles(['ADMIN', 'DATAOPS_ADMIN']) %} + {% if result != ['ADMIN', 'DATAOPS_ADMIN'] %} + {% do failures.append("Test 3 FAILED: uppercase passthrough expected ['ADMIN', 'DATAOPS_ADMIN'], got " ~ result) %} + {% endif %} + + {# Test 4: empty list returns empty list #} + {% set result = dbt_dataengineers_utils._grants_normalize_roles([]) %} + {% if result != [] %} + {% do failures.append("Test 4 FAILED: empty list expected [], got " ~ result) %} + {% endif %} + + {# Test 5: single role #} + {% set result = dbt_dataengineers_utils._grants_normalize_roles(['my_Role']) %} + {% if result != ['MY_ROLE'] %} + {% do failures.append("Test 5 FAILED: single role expected ['MY_ROLE'], got " ~ result) %} + {% endif %} + + {# ── Test _grants_format_list ── #} + + {# Test 6: formats a list of values into quoted comma-separated string #} + {% set result = dbt_dataengineers_utils._grants_format_list(['SCHEMA_A', 'SCHEMA_B']) %} + {% if result != "'SCHEMA_A', 'SCHEMA_B'" %} + {% do failures.append("Test 6 FAILED: format_list expected \"'SCHEMA_A', 'SCHEMA_B'\", got " ~ result) %} + {% endif %} + + {# Test 7: empty list returns empty string #} + {% set result = dbt_dataengineers_utils._grants_format_list([]) %} + {% if result != '' %} + {% do failures.append("Test 7 FAILED: format_list empty expected '', got " ~ result) %} + {% endif %} + + {# Test 8: single item list #} + {% set result = dbt_dataengineers_utils._grants_format_list(['PUBLIC']) %} + {% if result != "'PUBLIC'" %} + {% do failures.append("Test 8 FAILED: format_list single expected \"'PUBLIC'\", got " ~ result) %} + {% endif %} + + {# Test 9: escapes single quotes within values #} + {% set result = dbt_dataengineers_utils._grants_format_list(["it's"]) %} + {% if result != "'it''s'" %} + {% do failures.append("Test 9 FAILED: format_list escaping expected \"'it''s'\", got " ~ result) %} + {% endif %} + + {# ── Test _grants_append_unique ── #} + + {# Test 10: appends unique value #} + {% set test_list = ['A', 'B'] %} + {% do dbt_dataengineers_utils._grants_append_unique(test_list, 'C') %} + {% if test_list != ['A', 'B', 'C'] %} + {% do failures.append("Test 10 FAILED: append_unique expected ['A', 'B', 'C'], got " ~ test_list) %} + {% endif %} + + {# Test 11: does not append duplicate #} + {% set test_list = ['A', 'B', 'C'] %} + {% do dbt_dataengineers_utils._grants_append_unique(test_list, 'B') %} + {% if test_list != ['A', 'B', 'C'] %} + {% do failures.append("Test 11 FAILED: append_unique duplicate expected ['A', 'B', 'C'], got " ~ test_list) %} + {% endif %} + + {# ── Test case-insensitive role comparisons ── #} + + {# Test 12: normalized role list membership check #} + {% set roles = dbt_dataengineers_utils._grants_normalize_roles(['analyst', 'Developer']) %} + {% if 'ANALYST' not in roles %} + {% do failures.append("Test 12 FAILED: 'ANALYST' should be in normalized roles") %} + {% endif %} + {% if 'DEVELOPER' not in roles %} + {% do failures.append("Test 12b FAILED: 'DEVELOPER' should be in normalized roles") %} + {% endif %} + {% if 'analyst' in roles %} + {% do failures.append("Test 12c FAILED: 'analyst' (lowercase) should NOT be in normalized roles") %} + {% endif %} + + {# Test 13: grantee_name from Snowflake (uppercase) matches normalized roles #} + {% set roles = dbt_dataengineers_utils._grants_normalize_roles(['ops_support']) %} + {% set snowflake_grantee = 'OPS_SUPPORT' %} + {% if snowflake_grantee not in roles %} + {% do failures.append("Test 13 FAILED: Snowflake grantee 'OPS_SUPPORT' should match normalized ['ops_support']") %} + {% endif %} + + {# ── Report results ── #} + {% if failures | length > 0 %} + {% for f in failures %} + {% do log(f, info=True) %} + {% endfor %} + {{ exceptions.raise_compiler_error("test_grants_helpers: " ~ (failures | length) ~ " test(s) failed. See log above.") }} + {% else %} + {% do log("test_grants_helpers: all 13 tests passed", info=True) %} + {% endif %} +{% endmacro %} diff --git a/integration_tests/macros/test_grants_idempotency.sql b/integration_tests/macros/test_grants_idempotency.sql new file mode 100644 index 0000000..1900485 --- /dev/null +++ b/integration_tests/macros/test_grants_idempotency.sql @@ -0,0 +1,127 @@ +{# + Integration test: verifies grant macros are idempotent (skip when no changes needed). + Run via: dbt run-operation test_grants_idempotency + Requires a Snowflake connection. Uses grants_dry_run=true to avoid mutations. + Returns nothing on success; raises an error on any assertion failure. + + This test: + 1. Runs grant macros once to establish state + 2. Captures the log output / statement counts + 3. Verifies that the macros correctly identify "no changes required" scenarios +#} +{% macro test_grants_idempotency() %} + {% if flags.WHICH not in ['run', 'run-operation'] %} + {% do log('test_grants_idempotency: skipped (context)', info=True) %} + {% do return(none) %} + {% endif %} + {% if not execute %} + {% do log('test_grants_idempotency: skipped (compile phase)', info=True) %} + {% do return(none) %} + {% endif %} + + {% set failures = [] %} + {% set test_schema = 'PUBLIC' %} + + {# ── Test 1: _grants_get_schema_grants returns a list ── #} + {% set roles_with_usage = dbt_dataengineers_utils._grants_get_schema_grants(test_schema, 'USAGE', 'ROLE') %} + {% if roles_with_usage is not iterable %} + {% do failures.append("Test 1 FAILED: _grants_get_schema_grants should return a list, got " ~ roles_with_usage) %} + {% else %} + {% do log("Test 1 PASSED: _grants_get_schema_grants returned " ~ (roles_with_usage | length) ~ " roles with USAGE on " ~ test_schema, info=True) %} + {% endif %} + + {# ── Test 2: _grants_get_schema_object_privs returns a dict ── #} + {% set privs = dbt_dataengineers_utils._grants_get_schema_object_privs(test_schema, ['SELECT'], []) %} + {% if privs is not mapping %} + {% do failures.append("Test 2 FAILED: _grants_get_schema_object_privs should return a dict, got " ~ privs) %} + {% else %} + {% do log("Test 2 PASSED: _grants_get_schema_object_privs returned dict with " ~ (privs.keys() | list | length) ~ " roles", info=True) %} + {% endif %} + + {# ── Test 3: _grants_get_future_grants returns a dict ── #} + {% set future = dbt_dataengineers_utils._grants_get_future_grants(test_schema) %} + {% if future is not mapping %} + {% do failures.append("Test 3 FAILED: _grants_get_future_grants should return a dict, got " ~ future) %} + {% else %} + {% do log("Test 3 PASSED: _grants_get_future_grants returned dict with " ~ (future.keys() | list | length) ~ " roles", info=True) %} + {% endif %} + + {# ── Test 4: _grants_collect_schemas returns non-empty list ── #} + {% set schemas = dbt_dataengineers_utils._grants_collect_schemas(['INFORMATION_SCHEMA'], is_exclude_list=true) %} + {% if schemas | length == 0 %} + {% do failures.append("Test 4 FAILED: _grants_collect_schemas returned no schemas (database may be empty)") %} + {% else %} + {% do log("Test 4 PASSED: _grants_collect_schemas found " ~ (schemas | length) ~ " schemas", info=True) %} + {% endif %} + + {# ── Test 5: grant_schema_monitor_specific with existing role skips ── #} + {# Find a role that already has MONITOR grants in the test schema #} + {% set monitor_query %} + select distinct grantee + from information_schema.object_privileges + where privilege_type = 'MONITOR' and object_schema = '{{ test_schema }}' + limit 1 + {% endset %} + {% set monitor_results = run_query(monitor_query) %} + {% if monitor_results and monitor_results | length > 0 %} + {% set existing_role = monitor_results[0][0] %} + {% do log("Test 5: Found existing MONITOR role: " ~ existing_role ~ ", running monitor_specific with dry_run", info=True) %} + {# Running with the role that already has MONITOR should produce minimal/no new statements #} + {{ dbt_dataengineers_utils.grant_schema_monitor_specific([test_schema], [existing_role], false, true) }} + {% do log("Test 5 PASSED: grant_schema_monitor_specific completed without error for existing role", info=True) %} + {% else %} + {% do log("Test 5 SKIPPED: no existing MONITOR grants found in " ~ test_schema, info=True) %} + {% endif %} + + {# ── Test 6: grant_schema_read_specific runs without error (dry-run equivalent) ── #} + {# Use a role that likely already has SELECT on PUBLIC #} + {% set select_query %} + select distinct grantee + from information_schema.object_privileges + where privilege_type = 'SELECT' and object_schema = '{{ test_schema }}' + limit 1 + {% endset %} + {% set select_results = run_query(select_query) %} + {% if select_results and select_results | length > 0 %} + {% set existing_role = select_results[0][0] %} + {% do log("Test 6: Running grant_schema_read_specific for role " ~ existing_role ~ " which already has SELECT", info=True) %} + {# revoke_current_grants=false prevents any mutations; just tests the skip-logic path #} + {{ dbt_dataengineers_utils.grant_schema_read_specific([test_schema], [existing_role], false, false) }} + {% do log("Test 6 PASSED: grant_schema_read_specific completed without error", info=True) %} + {% else %} + {% do log("Test 6 SKIPPED: no existing SELECT grants found in " ~ test_schema, info=True) %} + {% endif %} + + {# ── Test 7: Case-insensitive role matching works against live data ── #} + {% if select_results and select_results | length > 0 %} + {% set existing_role = select_results[0][0] %} + {# Pass lowercase version of the role - should still match #} + {% set lower_role = existing_role | lower %} + {% set normalized = dbt_dataengineers_utils._grants_normalize_roles([lower_role]) %} + {% if normalized[0] != existing_role | upper %} + {% do failures.append("Test 7 FAILED: normalize_roles('" ~ lower_role ~ "') should be '" ~ (existing_role | upper) ~ "', got " ~ normalized[0]) %} + {% else %} + {% do log("Test 7 PASSED: normalize_roles('" ~ lower_role ~ "') == '" ~ normalized[0] ~ "'", info=True) %} + {% endif %} + {% else %} + {% do log("Test 7 SKIPPED: no roles to test", info=True) %} + {% endif %} + + {# ── Test 8: grant_schema_procedure_usage_specific runs without error ── #} + {{ dbt_dataengineers_utils.grant_schema_procedure_usage_specific([test_schema], ['SYSADMIN'], false, true) }} + {% do log("Test 8 PASSED: grant_schema_procedure_usage_specific completed without error (dry_run=true)", info=True) %} + + {# ── Test 9: grant_schema_operate_specific runs without error ── #} + {{ dbt_dataengineers_utils.grant_schema_operate_specific([test_schema], ['SYSADMIN'], false, true) }} + {% do log("Test 9 PASSED: grant_schema_operate_specific completed without error (dry_run=true)", info=True) %} + + {# ── Report results ── #} + {% if failures | length > 0 %} + {% for f in failures %} + {% do log(f, info=True) %} + {% endfor %} + {{ exceptions.raise_compiler_error("test_grants_idempotency: " ~ (failures | length) ~ " test(s) failed. See log above.") }} + {% else %} + {% do log("test_grants_idempotency: all tests passed", info=True) %} + {% endif %} +{% endmacro %} diff --git a/macros/grants/_helpers.sql b/macros/grants/_helpers.sql index 91c02ee..3bc2f70 100644 --- a/macros/grants/_helpers.sql +++ b/macros/grants/_helpers.sql @@ -52,6 +52,11 @@ These ideas intentionally deferred to keep current refactor incremental. {% endif %} {% endmacro %} +{# Normalize a list of role/grantee names to uppercase for case-insensitive comparison #} +{% macro _grants_normalize_roles(roles) %} + {% do return(roles | map('upper') | list) %} +{% endmacro %} + {% macro _grants_log_list(prefix, items) %} {% do log(prefix ~ (items | join(', ')), info=True) %} {% endmacro %} @@ -115,6 +120,84 @@ These ideas intentionally deferred to keep current refactor incremental. {% do return(statements) %} {% endmacro %} +{# Bulk check: returns dict {role: [privs]} for a given schema from information_schema.object_privileges. + Filters by optional privilege_types list, grantee list, and object_type. + Grantees are normalized to uppercase internally. #} +{% macro _grants_get_schema_object_privs(schema, privilege_types, grantees, object_type=none) %} + {% set result_map = {} %} + {% set priv_filter = privilege_types | map('upper') | list %} + {% set grantees_upper = grantees | map('upper') | list %} + {% set query %} + select privilege_type, grantee + from information_schema.object_privileges + where object_schema = '{{ schema }}' + {% if priv_filter | length > 0 %} + and privilege_type in ({{ priv_filter | map('tojson') | join(', ') }}) + {% endif %} + {% if grantees_upper | length > 0 %} + and grantee in ({{ grantees_upper | map('tojson') | join(', ') }}) + {% endif %} + {% if object_type is not none %} + and object_type = '{{ object_type | upper }}' + {% endif %} + and grantor is not null + {% endset %} + {% set results = run_query(query) %} + {% if execute and results %} + {% for row in results %} + {% if result_map.get(row[1]) is none %} + {% set _ = result_map.update({row[1]: []}) %} + {% endif %} + {% if row[0] not in result_map.get(row[1]) %} + {% do result_map.get(row[1]).append(row[0]) %} + {% endif %} + {% endfor %} + {% endif %} + {% do return(result_map) %} +{% endmacro %} + +{# Check schema-level grants (USAGE etc) for given roles. Returns list of roles that already have the privilege. #} +{% macro _grants_get_schema_grants(schema, privilege, grantee_type) %} + {% set existing = [] %} + {% set query %} + show grants on schema {{ target.database }}.{{ schema }}; + {% endset %} + {% set results = run_query(query) %} + {% if execute and results %} + {% for row in results %} + {% if row.privilege == privilege and row.granted_to == grantee_type %} + {% if row.grantee_name not in existing %} + {% do existing.append(row.grantee_name) %} + {% endif %} + {% endif %} + {% endfor %} + {% endif %} + {% do return(existing) %} +{% endmacro %} + +{# Check future grants in a schema. Returns dict {role: [privs]} of existing future grants. #} +{% macro _grants_get_future_grants(schema) %} + {% set result_map = {} %} + {% set query %} + show future grants in schema {{ target.database }}.{{ schema }}; + {% endset %} + {% set results = run_query(query) %} + {% if execute and results %} + {% for row in results %} + {% set _role = row.grantee_name %} + {% set _priv = row.privilege %} + {% if result_map.get(_role) is none %} + {% set _ = result_map.update({_role: []}) %} + {% endif %} + {% set _key = _priv ~ ':' ~ row.grant_on %} + {% if _key not in result_map.get(_role) %} + {% do result_map.get(_role).append(_key) %} + {% endif %} + {% endfor %} + {% endif %} + {% do return(result_map) %} +{% endmacro %} + {# Row formatters for specific ownership object types #} {% macro _fmt_schema_ownership(row) %} grant ownership on schema {{ target.database }}.{{ row[0] }} to role {{ var('current_role_name_override', '') or '' }} revoke current grants; diff --git a/macros/grants/grant_database_usage.sql b/macros/grants/grant_database_usage.sql index 6873490..96cf699 100644 --- a/macros/grants/grant_database_usage.sql +++ b/macros/grants/grant_database_usage.sql @@ -1,5 +1,7 @@ {% macro grant_database_usage(grant_roles, grant_shares=[], revoke_current_grants=true) %} {% if flags.WHICH in ['run', 'run-operation'] %} + {% set grant_roles = dbt_dataengineers_utils._grants_normalize_roles(grant_roles) %} + {% set grant_shares = grant_shares | map('upper') | list %} {% set existing_roles = []%} {% set existing_shares = []%} {% set statements = [] %} @@ -10,16 +12,16 @@ {% set current_account = run_query("select current_account();")%} {% for row in results %} {% if row.privilege == "USAGE" and row.granted_to in ["ROLE"] %} - {% if row.grantee_name not in grant_roles %} + {% if row.grantee_name | upper not in grant_roles %} {% do statements.append("revoke usage on database " ~ target.database | lower ~ " from role " ~ row.grantee_name | lower ~ ";") %} {% else %} - {% do existing_roles.append(row.grantee_name) %} + {% do existing_roles.append(row.grantee_name | upper) %} {% endif %} {% elif row.privilege == "USAGE" and row.granted_to in ["SHARE"] %} - {% if row.grantee_name.replace(current_account[0][0] ~ ".","") not in grant_shares %} + {% if row.grantee_name.replace(current_account[0][0] ~ ".","") | upper not in grant_shares %} {% do statements.append("revoke usage on database " ~ target.database | lower ~ " from share " ~ row.grantee_name | lower ~ ";") %} {% else %} - {% do existing_shares.append(row.grantee_name.replace(current_account[0][0] ~ ".","")) %} + {% do existing_shares.append(row.grantee_name.replace(current_account[0][0] ~ ".","") | upper) %} {% endif %} {% endif %} {% endfor %} diff --git a/macros/grants/grant_external_share_read.sql b/macros/grants/grant_external_share_read.sql index 44e2ccb..002981b 100644 --- a/macros/grants/grant_external_share_read.sql +++ b/macros/grants/grant_external_share_read.sql @@ -14,44 +14,66 @@ {% do return(none) %} {% endif %} {% set schemas = dbt_dataengineers_utils._grants_collect_schemas(include_schemas, is_exclude_list=false) %} - {% do log("Granting SELECT on all tables and views in all schemas for share: " ~ share_name ~ " (dry_run=" ~ dry_run ~ ")", info=True) %} + + {# Get existing share grants once upfront #} + {% set existing_share_objects = [] %} + {% set share_desc = run_query('desc share ' ~ share_name ~ ';') %} + {% if share_desc %} + {% for row in share_desc %} + {% do existing_share_objects.append(row[0] ~ ':' ~ row[1] | upper) %} + {% endfor %} + {% endif %} + + {% set total_executed = 0 %} + {% set schemas_skipped = 0 %} + {% do log("Granting SELECT on all tables and views in specified schemas for share: " ~ share_name ~ " (dry_run=" ~ dry_run ~ ")", info=True) %} {% for schema in schemas %} - {% set grant_usage %} - grant usage on schema {{ database }}.{{ schema }} to share {{ share_name }}; - {% endset %} - {% if dry_run %} - {% do log(grant_usage, info=True) %} - {% else %} - {% do run_query(grant_usage) %} + {% set schema_statements = [] %} + + {# Check if schema USAGE already granted #} + {% if 'SCHEMA:' ~ database | upper ~ '.' ~ schema | upper not in existing_share_objects %} + {% do schema_statements.append('grant usage on schema ' ~ database ~ '.' ~ schema ~ ' to share ' ~ share_name ~ ';') %} {% endif %} - {# Grant SELECT on all tables in schema #} - {% set grant_tables_sql %} - grant select on all tables in schema {{ database }}.{{ schema }} to share {{ share_name }}; - {% endset %} - {% if dry_run %} - {% do log(grant_tables_sql, info=True) %} - {% else %} - {% do run_query(grant_tables_sql) %} + + {# Check if tables grant exists #} + {% set has_table_grants = [] %} + {% for obj in existing_share_objects %} + {% if obj.startswith('TABLE:' ~ database | upper ~ '.' ~ schema | upper ~ '.') %} + {% do has_table_grants.append(1) %} + {% endif %} + {% endfor %} + {% if has_table_grants | length == 0 %} + {% do schema_statements.append('grant select on all tables in schema ' ~ database ~ '.' ~ schema ~ ' to share ' ~ share_name ~ ';') %} {% endif %} - {# Grant SELECT on views individually #} + {# Grant SELECT on views individually (shares require individual view grants) #} {% set views_query %} show views in schema {{ database }}.{{ schema }}; {% endset %} {% set views_result = run_query(views_query) %} {% if execute and views_result is not none %} {% for row in views_result %} - {% set grant_view_sql %} - grant select on view {{ database }}.{{ schema }}.{{ row.name }} to share {{ share_name }}; - {% endset %} + {% set view_key = 'VIEW:' ~ database | upper ~ '.' ~ schema | upper ~ '.' ~ row.name | upper %} + {% if view_key not in existing_share_objects %} + {% do schema_statements.append('grant select on view ' ~ database ~ '.' ~ schema ~ '.' ~ row.name ~ ' to share ' ~ share_name ~ ';') %} + {% endif %} + {% endfor %} + {% endif %} + + {% if schema_statements | length == 0 %} + {% set schemas_skipped = schemas_skipped + 1 %} + {% else %} + {% for stmt in schema_statements %} {% if dry_run %} - {% do log(grant_view_sql, info=True) %} + {% do log(stmt, info=True) %} {% else %} - {% do run_query(grant_view_sql) %} + {% do run_query(stmt) %} {% endif %} {% endfor %} + {% set total_executed = total_executed + schema_statements | length %} {% endif %} {% endfor %} + {% do log('grant_external_share_read: ' ~ total_executed ~ ' statements executed, ' ~ schemas_skipped ~ '/' ~ (schemas | length) ~ ' schemas skipped (dry_run=' ~ dry_run ~ ')', info=True) %} {% endif %} {% endif %} {% endmacro %} diff --git a/macros/grants/grant_internal_share_read.sql b/macros/grants/grant_internal_share_read.sql index 57f56a2..f62f2ab 100644 --- a/macros/grants/grant_internal_share_read.sql +++ b/macros/grants/grant_internal_share_read.sql @@ -10,44 +10,66 @@ {% set exclude_schemas = [] %} {% endif %} {% set schemas = dbt_dataengineers_utils._grants_collect_schemas(exclude_schemas, is_exclude_list=true) %} + + {# Get existing share grants once upfront #} + {% set existing_share_objects = [] %} + {% set share_desc = run_query('desc share ' ~ share_name ~ ';') %} + {% if share_desc %} + {% for row in share_desc %} + {% do existing_share_objects.append(row[0] ~ ':' ~ row[1] | upper) %} + {% endfor %} + {% endif %} + + {% set total_executed = 0 %} + {% set schemas_skipped = 0 %} {% do log("Granting SELECT on all tables and views in all schemas for share: " ~ share_name ~ " (dry_run=" ~ dry_run ~ ")", info=True) %} {% for schema in schemas %} - {% set grant_usage %} - grant usage on schema {{ database }}.{{ schema }} to share {{ share_name }}; - {% endset %} - {% if dry_run %} - {% do log(grant_usage, info=True) %} - {% else %} - {% do run_query(grant_usage) %} + {% set schema_statements = [] %} + + {# Check if schema USAGE already granted #} + {% if 'SCHEMA:' ~ database | upper ~ '.' ~ schema | upper not in existing_share_objects %} + {% do schema_statements.append('grant usage on schema ' ~ database ~ '.' ~ schema ~ ' to share ' ~ share_name ~ ';') %} {% endif %} - {# Grant SELECT on all tables in schema #} - {% set grant_tables_sql %} - grant select on all tables in schema {{ database }}.{{ schema }} to share {{ share_name }}; - {% endset %} - {% if dry_run %} - {% do log(grant_tables_sql, info=True) %} - {% else %} - {% do run_query(grant_tables_sql) %} + + {# Check if tables grant exists (heuristic: if any TABLE in this schema is in share, skip bulk) #} + {% set has_table_grants = [] %} + {% for obj in existing_share_objects %} + {% if obj.startswith('TABLE:' ~ database | upper ~ '.' ~ schema | upper ~ '.') %} + {% do has_table_grants.append(1) %} + {% endif %} + {% endfor %} + {% if has_table_grants | length == 0 %} + {% do schema_statements.append('grant select on all tables in schema ' ~ database ~ '.' ~ schema ~ ' to share ' ~ share_name ~ ';') %} {% endif %} - {# Grant SELECT on views individually #} + {# Grant SELECT on views individually (shares require individual view grants) #} {% set views_query %} show views in schema {{ database }}.{{ schema }}; {% endset %} {% set views_result = run_query(views_query) %} {% if execute and views_result is not none %} {% for row in views_result %} - {% set grant_view_sql %} - grant select on view {{ database }}.{{ schema }}.{{ row.name }} to share {{ share_name }}; - {% endset %} + {% set view_key = 'VIEW:' ~ database | upper ~ '.' ~ schema | upper ~ '.' ~ row.name | upper %} + {% if view_key not in existing_share_objects %} + {% do schema_statements.append('grant select on view ' ~ database ~ '.' ~ schema ~ '.' ~ row.name ~ ' to share ' ~ share_name ~ ';') %} + {% endif %} + {% endfor %} + {% endif %} + + {% if schema_statements | length == 0 %} + {% set schemas_skipped = schemas_skipped + 1 %} + {% else %} + {% for stmt in schema_statements %} {% if dry_run %} - {% do log(grant_view_sql, info=True) %} + {% do log(stmt, info=True) %} {% else %} - {% do run_query(grant_view_sql) %} + {% do run_query(stmt) %} {% endif %} {% endfor %} + {% set total_executed = total_executed + schema_statements | length %} {% endif %} {% endfor %} + {% do log('grant_internal_share_read: ' ~ total_executed ~ ' statements executed, ' ~ schemas_skipped ~ '/' ~ (schemas | length) ~ ' schemas skipped (dry_run=' ~ dry_run ~ ')', info=True) %} {% endif %} {% endif %} {% endmacro %} diff --git a/macros/grants/grant_object.sql b/macros/grants/grant_object.sql index ef72d0a..cc3fda7 100644 --- a/macros/grants/grant_object.sql +++ b/macros/grants/grant_object.sql @@ -1,5 +1,5 @@ {% macro grant_object(object_type, objects, grant_types, grant_roles) %} - {# Optimized: logs summary instead of returning a structure. Signature unchanged. #} + {# Grant-only: no revokes are performed. Signature unchanged. #} {% if flags.WHICH not in ['run', 'run-operation'] %} {% do log('Skipping grant_object: not run/run-operation context', info=True) %} {% do return(none) %} @@ -14,13 +14,11 @@ {% do return(none) %} {% endif %} - {% set excluded_privs = ['OWNERSHIP'] %} {# Always ignore these for grant/revoke logic #} - {% set revokable_read_privs = ['SELECT','REFERENCES','REBUILD'] %} - {% set revoke_statements = [] %} + {% set grant_roles = dbt_dataengineers_utils._grants_normalize_roles(grant_roles) %} {% set grant_statements = [] %} {% for object in objects %} - {% set existing_role_priv_map = {} %} {# key role -> list of privs #} + {% set existing_role_priv_map = {} %} {% do log('====> Processing ' ~ object_type ~ ' ' ~ object ~ ' with desired privileges ' ~ (grant_types | join(', ')) ~ ' for roles ' ~ (grant_roles | join(', ')), info=True) %} {% set query %} show grants on {{ object_type }} {{ target.database }}.{{ object }}; @@ -28,27 +26,14 @@ {% set results = run_query(query) %} {% if results %} {% for row in results %} - {% if row.privilege not in excluded_privs %} - {# classify existing privilege #} - {% set _role = row.grantee_name %} - {% set _priv = row.privilege %} - {% if _priv in grant_types %} - {% if _role not in grant_roles %} - {% do revoke_statements.append('revoke ' ~ _priv | lower ~ ' on ' ~ object_type ~ ' ' ~ target.database ~ '.' ~ object ~ ' from role ' ~ _role | lower ~ ';') %} - {% else %} - {# track existing desired priv #} - {% if existing_role_priv_map.get(_role) is none %} - {% set _ = existing_role_priv_map.update({_role: []}) %} - {% endif %} - {% if _priv not in existing_role_priv_map.get(_role) %} - {% set __ = existing_role_priv_map.get(_role).append(_priv) %} - {% endif %} - {% endif %} - {% else %} - {# privilege not desired -> revoke if granted to managed roles #} - {% if _role in grant_roles or _priv in revokable_read_privs %} - {% do revoke_statements.append('revoke ' ~ _priv | lower ~ ' on ' ~ object_type ~ ' ' ~ target.database ~ '.' ~ object ~ ' from role ' ~ _role | lower ~ ';') %} - {% endif %} + {% set _role = row.grantee_name | upper %} + {% set _priv = row.privilege %} + {% if _priv in grant_types and _role in grant_roles %} + {% if existing_role_priv_map.get(_role) is none %} + {% set _ = existing_role_priv_map.update({_role: []}) %} + {% endif %} + {% if _priv not in existing_role_priv_map.get(_role) %} + {% set __ = existing_role_priv_map.get(_role).append(_priv) %} {% endif %} {% endif %} {% endfor %} @@ -66,18 +51,15 @@ {% endfor %} {% endfor %} - {% set total_revokes = revoke_statements | length %} - {% set total_grants = grant_statements | length %} - {% set all_statements = revoke_statements + grant_statements %} - {% if all_statements | length == 0 %} + {% if grant_statements | length == 0 %} {% do log('grant_object: no changes required for supplied ' ~ object_type ~ ' objects', info=True) %} {% do return(none) %} {% endif %} - {% do log('grant_object summary: ' ~ total_revokes ~ ' revokes, ' ~ total_grants ~ ' grants (' ~ all_statements | length ~ ' total statements) for ' ~ object_type ~ ' objects', info=True) %} - {% for stmt in all_statements %} + {% do log('grant_object summary: ' ~ grant_statements | length ~ ' grants for ' ~ object_type ~ ' objects', info=True) %} + {% for stmt in grant_statements %} {% do log(stmt, info=True) %} {% set _ = run_query(stmt) %} {% endfor %} - {% do log('grant_object: completed privilege reconciliation for ' ~ object_type ~ ' objects', info=True) %} + {% do log('grant_object: completed granting privileges for ' ~ object_type ~ ' objects', info=True) %} {% endmacro %} \ No newline at end of file diff --git a/macros/grants/grant_object_application.sql b/macros/grants/grant_object_application.sql index caee31f..29473db 100644 --- a/macros/grants/grant_object_application.sql +++ b/macros/grants/grant_object_application.sql @@ -14,6 +14,7 @@ {% do return(none) %} {% endif %} + {% set grant_applications = dbt_dataengineers_utils._grants_normalize_roles(grant_applications) %} {% set excluded_privs = ['OWNERSHIP'] %} {# Always ignore these for grant/revoke logic #} {% set revokable_read_privs = ['SELECT'] %} {% set revoke_statements = [] %} @@ -30,7 +31,7 @@ {% for row in results %} {% if row.granted_to == 'APPLICATION' and row.privilege not in excluded_privs %} {# classify existing privilege #} - {% set _application = row.grantee_name %} + {% set _application = row.grantee_name | upper %} {% set _priv = row.privilege %} {% if _priv in grant_types %} {% if _application not in grant_applications %} diff --git a/macros/grants/grant_operate_to_application.sql b/macros/grants/grant_operate_to_application.sql index 70b7d25..fb79d9b 100644 --- a/macros/grants/grant_operate_to_application.sql +++ b/macros/grants/grant_operate_to_application.sql @@ -1,5 +1,6 @@ {% macro grant_operate_to_application(prefix, grant_applications) %} {% if flags.WHICH in ['run', 'run-operation'] %} + {% set grant_applications = dbt_dataengineers_utils._grants_normalize_roles(grant_applications) %} {% set revoke_statements = [] %} {% set grant_statements = [] %} {% set grant_schemas = [] %} @@ -30,10 +31,10 @@ {% for row in results %} {% if row.privilege not in ["OWNERSHIP"] %} {% if row.privilege in grant_types %} - {% if row.grantee_name not in grant_applications %} + {% if row.grantee_name | upper not in grant_applications %} {% do revoke_statements.append({ "privilege" : row.privilege, "role" : row.grantee_name, "schema" : object[1], "object" : object[2] }) %} {% else %} - {% do existing_grants.append({ "privilege" : row.privilege, "role" : row.grantee_name, "schema" : object[1], "object" : object[2] }) %} + {% do existing_grants.append({ "privilege" : row.privilege, "role" : row.grantee_name | upper, "schema" : object[1], "object" : object[2] }) %} {%endif%} {%endif%} {% endif %} diff --git a/macros/grants/grant_procedure_usage.sql b/macros/grants/grant_procedure_usage.sql index e307436..a18894b 100644 --- a/macros/grants/grant_procedure_usage.sql +++ b/macros/grants/grant_procedure_usage.sql @@ -11,118 +11,76 @@ {% macro grant_schema_procedure_usage_specific(schemas, grant_roles, revoke_current_grants, dry_run) %} {% if flags.WHICH not in ['run','run-operation'] %}{% do return(none) %}{% endif %} {% if schemas | length == 0 or grant_roles | length == 0 %}{% do log('grant_schema_procedure_usage_specific: nothing to do', info=True) %}{% do return(none) %}{% endif %} - {% set total_revokes = 0 %} + {% set grant_roles = dbt_dataengineers_utils._grants_normalize_roles(grant_roles) %} {% set total_grants = 0 %} + {% set schemas_skipped = 0 %} {% for schema in schemas %} - {% set existing_roles = [] %} + {% set schema_statements = [] %} - {# Get list of procedures in the schema using the pattern provided #} + {# Check if there are any procedures in this schema #} {% set proc_query %} - show procedures in schema {{ target.database }}.{{ schema }} - ->> - select - "name" as object_name, - "schema_name" as routine_schema, - SUBSTR( - "arguments", - POSITION('(' IN "arguments") + 1, - POSITION(')' IN "arguments") - POSITION('(' IN "arguments") - 1 - ) AS arguments, - concat("name", '(', - SUBSTR( - "arguments", - POSITION('(' IN "arguments") + 1, - POSITION(')' IN "arguments") - POSITION('(' IN "arguments") - 1 - ), - ')') as routine_name - from $1 - where "is_builtin" = 'N' + select count(*) as cnt + from information_schema.procedures + where procedure_schema = '{{ schema }}' {% endset %} + {% set proc_count_result = run_query(proc_query) %} + {% set proc_count = proc_count_result[0][0] if (execute and proc_count_result) else 0 %} - {# First run show procedures to populate result_scan #} - {% set show_proc_stmt = 'show procedures in schema ' ~ target.database ~ '.' ~ schema %} - {% set _ = run_query(show_proc_stmt) %} - - {# Now get the formatted procedure list #} - {% set proc_results = run_query(proc_query) %} - {% set procedures = [] %} - - {% if execute and proc_results %} - {% for row in proc_results %} - {% set full_proc_name = row[3] %} - {% do procedures.append(full_proc_name) %} - {% endfor %} - {% endif %} - - {% if procedures | length == 0 %} + {% if proc_count == 0 %} {% do log('grant_schema_procedure_usage_specific: no procedures found in schema ' ~ schema, info=True) %} {% else %} - {% do log('grant_schema_procedure_usage_specific: found ' ~ (procedures | length) ~ ' procedures in schema ' ~ schema, info=True) %} + {% do log('grant_schema_procedure_usage_specific: found ' ~ proc_count ~ ' procedures in schema ' ~ schema, info=True) %} - {# Check existing grants for each procedure #} - {% for procedure in procedures %} - {% set grant_query %} - show grants on procedure {{ target.database }}.{{ schema }}.{{ procedure }} - {% endset %} - {% set grant_results = run_query(grant_query) %} + {# Get existing USAGE grants on procedures in this schema in one query #} + {% set existing_usage_roles = [] %} + {% set usage_query %} + select distinct grantee + from information_schema.object_privileges + where object_schema = '{{ schema }}' + and privilege_type = 'USAGE' + and object_type = 'PROCEDURE' + {% endset %} + {% set usage_results = run_query(usage_query) %} + {% if execute and usage_results %} + {% for row in usage_results %} + {% if row[0] not in existing_usage_roles %} + {% do existing_usage_roles.append(row[0]) %} + {% endif %} + {% endfor %} + {% endif %} - {% if execute and grant_results %} - {% for row in grant_results %} - {% set priv = row.privilege %}{% set grantee = row.grantee_name %} - {% if priv == 'USAGE' %} - {% if grantee not in grant_roles %} - {% if revoke_current_grants %} - {% set stmt = 'revoke usage on procedure ' ~ target.database ~ '.' ~ schema ~ '.' ~ procedure ~ ' from role ' ~ grantee ~ ';' %} - {% do log(stmt, info=True) %} - {% if not dry_run %}{% set _ = run_query(stmt) %}{% endif %} - {% set total_revokes = total_revokes + 1 %} - {% endif %} - {% else %} - {% if grantee not in existing_roles %} - {% do existing_roles.append(grantee) %} - {% endif %} - {% endif %} - {% endif %} - {% endfor %} - {% endif %} - {% endfor %} + {# Revoke from roles not in grant_roles that currently have USAGE #} + {% if revoke_current_grants %} + {% for role_with_usage in existing_usage_roles %} + {% if role_with_usage not in grant_roles %} + {% do schema_statements.append('revoke usage on all procedures in schema ' ~ target.database ~ '.' ~ schema ~ ' from role ' ~ role_with_usage ~ ';') %} + {% endif %} + {% endfor %} + {% endif %} - {# Grant schema usage first #} - {% for role in grant_roles %} - {% set schema_stmt = 'grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';' %} - {% do log(schema_stmt, info=True) %} - {% if not dry_run %}{% set _ = run_query(schema_stmt) %}{% endif %} - {% set total_grants = total_grants + 1 %} - {% endfor %} + {# Check schema USAGE #} + {% set roles_with_usage = dbt_dataengineers_utils._grants_get_schema_grants(schema, 'USAGE', 'ROLE') %} - {# Grant procedure usage #} + {# Grant procedure usage only to roles that don't already have it #} {% for role in grant_roles %} - {% set needs_grant = true %} - {# Check if role already has USAGE on any procedure in this schema #} - {% for procedure in procedures %} - {% set grant_query %} - show grants on procedure {{ target.database }}.{{ schema }}.{{ procedure }} - {% endset %} - {% set grant_results = run_query(grant_query) %} - {% if execute and grant_results %} - {% for row in grant_results %} - {% if row.privilege == 'USAGE' and row.grantee_name == role %} - {% set needs_grant = false %} - {% break %} - {% endif %} - {% endfor %} + {% if role not in existing_usage_roles %} + {% if role not in roles_with_usage %} + {% do schema_statements.append('grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} {% endif %} - {% if not needs_grant %}{% break %}{% endif %} - {% endfor %} - - {% if needs_grant %} - {% set stmt = 'grant usage on all procedures in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';' %} - {% do log(stmt, info=True) %} - {% if not dry_run %}{% set _ = run_query(stmt) %}{% endif %} - {% set total_grants = total_grants + 1 %} + {% do schema_statements.append('grant usage on all procedures in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} {% endif %} {% endfor %} {% endif %} + + {% if schema_statements | length == 0 %} + {% set schemas_skipped = schemas_skipped + 1 %} + {% else %} + {% for s in schema_statements %} + {% do log(s, info=True) %} + {% if not dry_run %}{% set _ = run_query(s) %}{% endif %} + {% endfor %} + {% set total_grants = total_grants + schema_statements | length %} + {% endif %} {% endfor %} - {% do log('grant_schema_procedure_usage_specific summary: ' ~ total_revokes ~ ' revokes, ' ~ total_grants ~ ' grants (dry_run=' ~ dry_run ~ ')', info=True) %} + {% do log('grant_schema_procedure_usage_specific summary: ' ~ total_grants ~ ' statements executed, ' ~ schemas_skipped ~ '/' ~ (schemas | length) ~ ' schemas skipped (dry_run=' ~ dry_run ~ ')', info=True) %} {% endmacro %} diff --git a/macros/grants/grant_schema_monitor.sql b/macros/grants/grant_schema_monitor.sql index d7661f1..0c650be 100644 --- a/macros/grants/grant_schema_monitor.sql +++ b/macros/grants/grant_schema_monitor.sql @@ -11,47 +11,60 @@ {% macro grant_schema_monitor_specific(schemas, grant_roles, revoke_current_grants, dry_run) %} {% if flags.WHICH not in ['run'] %}{% do return(none) %}{% endif %} {% if schemas | length == 0 or grant_roles | length == 0 %}{% do log('grant_schema_monitor_specific: nothing to do', info=True) %}{% do return(none) %}{% endif %} + {% set grant_roles = dbt_dataengineers_utils._grants_normalize_roles(grant_roles) %} {% set total_revokes = 0 %} {% set total_grants = 0 %} + {% set schemas_skipped = 0 %} {% for schema in schemas %} - {% set existing_roles = [] %} + {% set schema_statements = [] %} + {# Query existing MONITOR grants for this schema in one call #} + {% set existing_monitor_roles = [] %} {% set query %} - select object_type, concat(object_schema, '.', object_name) as object_name, privilege_type, grantee + select privilege_type, grantee from information_schema.object_privileges where privilege_type = 'MONITOR' and object_schema = '{{ schema }}' {% endset %} {% set results = run_query(query) %} {% if execute and results %} {% for row in results %} - {% set priv = row[2] %}{% set grantee = row[3] %} + {% set priv = row[0] %}{% set grantee = row[1] %} {% if priv == 'MONITOR' %} {% if grantee not in grant_roles %} {% if revoke_current_grants %} - {% set stmt = 'revoke monitor on ' ~ row[0] ~ ' in schema ' ~ target.database ~ '.' ~ row[1] ~ ' from role ' ~ grantee ~ ';' %} - {% do log(stmt, info=True) %} - {% if not dry_run %}{% set _ = run_query(stmt) %}{% endif %} - {% set total_revokes = total_revokes + 1 %} + {% do schema_statements.append('revoke monitor on all tasks in schema ' ~ target.database ~ '.' ~ schema ~ ' from role ' ~ grantee ~ ';') %} + {% do schema_statements.append('revoke monitor on all pipes in schema ' ~ target.database ~ '.' ~ schema ~ ' from role ' ~ grantee ~ ';') %} {% endif %} {% else %} - {% do existing_roles.append(grantee) %} + {% if grantee not in existing_monitor_roles %} + {% do existing_monitor_roles.append(grantee) %} + {% endif %} {% endif %} {% endif %} {% endfor %} {% endif %} + + {# Check schema USAGE for each role #} + {% set roles_with_usage = dbt_dataengineers_utils._grants_get_schema_grants(schema, 'USAGE', 'ROLE') %} + {% for role in grant_roles %} - {% if role not in existing_roles %} - {% set stmts = [ - 'grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';', - 'grant monitor on all pipes in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';', - 'grant monitor on all tasks in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';' - ] %} - {% for s in stmts %} - {% do log(s, info=True) %} - {% if not dry_run %}{% set _ = run_query(s) %}{% endif %} - {% set total_grants = total_grants + 1 %} - {% endfor %} + {% if role not in existing_monitor_roles %} + {% if role not in roles_with_usage %} + {% do schema_statements.append('grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} + {% endif %} + {% do schema_statements.append('grant monitor on all pipes in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} + {% do schema_statements.append('grant monitor on all tasks in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} {% endif %} {% endfor %} + + {% if schema_statements | length == 0 %} + {% set schemas_skipped = schemas_skipped + 1 %} + {% else %} + {% for s in schema_statements %} + {% do log(s, info=True) %} + {% if not dry_run %}{% set _ = run_query(s) %}{% endif %} + {% endfor %} + {% set total_grants = total_grants + schema_statements | length %} + {% endif %} {% endfor %} - {% do log('grant_schema_monitor_specific summary: ' ~ total_revokes ~ ' revokes, ' ~ total_grants ~ ' grants (dry_run=' ~ dry_run ~ ')', info=True) %} + {% do log('grant_schema_monitor_specific summary: ' ~ total_grants ~ ' statements executed, ' ~ schemas_skipped ~ '/' ~ (schemas | length) ~ ' schemas skipped (dry_run=' ~ dry_run ~ ')', info=True) %} {% endmacro %} \ No newline at end of file diff --git a/macros/grants/grant_schema_object_privileges.sql b/macros/grants/grant_schema_object_privileges.sql index 29c33b0..6997fc5 100644 --- a/macros/grants/grant_schema_object_privileges.sql +++ b/macros/grants/grant_schema_object_privileges.sql @@ -1,5 +1,5 @@ {% macro grant_schema_object_privileges(object_type, schema_name, permissions, roles) %} - {# Enhanced macro that grants privileges on all objects of a specific type within a schema #} + {# Grants privileges on all objects of a specific type within a schema. Uses bulk queries to check state. #} {% if flags.WHICH not in ['run', 'run-operation'] %} {% do log('Skipping grant_schema_object_privileges: not run/run-operation context', info=True) %} {% do return(none) %} @@ -17,35 +17,31 @@ {% endif %} {% if roles is string %} - {% set role_list = roles.split(',') | map('trim') | list %} + {% set role_list = roles.split(',') | map('trim') | map('upper') | list %} {% else %} - {% set role_list = roles %} + {% set role_list = dbt_dataengineers_utils._grants_normalize_roles(roles) %} {% endif %} {% do log('grant_schema_object_privileges: processing ' ~ object_type ~ ' objects in schema ' ~ schema_name, info=True) %} {% do log('====> Target permissions: ' ~ (permission_list | join(', ')), info=True) %} {% do log('====> Target roles: ' ~ (role_list | join(', ')), info=True) %} - + {# Discover objects #} {% set discovered_objects = [] %} {% if object_type.upper() in ['FUNCTION', 'PROCEDURE'] %} - {% set objects = [] %} {% if object_type.upper() == 'FUNCTION' %} {% do discovered_objects.extend(dbt_dataengineers_utils.get_functions("'" ~ schema_name ~ "'")) %} {% elif object_type.upper() == 'PROCEDURE' %} {% do discovered_objects.extend(dbt_dataengineers_utils.get_procedures("'" ~ schema_name ~ "'")) %} {% endif %} {% else %} - - {# Discover objects of the specified type in the schema #} {% set objects_query %} show {{ object_type }}s in schema {{ target.database }}.{{ schema_name }}; {% endset %} {% set objects_results = run_query(objects_query) %} - {% if objects_results %} {% for row in objects_results %} - {% set object_name = target.database ~ '.' ~ schema_name ~ '.' ~ row[1] %} {# default to name column #} + {% set object_name = target.database ~ '.' ~ schema_name ~ '.' ~ row[1] %} {% do discovered_objects.append(object_name) %} {% endfor %} {% endif %} @@ -58,99 +54,55 @@ {% do log('====> Found ' ~ (discovered_objects | length) ~ ' ' ~ object_type ~ 's', info=True) %} - {# Enhanced logic with bulk grants and individual revokes #} - {% set excluded_privs = ['OWNERSHIP'] %} {# Always ignore these for grant/revoke logic #} - {% set revokable_read_privs = ['SELECT','REFERENCES','REBUILD'] %} - {% set revoke_statements = [] %} - {% set bulk_grant_needed = {} %} {# Track role->privilege combinations that need bulk grants #} - {% set all_existing_role_privs = {} %} {# Global tracking of existing privileges across all objects #} - - {# First pass: analyze existing grants and determine revokes #} - {% for object in discovered_objects %} - {% set existing_role_priv_map = {} %} {# key role -> list of privs for this object #} - {% do log('====> Processing ' ~ object_type ~ ' ' ~ object ~ ' with desired privileges ' ~ (permission_list | join(', ')) ~ ' for roles ' ~ (role_list | join(', ')), info=True) %} - - {% set query %} - show grants on {{ object_type }} {{ object }}; - {% endset %} - {% set results = run_query(query) %} - - {% if results %} - {% for row in results %} - {% if row.privilege not in excluded_privs %} - {# classify existing privilege #} - {% set _role = row.grantee_name %} - {% set _priv = row.privilege %} - - {# Track existing privileges globally #} - {% if all_existing_role_privs.get(_role) is none %} - {% set _ = all_existing_role_privs.update({_role: {}}) %} - {% endif %} - {% if all_existing_role_privs.get(_role).get(_priv) is none %} - {% set _ = all_existing_role_privs.get(_role).update({_priv: []}) %} - {% endif %} - {% do all_existing_role_privs.get(_role).get(_priv).append(object) %} - - {% if _priv in permission_list %} - {% if _role not in role_list %} - {# Revoke privilege from unwanted roles #} - {% do revoke_statements.append('revoke ' ~ _priv | lower ~ ' on ' ~ object_type ~ ' ' ~ target.database ~ '.' ~ object ~ ' from role ' ~ _role | lower ~ ';') %} - {% else %} - {# Track existing desired priv for this object #} - {% if existing_role_priv_map.get(_role) is none %} - {% set _ = existing_role_priv_map.update({_role: []}) %} - {% endif %} - {% if _priv not in existing_role_priv_map.get(_role) %} - {% set __ = existing_role_priv_map.get(_role).append(_priv) %} - {% endif %} - {% endif %} - {% else %} - {# privilege not desired -> revoke if granted to managed roles #} - {% if _role in role_list or _priv in revokable_read_privs %} - {% do revoke_statements.append('revoke ' ~ _priv | lower ~ ' on ' ~ object_type ~ ' ' ~ target.database ~ '.' ~ object ~ ' from role ' ~ _role | lower ~ ';') %} - {% endif %} - {% endif %} + {# Bulk query: get all existing privileges for this object type in this schema #} + {% set existing_privs = dbt_dataengineers_utils._grants_get_schema_object_privs(schema_name, permission_list, role_list, object_type) %} + + {# Check which roles need grants #} + {% set bulk_grant_needed = {} %} + {% for role in role_list %} + {% set role_privs = existing_privs.get(role) if existing_privs.get(role) is not none else [] %} + {% for privilege in permission_list %} + {% if privilege not in role_privs %} + {% if bulk_grant_needed.get(role) is none %} + {% set _ = bulk_grant_needed.update({role: []}) %} {% endif %} - {% endfor %} - {% endif %} - - {# Track what bulk grants are needed #} - {% for role in role_list %} - {% set existing_for_role = existing_role_priv_map.get(role) if existing_role_priv_map.get(role) is not none else [] %} - {% do log('====> Existing grants for role ' ~ role ~ ' on ' ~ object ~ ' : ' ~ (existing_for_role | join(', ')), info=True) %} - {% for privilege in permission_list %} - {% if privilege not in existing_for_role %} - {# This role needs this privilege - mark for bulk grant #} - {% if bulk_grant_needed.get(role) is none %} - {% set _ = bulk_grant_needed.update({role: []}) %} - {% endif %} - {% if privilege not in bulk_grant_needed.get(role) %} - {% do bulk_grant_needed.get(role).append(privilege) %} - {% endif %} + {% if privilege not in bulk_grant_needed.get(role) %} + {% do bulk_grant_needed.get(role).append(privilege) %} {% endif %} - {% endfor %} + {% endif %} {% endfor %} {% endfor %} - {# Second pass: generate bulk grant statements #} + {# Build revoke statements for roles NOT in the list that have these privs on this object type #} + {% set revoke_statements = [] %} + {% set revoke_query %} + select distinct privilege_type, grantee + from information_schema.object_privileges + where object_schema = '{{ schema_name }}' + and privilege_type in ({{ permission_list | map('tojson') | join(', ') }}) + and grantee not in ({{ role_list | map('tojson') | join(', ') }}) + and object_type = '{{ object_type | upper }}' + and grantor is not null + {% endset %} + {% set revoke_results = run_query(revoke_query) %} + {% if execute and revoke_results %} + {% for row in revoke_results %} + {% set stmt = 'revoke ' ~ row[0] | lower ~ ' on all ' ~ object_type | lower ~ 's in schema ' ~ target.database ~ '.' ~ schema_name ~ ' from role ' ~ row[1] | lower ~ ';' %} + {% if stmt not in revoke_statements %} + {% do revoke_statements.append(stmt) %} + {% endif %} + {% endfor %} + {% endif %} + + {# Build bulk grant statements #} {% set bulk_grant_statements = [] %} {% for role in bulk_grant_needed.keys() %} {% for privilege in bulk_grant_needed.get(role) %} - {# Check if ALL objects in the schema need this privilege for this role #} - {% set objects_with_priv = all_existing_role_privs.get(role, {}).get(privilege, []) %} - {% set objects_needing_priv = discovered_objects | length - (objects_with_priv | length) %} - - {% if objects_needing_priv > 0 %} - {# Generate bulk grant statement #} - {% set bulk_stmt = 'grant ' ~ privilege | lower ~ ' on all ' ~ object_type | lower ~ 's in schema ' ~ target.database ~ '.' ~ schema_name ~ ' to role ' ~ role | lower ~ ';' %} - {% do bulk_grant_statements.append(bulk_stmt) %} - {% do log('====> Bulk grant needed: ' ~ privilege ~ ' to ' ~ role ~ ' (' ~ objects_needing_priv ~ ' objects)', info=True) %} - {% endif %} + {% set bulk_stmt = 'grant ' ~ privilege | lower ~ ' on all ' ~ object_type | lower ~ 's in schema ' ~ target.database ~ '.' ~ schema_name ~ ' to role ' ~ role | lower ~ ';' %} + {% do bulk_grant_statements.append(bulk_stmt) %} {% endfor %} {% endfor %} - {% set total_revokes = revoke_statements | length %} - {% set total_bulk_grants = bulk_grant_statements | length %} {% set all_statements = revoke_statements + bulk_grant_statements %} {% if all_statements | length == 0 %} @@ -158,40 +110,19 @@ {% do return(none) %} {% endif %} - {% do log('grant_schema_object_privileges summary: ' ~ total_revokes ~ ' individual revokes, ' ~ total_bulk_grants ~ ' bulk grants (' ~ all_statements | length ~ ' total statements) for ' ~ object_type ~ ' objects in schema ' ~ schema_name, info=True) %} + {% do log('grant_schema_object_privileges summary: ' ~ (revoke_statements | length) ~ ' revokes, ' ~ (bulk_grant_statements | length) ~ ' bulk grants for ' ~ object_type ~ ' objects in schema ' ~ schema_name, info=True) %} - {# Execute statements if not in dry run mode #} + {# Execute statements #} {% if var('grants_dry_run', false) %} {% do log('DRY RUN MODE - statements would be:', info=True) %} - {% if revoke_statements | length > 0 %} - {% do log('=== Individual Revoke Statements ===', info=True) %} - {% for stmt in revoke_statements %} - {% do log(stmt, info=True) %} - {% endfor %} - {% endif %} - {% if bulk_grant_statements | length > 0 %} - {% do log('=== Bulk Grant Statements ===', info=True) %} - {% for stmt in bulk_grant_statements %} - {% do log(stmt, info=True) %} - {% endfor %} - {% endif %} + {% for stmt in all_statements %} + {% do log(stmt, info=True) %} + {% endfor %} {% else %} - {# Execute revoke statements first #} - {% if revoke_statements | length > 0 %} - {% do log('=== Executing Individual Revoke Statements ===', info=True) %} - {% for stmt in revoke_statements %} - {% do log(stmt, info=True) %} - {% set _ = run_query(stmt) %} - {% endfor %} - {% endif %} - {# Then execute bulk grant statements #} - {% if bulk_grant_statements | length > 0 %} - {% do log('=== Executing Bulk Grant Statements ===', info=True) %} - {% for stmt in bulk_grant_statements %} - {% do log(stmt, info=True) %} - {% set _ = run_query(stmt) %} - {% endfor %} - {% endif %} + {% for stmt in all_statements %} + {% do log(stmt, info=True) %} + {% set _ = run_query(stmt) %} + {% endfor %} {% endif %} {% do log('grant_schema_object_privileges: completed privilege reconciliation for ' ~ object_type ~ ' objects in schema ' ~ schema_name, info=True) %} diff --git a/macros/grants/grant_schema_operate.sql b/macros/grants/grant_schema_operate.sql index a76ed5d..4bbc8ba 100644 --- a/macros/grants/grant_schema_operate.sql +++ b/macros/grants/grant_schema_operate.sql @@ -11,47 +11,59 @@ {% macro grant_schema_operate_specific(schemas, grant_roles, revoke_current_grants, dry_run) %} {% if flags.WHICH not in ['run'] %}{% do return(none) %}{% endif %} {% if schemas | length == 0 or grant_roles | length == 0 %}{% do log('grant_schema_operate_specific: nothing to do', info=True) %}{% do return(none) %}{% endif %} - {% set total_revokes = 0 %} + {% set grant_roles = dbt_dataengineers_utils._grants_normalize_roles(grant_roles) %} {% set total_grants = 0 %} + {% set schemas_skipped = 0 %} {% for schema in schemas %} - {% set existing_roles = [] %} + {% set schema_statements = [] %} + {# Query existing OPERATE grants for this schema in one call #} + {% set existing_operate_roles = [] %} {% set query %} - select object_type, concat(object_schema, '.', object_name) as object_name, privilege_type, grantee + select privilege_type, grantee from information_schema.object_privileges where privilege_type = 'OPERATE' and object_schema = '{{ schema }}' {% endset %} {% set results = run_query(query) %} {% if execute and results %} {% for row in results %} - {% set priv = row[2] %}{% set grantee = row[3] %} + {% set priv = row[0] %}{% set grantee = row[1] %} {% if priv == 'OPERATE' %} {% if grantee not in grant_roles %} {% if revoke_current_grants %} - {% set stmt = 'revoke operate on ' ~ row[0] ~ ' in schema ' ~ target.database ~ '.' ~ row[1] ~ ' from role ' ~ grantee ~ ';' %} - {% do log(stmt, info=True) %} - {% if not dry_run %}{% set _ = run_query(stmt) %}{% endif %} - {% set total_revokes = total_revokes + 1 %} + {% do schema_statements.append('revoke operate on all tasks in schema ' ~ target.database ~ '.' ~ schema ~ ' from role ' ~ grantee ~ ';') %} + {% do schema_statements.append('revoke operate on all pipes in schema ' ~ target.database ~ '.' ~ schema ~ ' from role ' ~ grantee ~ ';') %} {% endif %} {% else %} - {% do existing_roles.append(grantee) %} + {% if grantee not in existing_operate_roles %} + {% do existing_operate_roles.append(grantee) %} + {% endif %} {% endif %} {% endif %} {% endfor %} {% endif %} + + {# Check schema USAGE for each role #} + {% set roles_with_usage = dbt_dataengineers_utils._grants_get_schema_grants(schema, 'USAGE', 'ROLE') %} + {% for role in grant_roles %} - {% if role not in existing_roles %} - {% set stmts = [ - 'grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';', - 'grant operate on all pipes in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';', - 'grant operate on all tasks in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';' - ] %} - {% for s in stmts %} - {% do log(s, info=True) %} - {% if not dry_run %}{% set _ = run_query(s) %}{% endif %} - {% set total_grants = total_grants + 1 %} - {% endfor %} + {% if role not in existing_operate_roles %} + {% if role not in roles_with_usage %} + {% do schema_statements.append('grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} + {% endif %} + {% do schema_statements.append('grant operate on all pipes in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} + {% do schema_statements.append('grant operate on all tasks in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role ~ ';') %} {% endif %} {% endfor %} + + {% if schema_statements | length == 0 %} + {% set schemas_skipped = schemas_skipped + 1 %} + {% else %} + {% for s in schema_statements %} + {% do log(s, info=True) %} + {% if not dry_run %}{% set _ = run_query(s) %}{% endif %} + {% endfor %} + {% set total_grants = total_grants + schema_statements | length %} + {% endif %} {% endfor %} - {% do log('grant_schema_operate_specific summary: ' ~ total_revokes ~ ' revokes, ' ~ total_grants ~ ' grants (dry_run=' ~ dry_run ~ ')', info=True) %} + {% do log('grant_schema_operate_specific summary: ' ~ total_grants ~ ' statements executed, ' ~ schemas_skipped ~ '/' ~ (schemas | length) ~ ' schemas skipped (dry_run=' ~ dry_run ~ ')', info=True) %} {% endmacro %} \ No newline at end of file diff --git a/macros/grants/grant_schema_read.sql b/macros/grants/grant_schema_read.sql index bb57ff0..f1af3cb 100644 --- a/macros/grants/grant_schema_read.sql +++ b/macros/grants/grant_schema_read.sql @@ -36,91 +36,130 @@ {% do return(none) %} {% endif %} + {% set grant_roles = dbt_dataengineers_utils._grants_normalize_roles(grant_roles) %} {% set snowflake_roles = dbt_dataengineers_utils._grants_collect_roles() %} {% set execute_statements = [] %} - {% set read_object_types = ['views','materialized views','tables','external tables','dynamic tables','streams'] %} - {% set revoke_table_privs = ['SELECT','REFERENCES','REBUILD'] %} + {% set schemas_skipped = [0] %} {% for schema in schemas %} {% do log('====> Processing schema read grants for ' ~ schema, info=True) %} + {% set schema_statements = [] %} + + {# Get existing schema-level grants once #} + {% set roles_with_usage = dbt_dataengineers_utils._grants_get_schema_grants(schema, 'USAGE', 'ROLE') %} + {# Revoke schema usage from roles not in grant_roles if requested #} - {% set show_query %} show grants on schema {{ target.database }}.{{ schema }}; {% endset %} - {% set grants_results = run_query(show_query) %} - {% if grants_results %} - {% for row in grants_results %} - {% if row.privilege == 'USAGE' and row.granted_to == 'ROLE' %} - {% if row.grantee_name not in grant_roles and revoke_current_grants and row.grantee_name in snowflake_roles %} - {% do execute_statements.append('revoke usage on schema ' ~ target.database ~ '.' ~ schema ~ ' from role ' ~ row.grantee_name | lower ~ ';') %} - {% endif %} + {% if revoke_current_grants %} + {% for role_with_usage in roles_with_usage %} + {% if role_with_usage not in grant_roles and role_with_usage in snowflake_roles %} + {% do schema_statements.append('revoke usage on schema ' ~ target.database ~ '.' ~ schema ~ ' from role ' ~ role_with_usage | lower ~ ';') %} {% endif %} {% endfor %} {% endif %} - {# Revoke object-level read privileges #} - {% set priv_query %} - select distinct - case - when is_dynamic = 'YES' then 'DYNAMIC TABLE' - when is_iceberg = 'YES' then 'ICEBERG TABLE' - when table_type = 'BASE TABLE' then 'TABLE' - else tables.table_type - end as object_type, - tables.table_schema, - table_privileges.privilege_type as privilege, - table_privileges.grantee as grantee_name - from information_schema.table_privileges - inner join information_schema.tables - on table_privileges.table_name = tables.table_name - and table_privileges.table_schema = tables.table_schema - and table_privileges.table_catalog = tables.table_catalog - where tables.table_schema = '{{ schema }}' - and table_privileges.privilege_type in ('SELECT','REFERENCES','REBUILD') - and granted_to = 'ROLE' - {% endset %} - {% set tbl_privs = run_query(priv_query) %} - {% if tbl_privs %} - {% for row in tbl_privs %} - {% set obj_type = row[0] %} - {% set sch = row[1] %} - {% set priv = row[2] %} - {% set grantee = row[3] %} - {% if priv in revoke_table_privs and grantee not in grant_roles and revoke_current_grants and grantee in snowflake_roles %} - {% do execute_statements.append('revoke ' ~ priv | lower ~ ' on all ' ~ obj_type | lower ~ 's in schema ' ~ target.database ~ '.' ~ sch | lower ~ ' from role ' ~ grantee | lower ~ ';') %} - {% endif %} - {% endfor %} + {# Revoke object-level read privileges from roles not in grant_roles #} + {% if revoke_current_grants %} + {% set revoke_query %} + select distinct + case + when is_dynamic = 'YES' then 'DYNAMIC TABLE' + when is_iceberg = 'YES' then 'ICEBERG TABLE' + when table_type = 'BASE TABLE' then 'TABLE' + else tables.table_type + end as object_type, + tables.table_schema, + table_privileges.privilege_type as privilege, + table_privileges.grantee as grantee_name + from information_schema.table_privileges + inner join information_schema.tables + on table_privileges.table_name = tables.table_name + and table_privileges.table_schema = tables.table_schema + and table_privileges.table_catalog = tables.table_catalog + where tables.table_schema = '{{ schema }}' + and table_privileges.privilege_type in ('SELECT','REFERENCES','REBUILD') + and table_privileges.grantee not in ({{ grant_roles | map('tojson') | join(', ') }}) + and granted_to = 'ROLE' + {% endset %} + {% set tbl_privs = run_query(revoke_query) %} + {% if tbl_privs %} + {% for row in tbl_privs %} + {% set grantee = row[3] %} + {% if grantee in snowflake_roles %} + {% do schema_statements.append('revoke ' ~ row[2] | lower ~ ' on all ' ~ row[0] | lower ~ 's in schema ' ~ target.database ~ '.' ~ row[1] | lower ~ ' from role ' ~ grantee | lower ~ ';') %} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} + + {# Get future grants once for this schema if needed #} + {% set future_grants = {} %} + {% if include_future_grants %} + {% set future_grants = dbt_dataengineers_utils._grants_get_future_grants(schema) %} {% endif %} - {# Build grant statements #} + {# Build grant statements — GRANT ON ALL is idempotent in Snowflake (no-op if already granted) #} {% for role in grant_roles %} - {% do execute_statements.append('grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on all views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on all materialized views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on all tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant rebuild on all tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant references on all tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on all external tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on all dynamic tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on all streams in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant read on all stages in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% set role_future = future_grants.get(role | upper) if future_grants.get(role | upper) is not none else [] %} + + {# Schema USAGE — only grant if not already present #} + {% if role not in roles_with_usage %} + {% do schema_statements.append('grant usage on schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + + {# Object grants — always issue, Snowflake handles idempotency #} + {% do schema_statements.append('grant select on all views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant select on all materialized views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant select on all tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant select on all external tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant select on all dynamic tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant select on all streams in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant rebuild on all tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant references on all tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% do schema_statements.append('grant read on all stages in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + + {# Future grants - only issue if not already set (these are NOT idempotent — duplicates error) #} {% if include_future_grants %} - {% do execute_statements.append('grant select on future views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on future materialized views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on future tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant rebuild on future tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant references on future tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on future external tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on future dynamic tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} - {% do execute_statements.append('grant select on future streams in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% if 'SELECT:VIEW' not in role_future %} + {% do schema_statements.append('grant select on future views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + {% if 'SELECT:MATERIALIZED VIEW' not in role_future %} + {% do schema_statements.append('grant select on future materialized views in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + {% if 'SELECT:TABLE' not in role_future %} + {% do schema_statements.append('grant select on future tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + {% if 'REBUILD:TABLE' not in role_future %} + {% do schema_statements.append('grant rebuild on future tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + {% if 'REFERENCES:TABLE' not in role_future %} + {% do schema_statements.append('grant references on future tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + {% if 'SELECT:EXTERNAL TABLE' not in role_future %} + {% do schema_statements.append('grant select on future external tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + {% if 'SELECT:DYNAMIC TABLE' not in role_future %} + {% do schema_statements.append('grant select on future dynamic tables in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} + {% if 'SELECT:STREAM' not in role_future %} + {% do schema_statements.append('grant select on future streams in schema ' ~ target.database ~ '.' ~ schema ~ ' to role ' ~ role | lower ~ ';') %} + {% endif %} {% endif %} {% endfor %} + + {% if schema_statements | length == 0 %} + {% do log('====> Schema ' ~ schema ~ ': no changes required (skipped)', info=True) %} + {% do schemas_skipped.append(1) %} + {% else %} + {% do execute_statements.extend(schema_statements) %} + {% endif %} {% endfor %} {% set total_statements = execute_statements | length %} {% if total_statements == 0 %} - {% do log('grant_schema_read_specific: no changes required', info=True) %} + {% do log('grant_schema_read_specific: no changes required across ' ~ (schemas | length) ~ ' schemas', info=True) %} {% do return(none) %} {% endif %} - {% do log('grant_schema_read_specific: executing ' ~ total_statements ~ ' statements across ' ~ (schemas | length) ~ ' schemas', info=True) %} + {% do log('grant_schema_read_specific: executing ' ~ total_statements ~ ' statements (' ~ (schemas_skipped | length - 1) ~ '/' ~ (schemas | length) ~ ' schemas skipped, no changes needed)', info=True) %} {% for stmt in execute_statements %} {% do log(stmt, info=True) %} {% set _ = run_query(stmt) %} diff --git a/macros/grants/grant_share_read.sql b/macros/grants/grant_share_read.sql index ab89b0f..a4e6f42 100644 --- a/macros/grants/grant_share_read.sql +++ b/macros/grants/grant_share_read.sql @@ -52,6 +52,22 @@ {% if share.kind == 'OUTBOUND' and share.name not in snowflake_shares %}{% do snowflake_shares.append(share.name) %}{% endif %} {% endfor %} {% do log('grant_share_read_specific_schema: processing schema ' ~ schema_name | lower, info=True) %} + + {# Build a map of existing share grants from DESC SHARE to avoid redundant grants #} + {% set existing_share_grants = {} %} + {% for share in grant_shares %} + {% if share in snowflake_shares %} + {% set share_objects = [] %} + {% set share_desc = run_query('desc share ' ~ share | lower ~ ';') %} + {% for row in share_desc %} + {% if row[1].split('.')[0] | lower == target.database | lower and row[1].split('.')[1] | lower == schema_name | lower %} + {% do share_objects.append(row[0] ~ ':' ~ row[1] | upper) %} + {% endif %} + {% endfor %} + {% set _ = existing_share_grants.update({share: share_objects}) %} + {% endif %} + {% endfor %} + {% if revoke_current_grants %} {% set results = run_query('show grants on schema ' ~ target.database | lower ~ '.' ~ schema_name | lower ~ ';') %} {% for row in results %} @@ -74,16 +90,30 @@ {% endfor %} {% endfor %} {% endif %} + + {# Grant only where not already present #} {% for share in grant_shares %} - {% do execute_statements.append('grant usage on schema ' ~ target.database ~ '.' ~ schema_name ~ ' to share ' ~ share ~ ';') %} + {% set share_existing = existing_share_grants.get(share) if existing_share_grants.get(share) is not none else [] %} + {% set schema_key = 'SCHEMA:' ~ target.database | upper ~ '.' ~ schema_name | upper %} + {% if schema_key not in share_existing %} + {% do execute_statements.append('grant usage on schema ' ~ target.database ~ '.' ~ schema_name ~ ' to share ' ~ share ~ ';') %} + {% endif %} {% for view in view_names %} - {% do execute_statements.append('grant select on view ' ~ target.database ~ '.' ~ schema_name ~ '.' ~ view ~ ' to share ' ~ share ~ ';') %} + {% set view_key = 'VIEW:' ~ target.database | upper ~ '.' ~ schema_name | upper ~ '.' ~ view | upper %} + {% if view_key not in share_existing %} + {% do execute_statements.append('grant select on view ' ~ target.database ~ '.' ~ schema_name ~ '.' ~ view ~ ' to share ' ~ share ~ ';') %} + {% endif %} {% endfor %} {% endfor %} - {% for statement in execute_statements %} - {% do log(statement, info=True) %} - {% if not dry_run %}{% set _ = run_query(statement) %}{% endif %} - {% endfor %} - {% do log('grant_share_read_specific_schema summary: executed ' ~ (execute_statements | length) ~ ' statements (dry_run=' ~ dry_run ~ ')', info=True) %} + + {% if execute_statements | length == 0 %} + {% do log('grant_share_read_specific_schema: no changes required for schema ' ~ schema_name, info=True) %} + {% else %} + {% for statement in execute_statements %} + {% do log(statement, info=True) %} + {% if not dry_run %}{% set _ = run_query(statement) %}{% endif %} + {% endfor %} + {% do log('grant_share_read_specific_schema summary: executed ' ~ (execute_statements | length) ~ ' statements (dry_run=' ~ dry_run ~ ')', info=True) %} + {% endif %} {% endif %} {% endmacro %} \ No newline at end of file diff --git a/macros/grants/grant_usage_to_application.sql b/macros/grants/grant_usage_to_application.sql index 1526048..8eeb9b2 100644 --- a/macros/grants/grant_usage_to_application.sql +++ b/macros/grants/grant_usage_to_application.sql @@ -1,5 +1,6 @@ {% macro grant_usage_to_application(object_type, prefix, grant_applications) %} {% if flags.WHICH in ['run', 'run-operation'] %} + {% set grant_applications = dbt_dataengineers_utils._grants_normalize_roles(grant_applications) %} {% set revoke_statements = [] %} {% set grant_statements = [] %} {% set grant_schemas = [] %} @@ -30,10 +31,10 @@ {% for row in results %} {% if row.privilege not in ["OWNERSHIP", "SELECT", "REFERENCES", "REBUILD"] %} {% if row.privilege in grant_types %} - {% if row.grantee_name not in grant_applications %} + {% if row.grantee_name | upper not in grant_applications %} {% do revoke_statements.append({ "privilege" : row.privilege, "role" : row.grantee_name, "schema" : object[1], "object" : object[2] }) %} {% else %} - {% do existing_grants.append({ "privilege" : row.privilege, "role" : row.grantee_name, "schema" : object[1], "object" : object[2] }) %} + {% do existing_grants.append({ "privilege" : row.privilege, "role" : row.grantee_name | upper, "schema" : object[1], "object" : object[2] }) %} {%endif%} {% else %} {% do revoke_statements.append({ "privilege" : row.privilege, "role" : row.grantee_name, "schema" : object[1], "object" : object[2] }) %} diff --git a/macros/grants/grants_smoke_test.sql b/macros/grants/grants_smoke_test.sql index c9c2461..954a7d8 100644 --- a/macros/grants/grants_smoke_test.sql +++ b/macros/grants/grants_smoke_test.sql @@ -6,6 +6,7 @@ Behavior: - Confirms early-exit guard works - Invokes a representative subset of macros with constrained scope + - Tests case-insensitive role handling - Does not mutate state when grants_dry_run=true #} {% if flags.WHICH not in ['run','run-operation'] %} @@ -15,16 +16,29 @@ {% set dry_run = var('grants_dry_run', true) %} {% do log('grants_smoke_test: starting (dry_run=' ~ dry_run ~ ')', info=True) %} - {# Minimal object lists for grant_object test (uses SELECT privilege on a non-existent object will simply error if executed; rely on dry-run). #} - {% set test_objects = [] %} + {# Test case-insensitive role normalization #} + {% set mixed_case_role = role | lower %} + {% do log('grants_smoke_test: testing with mixed-case role "' ~ mixed_case_role ~ '" (should normalize to "' ~ role | upper ~ '")', info=True) %} - {# Invoke monitor / operate / read on narrow scope #} - {{ dbt_dataengineers_utils.grant_schema_monitor_specific([sample_schema], [role], true, dry_run) }} - {{ dbt_dataengineers_utils.grant_schema_operate_specific([sample_schema], [role], true, dry_run) }} - {{ dbt_dataengineers_utils.grant_schema_read_specific([sample_schema], [role], false, false) }} + {# Invoke monitor / operate / read on narrow scope with mixed-case role #} + {{ dbt_dataengineers_utils.grant_schema_monitor_specific([sample_schema], [mixed_case_role], false, dry_run) }} + {{ dbt_dataengineers_utils.grant_schema_operate_specific([sample_schema], [mixed_case_role], false, dry_run) }} + {{ dbt_dataengineers_utils.grant_schema_read_specific([sample_schema], [mixed_case_role], false, false) }} + + {# Test procedure usage with dry_run #} + {{ dbt_dataengineers_utils.grant_schema_procedure_usage_specific([sample_schema], [role], false, dry_run) }} + + {# Test grant_schema_object_privileges with dry_run #} + {{ dbt_dataengineers_utils.grant_schema_object_privileges('table', sample_schema, ['SELECT'], [role]) }} + + {# Test grant_object with empty objects (should exit early) #} + {{ dbt_dataengineers_utils.grant_object('table', [], ['SELECT'], [role]) }} {# Share read test (no views passed, revokes only if any) #} {{ dbt_dataengineers_utils.grant_share_read([], [], false) }} + {# Test database usage with mixed-case roles #} + {{ dbt_dataengineers_utils.grant_database_usage([mixed_case_role], [], false) }} + {% do log('grants_smoke_test: complete', info=True) %} {% endmacro %}