diff --git a/bootstrap/101-enable_cdc_on_odse_database-001.sql b/bootstrap/101-enable_cdc_on_odse_database-001.sql deleted file mode 100644 index 02e6928b7..000000000 --- a/bootstrap/101-enable_cdc_on_odse_database-001.sql +++ /dev/null @@ -1,84 +0,0 @@ --- ------------------------------------------ --- 1. Enable CDC at Database Level --- ------------------------------------------ -IF (SELECT is_cdc_enabled FROM sys.databases WHERE name = 'NBS_ODSE') = 0 -BEGIN - IF EXISTS (SELECT 1 FROM sys.databases WHERE NAME = 'rdsadmin') -- for aws - BEGIN - PRINT 'AWS RDS detected. Enabling CDC for NBS_ODSE using rds_cdc_enable_db'; - EXEC msdb.dbo.rds_cdc_enable_db 'NBS_ODSE'; - END - ELSE - BEGIN - PRINT 'Standard SQL Server detected. Enabling CDC for NBS_ODSE using sp_cdc_enable_db'; - EXEC('USE [NBS_ODSE]; EXEC sys.sp_cdc_enable_db;'); - END -END -ELSE -BEGIN - PRINT 'CDC is already enabled for NBS_ODSE'; -END -GO - --- ------------------------------------------ --- 2. Enable CDC for Tables --- ------------------------------------------ -USE [NBS_ODSE]; -GO - -DECLARE @tablesToEnable TABLE (TableName NVARCHAR(128)); - -INSERT INTO @tablesToEnable (TableName) -SELECT TableName -FROM (VALUES - ('Act_relationship'), - ('Auth_user'), - ('CT_contact'), - ('Intervention'), - ('Interview'), - ('NBS_page'), - ('NBS_rdb_metadata'), - ('NBS_ui_metadata'), - ('Notification'), - ('Observation'), - ('Organization'), - ('Page_cond_mapping'), - ('Person'), - ('Place'), - ('Public_health_case'), - ('state_defined_field_data'), - ('State_Defined_Field_Metadata'), - ('Treatment'), - ('NBS_configuration'), - ('LOOKUP_QUESTION') -) AS NewRows(TableName) -EXCEPT -SELECT name -FROM sys.tables -WHERE is_tracked_by_cdc = 1; - -DECLARE @tableName NVARCHAR(128); -DECLARE cur CURSOR FOR SELECT TableName FROM @tablesToEnable; - -OPEN cur; -FETCH NEXT FROM cur INTO @tableName; - -WHILE @@FETCH_STATUS = 0 -BEGIN - PRINT 'Enabling CDC for table: ' + @tableName; - BEGIN TRY - EXEC sys.sp_cdc_enable_table - @source_schema = N'dbo', - @source_name = @tableName, - @role_name = NULL; - END TRY - BEGIN CATCH - PRINT 'ERROR: Could not enable CDC for table ' + @tableName + '. Error: ' + ERROR_MESSAGE(); - END CATCH - - FETCH NEXT FROM cur INTO @tableName; -END - -CLOSE cur; -DEALLOCATE cur; -GO diff --git a/bootstrap/101-enable_cdc_on_odse_srte_databases-001.sql b/bootstrap/101-enable_cdc_on_odse_srte_databases-001.sql new file mode 100644 index 000000000..ba9fc1b02 --- /dev/null +++ b/bootstrap/101-enable_cdc_on_odse_srte_databases-001.sql @@ -0,0 +1,224 @@ +IF IS_SRVROLEMEMBER('sysadmin') <> 1 + BEGIN + THROW 50000, + 'This bootstrap script must be run by a SQL Server sysadmin. Please rerun it using an account with sysadmin permissions.', + 1; + END +GO + +-- ------------------------------------------ +-- 1. Enable CDC at Database Level - NBS_ODSE +-- ------------------------------------------ +IF ( + SELECT is_cdc_enabled FROM sys.databases + WHERE name = 'NBS_ODSE' +) = 0 + BEGIN + -- for aws + IF + EXISTS ( + SELECT 1 FROM sys.databases + WHERE name = 'rdsadmin' + ) + BEGIN + PRINT 'AWS RDS detected. Enabling CDC for NBS_ODSE using rds_cdc_enable_db'; + EXEC msdb.dbo.rds_cdc_enable_db 'NBS_ODSE'; + END + ELSE + BEGIN + PRINT 'Standard SQL Server detected. Enabling CDC for NBS_ODSE using sp_cdc_enable_db'; + EXEC ('USE [NBS_ODSE]; EXEC sys.sp_cdc_enable_db;'); + END + END +ELSE + BEGIN + PRINT 'CDC is already enabled for NBS_ODSE'; + END +GO + +-- ------------------------------------------ +-- 2. Enable CDC for Tables - NBS_ODSE +-- ------------------------------------------ +USE nbs_odse; +GO + +DECLARE @odseTablesToEnable TABLE (tablename NVARCHAR(128)); + +INSERT INTO @odseTablesToEnable (tablename) +SELECT tablename +FROM ( + VALUES + ('Act_relationship'), + ('Auth_user'), + ('CT_contact'), + ('Intervention'), + ('Interview'), + ('NBS_page'), + ('NBS_rdb_metadata'), + ('NBS_ui_metadata'), + ('Notification'), + ('Observation'), + ('Organization'), + ('Page_cond_mapping'), + ('Person'), + ('Place'), + ('Public_health_case'), + ('state_defined_field_data'), + ('State_Defined_Field_Metadata'), + ('Treatment'), + ('NBS_configuration'), + ('LOOKUP_QUESTION') +) AS newrows (tablename) +EXCEPT +SELECT name +FROM sys.tables +WHERE is_tracked_by_cdc = 1; + +DECLARE @odseTableName NVARCHAR(128); +DECLARE odseCur CURSOR FOR SELECT tablename FROM @odseTablesToEnable; + +OPEN odseCur; +FETCH NEXT FROM odseCur INTO @odseTableName; + +WHILE @@FETCH_STATUS = 0 + BEGIN + PRINT 'Enabling CDC for table: ' + @odseTableName; + BEGIN TRY + EXEC sys.sp_cdc_enable_table + @source_schema = N'dbo', + @source_name = @odseTableName, + @role_name = NULL; + END TRY + BEGIN CATCH + PRINT 'ERROR: Could not enable CDC for table ' + + @odseTableName + + '. Error: ' + + ERROR_MESSAGE(); + END CATCH + + FETCH NEXT FROM odseCur INTO @odseTableName; + END + +CLOSE odseCur; +DEALLOCATE odseCur; +GO + +-- ------------------------------------------ +-- 3. Enable CDC at Database Level - NBS_SRTE +-- ------------------------------------------ +IF ( + SELECT is_cdc_enabled FROM sys.databases + WHERE name = 'NBS_SRTE' +) = 0 + BEGIN + -- for aws + IF + EXISTS ( + SELECT 1 FROM sys.databases + WHERE name = 'rdsadmin' + ) + BEGIN + PRINT 'AWS RDS detected. Enabling CDC for NBS_SRTE using rds_cdc_enable_db'; + EXEC msdb.dbo.rds_cdc_enable_db 'NBS_SRTE'; + END + ELSE + BEGIN + PRINT 'Standard SQL Server detected. Enabling CDC for NBS_SRTE using sp_cdc_enable_db'; + EXEC ('USE [NBS_SRTE]; EXEC sys.sp_cdc_enable_db;'); + END + END +ELSE + BEGIN + PRINT 'CDC is already enabled for NBS_SRTE'; + END +GO + +-- ------------------------------------------ +-- 4. Enable CDC for Tables - NBS_SRTE +-- ------------------------------------------ +USE nbs_srte; +GO + +DECLARE @srteTablesToEnable TABLE (tablename NVARCHAR(128)); + +INSERT INTO @srteTablesToEnable (tablename) +SELECT tablename +FROM ( + VALUES + ('Anatomic_site_code'), + ('City_code_value'), + ('Cntycity_code_value'), + ('Code_value_clinical'), + ('Code_value_general'), + ('Codeset'), + ('Codeset_Group_Metadata'), + ('Condition_code'), + ('Country_code'), + ('Country_Code_ISO'), + ('Country_XREF'), + ('ELR_XREF'), + ('IMRDBMapping'), + ('Investigation_code'), + ('Jurisdiction_code'), + ('Jurisdiction_participation'), + ('Lab_coding_system'), + ('Lab_result'), + ('Lab_result_Snomed'), + ('Lab_test'), + ('Labtest_loinc'), + ('Labtest_Progarea_Mapping'), + ('Language_code'), + ('LDF_page_set'), + ('LOINC_code'), + ('Loinc_condition'), + ('Loinc_snomed_condition'), + ('NAICS_Industry_code'), + ('Occupation_code'), + ('Participation_type'), + ('Program_area_code'), + ('Race_code'), + ('Snomed_code'), + ('Specimen_source_code'), + ('Standard_XREF'), + ('State_code'), + ('State_county_code_value'), + ('State_model'), + ('TotalIDM'), + ('Treatment_code'), + ('Unit_code'), + ('Zip_code_value'), + ('Zipcnty_code_value') +) AS newrows (tablename) +EXCEPT +SELECT name +FROM sys.tables +WHERE is_tracked_by_cdc = 1; + +DECLARE @srteTableName NVARCHAR(128); +DECLARE srteCur CURSOR FOR SELECT tablename FROM @srteTablesToEnable; + +OPEN srteCur; +FETCH NEXT FROM srteCur INTO @srteTableName; + +WHILE @@FETCH_STATUS = 0 + BEGIN + PRINT 'Enabling CDC for table: ' + @srteTableName; + BEGIN TRY + EXEC sys.sp_cdc_enable_table + @source_schema = N'dbo', + @source_name = @srteTableName, + @role_name = NULL; + END TRY + BEGIN CATCH + PRINT 'ERROR: Could not enable CDC for table ' + + @srteTableName + + '. Error: ' + + ERROR_MESSAGE(); + END CATCH + + FETCH NEXT FROM srteCur INTO @srteTableName; + END + +CLOSE srteCur; +DEALLOCATE srteCur; +GO diff --git a/bootstrap/102-enable_cdc_on_srte_database-001.sql b/bootstrap/102-enable_cdc_on_srte_database-001.sql deleted file mode 100644 index 6f6c000c0..000000000 --- a/bootstrap/102-enable_cdc_on_srte_database-001.sql +++ /dev/null @@ -1,107 +0,0 @@ --- ------------------------------------------ --- 1. Enable CDC at Database Level --- ------------------------------------------ -IF (SELECT is_cdc_enabled FROM sys.databases WHERE name = 'NBS_SRTE') = 0 -BEGIN - IF EXISTS (SELECT 1 FROM sys.databases WHERE NAME = 'rdsadmin') -- for aws - BEGIN - PRINT 'AWS RDS detected. Enabling CDC for NBS_SRTE using rds_cdc_enable_db'; - EXEC msdb.dbo.rds_cdc_enable_db 'NBS_SRTE'; - END - ELSE - BEGIN - PRINT 'Standard SQL Server detected. Enabling CDC for NBS_SRTE using sp_cdc_enable_db'; - EXEC('USE [NBS_SRTE]; EXEC sys.sp_cdc_enable_db;'); - END -END -ELSE -BEGIN - PRINT 'CDC is already enabled for NBS_SRTE'; -END -GO - --- ------------------------------------------ --- 2. Enable CDC for Tables --- ------------------------------------------ -USE [NBS_SRTE]; -GO - -DECLARE @tablesToEnable TABLE (TableName NVARCHAR(128)); - -INSERT INTO @tablesToEnable (TableName) -SELECT TableName -FROM (VALUES - ('Anatomic_site_code'), - ('City_code_value'), - ('Cntycity_code_value'), - ('Code_value_clinical'), - ('Code_value_general'), - ('Codeset'), - ('Codeset_Group_Metadata'), - ('Condition_code'), - ('Country_code'), - ('Country_Code_ISO'), - ('Country_XREF'), - ('ELR_XREF'), - ('IMRDBMapping'), - ('Investigation_code'), - ('Jurisdiction_code'), - ('Jurisdiction_participation'), - ('Lab_coding_system'), - ('Lab_result'), - ('Lab_result_Snomed'), - ('Lab_test'), - ('Labtest_loinc'), - ('Labtest_Progarea_Mapping'), - ('Language_code'), - ('LDF_page_set'), - ('LOINC_code'), - ('Loinc_condition'), - ('Loinc_snomed_condition'), - ('NAICS_Industry_code'), - ('Occupation_code'), - ('Participation_type'), - ('Program_area_code'), - ('Race_code'), - ('Snomed_code'), - ('Specimen_source_code'), - ('Standard_XREF'), - ('State_code'), - ('State_county_code_value'), - ('State_model'), - ('TotalIDM'), - ('Treatment_code'), - ('Unit_code'), - ('Zip_code_value'), - ('Zipcnty_code_value') -) AS NewRows(TableName) -EXCEPT -SELECT name -FROM sys.tables -WHERE is_tracked_by_cdc = 1; - -DECLARE @tableName NVARCHAR(128); -DECLARE cur CURSOR FOR SELECT TableName FROM @tablesToEnable; - -OPEN cur; -FETCH NEXT FROM cur INTO @tableName; - -WHILE @@FETCH_STATUS = 0 -BEGIN - PRINT 'Enabling CDC for table: ' + @tableName; - BEGIN TRY - EXEC sys.sp_cdc_enable_table - @source_schema = N'dbo', - @source_name = @tableName, - @role_name = NULL; - END TRY - BEGIN CATCH - PRINT 'ERROR: Could not enable CDC for table ' + @tableName + '. Error: ' + ERROR_MESSAGE(); - END CATCH - - FETCH NEXT FROM cur INTO @tableName; -END - -CLOSE cur; -DEALLOCATE cur; -GO diff --git a/bootstrap/README.md b/bootstrap/README.md index ea2eb0f59..e45b31c1a 100644 --- a/bootstrap/README.md +++ b/bootstrap/README.md @@ -8,37 +8,32 @@ Running the RTR pipeline requires a SQL Server login account for two distinct pu ### Account Permissions (v7.13.0+) -| Account | Purpose | Databases | Key Permissions | -| :--- | :--- | :--- | :--- | -| **Service account** (e.g. `rtr-service-user`) | Application services reading source data and writing to the reporting database | `NBS_ODSE`, `NBS_SRTE`, `RDB / RDB_MODERN` | ODSE/SRTE: `db_datareader`
RDB/RDB_MODERN: `db_owner` | - -### Account Names Are Flexible - -Unlike the older per-service model (v7.12.0 and prior), **account names in v7.13.0+ are not hardcoded** — name them whatever fits your environment conventions. Just make sure the application configuration (Helm charts, `application.yaml`) reflects the chosen names and credentials. +1. **Name**: Any name works, but we recommend using a name descriptive to the role, such as `rtr-service-user`. +1. **Purpose**: The application services reading source database and writing to the reporting database. +1. **Databases Permissions**: + - `NBS_ODSE`: `db_datareader` + - `NBS_SRTE`: `db_datareader` + - `RDB` / `RDB_MODERN`: `db_owner` ### Who Runs the Bootstrap Scripts -The bootstrap scripts in this directory (CDC enablement, SQL Agent job creation) require the admin account. Specifically: +The bootstrap script in this directory (CDC enablement, SQL Agent job creation) require the admin account. Specifically: -- Scripts 101 and 102 call `sys.sp_cdc_enable_db` / `msdb.dbo.rds_cdc_enable_db` — requires `sysadmin` locally or `setupadmin` + CDC stored procedure execute rights on RDS. +- Script 101 checks for `sysadmin` and then calls `sys.sp_cdc_enable_db` / `msdb.dbo.rds_cdc_enable_db` — requires `sysadmin` locally or `setupadmin` + CDC stored procedure execute rights on RDS. --- ## Bootstrap Scripts -### 101 — Enable CDC on NBS_ODSE - -Enables Change Data Capture at the database level on `NBS_ODSE` and then enables CDC tracking on each RTR-relevant table. Handles both AWS RDS (`rds_cdc_enable_db`) and standard SQL Server (`sp_cdc_enable_db`) automatically. - -**Run against:** `NBS_ODSE` +### 101 — Enable CDC on NBS_ODSE + NBS_SRTE -### 102 — Enable CDC on NBS_SRTE +**File:** `101-enable_cdc_on_odse_srte_databases-001.sql` -Same as 101 but targets `NBS_SRTE` and its reference-data tables. +Enables Change Data Capture at the database level on `NBS_ODSE` and `NBS_SRTE`, then enables CDC tracking on each RTR-relevant table. Handles both AWS RDS (`rds_cdc_enable_db`) and standard SQL Server (`sp_cdc_enable_db`) automatically. -**Run against:** `NBS_SRTE` +**Run against:** `NBS_ODSE` and `NBS_SRTE` ## Prerequisites - The executing login must be the admin account described above -- Scripts 101 and 102 should be run before starting the Debezium connectors or the reporting-pipeline-service +- Script 101 should be run before starting the reporting-pipeline-service diff --git a/containers/db/initialize/002-fix-database-ownership.sql b/containers/db/initialize/002-fix-database-ownership.sql index c33c569eb..7e564d0e4 100644 --- a/containers/db/initialize/002-fix-database-ownership.sql +++ b/containers/db/initialize/002-fix-database-ownership.sql @@ -1,49 +1,48 @@ -- ========================================== --- Fix Database Ownership --- This prevents "Msg 15517: Principal 'dbo' does not exist" --- which occurs when a database is restored from a backup --- and the original owner principal is missing. +-- Dev/CI bootstrap +-- Ensures [sa] can enable CDC and owns the restored databases. -- ========================================== USE [master]; GO -DECLARE @DBName NVARCHAR(128); -DECLARE @OwnerName NVARCHAR(128) = 'sa'; - --- On AWS RDS, the master user should be the owner, not 'sa' --- We identify the master user by selecting the top enabled SQL login -IF EXISTS (SELECT 1 FROM sys.databases WHERE NAME = 'rdsadmin') -BEGIN - PRINT 'AWS RDS detected. Resolving master user for ownership...'; - SELECT TOP 1 @OwnerName = name - FROM sys.server_principals - WHERE type_desc = 'SQL_LOGIN' - AND is_disabled = 0 - AND name NOT IN ('rdsadmin', 'rdsa') - ORDER BY create_date; -END - -DECLARE db_cursor CURSOR FOR -SELECT name FROM sys.databases -WHERE name IN ('NBS_ODSE', 'NBS_SRTE', 'RDB', 'RDB_MODERN', 'NBS_MSGOUTE'); - -OPEN db_cursor; -FETCH NEXT FROM db_cursor INTO @DBName; +DECLARE @Databases TABLE (Name sysname PRIMARY KEY); + +INSERT INTO @Databases (Name) +VALUES +('NBS_ODSE'), +('NBS_SRTE'), +('RDB'), +('RDB_MODERN'), +('NBS_MSGOUTE'); + +DECLARE @DBName sysname; +DECLARE Db_cursor CURSOR LOCAL FAST_FORWARD FOR +SELECT Name +FROM @Databases +WHERE DB_ID(Name) IS NOT NULL; + +OPEN Db_cursor; +FETCH NEXT FROM Db_cursor INTO @DBName; WHILE @@FETCH_STATUS = 0 -BEGIN - PRINT 'Setting owner of database [' + @DBName + '] to [' + @OwnerName + ']'; - BEGIN TRY - EXEC('ALTER AUTHORIZATION ON DATABASE::[' + @DBName + '] TO [' + @OwnerName + ']'); - END TRY - BEGIN CATCH - PRINT 'WARNING: Could not set owner for ' + @DBName + '. Error: ' + ERROR_MESSAGE(); - END CATCH - - FETCH NEXT FROM db_cursor INTO @DBName; -END - -CLOSE db_cursor; -DEALLOCATE db_cursor; + BEGIN + BEGIN TRY + EXEC ( + N'ALTER AUTHORIZATION ON DATABASE::[' + @DBName + N'] TO [sa];' + ); + PRINT 'Set owner of database [' + @DBName + '] to [sa].'; + END TRY + BEGIN CATCH + PRINT 'WARNING: Could not set owner for [' + + @DBName + + ']. Error: ' + + ERROR_MESSAGE(); + END CATCH; + + FETCH NEXT FROM Db_cursor INTO @DBName; + END + +CLOSE Db_cursor; +DEALLOCATE Db_cursor; GO diff --git a/containers/db/initialize/README.md b/containers/db/initialize/README.md index e60f5faf4..d7f8e5e15 100644 --- a/containers/db/initialize/README.md +++ b/containers/db/initialize/README.md @@ -11,7 +11,7 @@ Custom scripts that prepare the SQL Server instance for local development and CI | File | Purpose | | :--- | :--- | | `001-restore-modern.sql` | Restores the RDB_MODERN database | -| `002-fix-database-ownership.sql` | Corrects database ownership settings | +| `002-fix-database-ownership.sql` | Grants `sa` sysadmin and corrects database ownership | | `003-fix-lab-test.sql` | Fixes lab test data | | `004-prep-for-masterEtl-trace.sql` | Prepares trace settings for MasterETL | | `005-clear-job_flow_log.sql` | Clears the job flow log table |