Sync Dataverse to Azure SQL DB.json

{"$schema":"http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#","contentVersion":"1.0.0.0","parameters":{"workspaceName":{"type":"string","metadata":"Workspace name","defaultValue":"sawstx"},"DestinationSQLDB":{"type":"string"},"SourceSynapseServerlessDB":{"type":"string"}},"variables":{"workspaceId":"[concat('Microsoft.Synapse/workspaces/', parameters('workspaceName'))]"},"resources":[{"name":"[concat(parameters('workspaceName'), '/Sync Orchestration')]","type":"Microsoft.Synapse/workspaces/pipelines","apiVersion":"2019-06-01-preview","properties":{"description":"Main pipeline that orchestrates the synchronization process and calls other pipelines as needed.","activities":[{"name":"Get tables to process","description":"Get a list of tables that need to be synchronized from a control table.","type":"Lookup","dependsOn":[{"activity":"Auto-Populate List of Tables to Sync","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"typeProperties":{"source":{"type":"AzureSqlSource","sqlReaderStoredProcedureName":"[[orchestration].[GetTablesToProcess]","storedProcedureParameters":{"TableGroup":{"type":"Int16","value":{"value":"@pipeline().parameters.TableGroupToSync","type":"Expression"}}},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"DataverseSQLDB","type":"DatasetReference","parameters":{"SchemaName":"n/a","TableName":"n/a"}},"firstRowOnly":false}},{"name":"For each table","description":"Determine whether to perform a full or an incremental copy and execute an appropriate child pipeline.","type":"ForEach","dependsOn":[{"activity":"Get tables to process","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"items":{"value":"@activity('Get tables to process').output.value","type":"Expression"},"isSequential":false,"activities":[{"name":"Separate Full Copy from Incremental Copy Tables","type":"IfCondition","dependsOn":[{"activity":"Execute Schema Drift Handler","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"expression":{"value":"@item().IsIncremental","type":"Expression"},"ifFalseActivities":[{"name":"Execute Full Copy Pipeline","type":"ExecutePipeline","dependsOn":[],"userProperties":[],"typeProperties":{"pipeline":{"referenceName":"Table Full Copy","type":"PipelineReference"},"waitOnCompletion":true,"parameters":{"SourceSchema":{"value":"@item().SourceSchema","type":"Expression"},"SourceTable":{"value":"@item().SourceTable","type":"Expression"},"TargetSchema":{"value":"@item().TargetSchema","type":"Expression"},"TargetTable":{"value":"@item().TargetTable","type":"Expression"},"TableId":{"value":"@item().TableId","type":"Expression"},"KeyColumns":{"value":"@item().KeyColumns","type":"Expression"}}}}],"ifTrueActivities":[{"name":"Execute Incremental Copy Pipeline","type":"ExecutePipeline","dependsOn":[],"userProperties":[],"typeProperties":{"pipeline":{"referenceName":"Table Incremental Copy","type":"PipelineReference"},"waitOnCompletion":true,"parameters":{"SourceSchema":{"value":"@item().SourceSchema","type":"Expression"},"SourceTable":{"value":"@item().SourceTable","type":"Expression"},"TargetSchema":{"value":"@item().TargetSchema","type":"Expression"},"TargetTable":{"value":"@item().TargetTable","type":"Expression"},"LowWatermark":{"value":"@item().LowWatermark","type":"Expression"},"HighWatermark":{"value":"@item().HighWatermark","type":"Expression"},"KeyColumn":{"value":"@item().KeyColumns","type":"Expression"},"TableId":{"value":"@item().TableId","type":"Expression"},"GeneratePrimaryKey":{"value":"@and(pipeline().parameters.AutoGeneratePrimaryKeys, item().NeedsPrimaryKey)","type":"Expression"}}}}]}},{"name":"Execute Schema Drift Handler","type":"ExecutePipeline","dependsOn":[],"userProperties":[],"typeProperties":{"pipeline":{"referenceName":"Handle Schema Drift","type":"PipelineReference"},"waitOnCompletion":true,"parameters":{"SourceSchema":{"value":"@item().SourceSchema","type":"Expression"},"SourceTable":{"value":"@item().SourceTable","type":"Expression"},"TargetSchema":{"value":"@item().TargetSchema","type":"Expression"},"TargetTable":{"value":"@item().TargetTable","type":"Expression"}}}},{"name":"Add a primary key to the target table","description":"If requested and necessary, add a primary key to the target table.","type":"IfCondition","dependsOn":[{"activity":"Separate Full Copy from Incremental Copy Tables","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"expression":{"value":"@and(pipeline().parameters.AutoGeneratePrimaryKeys, item().NeedsPrimaryKey)","type":"Expression"},"ifTrueActivities":[{"name":"Generate Primary Key","type":"SqlServerStoredProcedure","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"typeProperties":{"storedProcedureName":"[[orchestration].[GeneratePrimaryKey]","storedProcedureParameters":{"KeyColumns":{"value":{"value":"@item().KeyColumns","type":"Expression"},"type":"String"},"Schema":{"value":{"value":"@item().TargetSchema","type":"Expression"},"type":"String"},"Table":{"value":{"value":"@item().TargetTable","type":"Expression"},"type":"String"}}},"linkedServiceName":{"referenceName":"[parameters('DestinationSQLDB')]","type":"LinkedServiceReference"}}]}}]}},{"name":"Auto-Populate List of Tables to Sync","description":"Check pipeline parameter setting to determine whether to auto-populate list of tables to sync.","type":"IfCondition","dependsOn":[{"activity":"Add orchestration objects to target DB","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"expression":{"value":"@pipeline().parameters.AutoPopulateListOfTablesToSync","type":"Expression"},"ifTrueActivities":[{"name":"Populate List of Tables to Sync","type":"ExecutePipeline","dependsOn":[],"userProperties":[],"typeProperties":{"pipeline":{"referenceName":"Populate List of Tables to Sync","type":"PipelineReference"},"waitOnCompletion":true,"parameters":{"UsePartitionedTables":{"value":"@pipeline().parameters.UsePartitionedTables","type":"Expression"},"TargetSchema":{"value":"@pipeline().parameters.DefaultTargetSchema","type":"Expression"}}}}]}},{"name":"Add orchestration objects to target DB","description":"Add helper objects to the target database","type":"Script","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"linkedServiceName":{"referenceName":"[parameters('DestinationSQLDB')]","type":"LinkedServiceReference"},"typeProperties":{"scripts":[{"type":"Query","text":"/****** Object:  Schema [orchestration] ******/\nIF NOT EXISTS (SELECT name FROM sys.schemas WHERE name = N'orchestration')\nEXEC sp_executesql @stmt = N'CREATE SCHEMA [orchestration]'\n\n/****** Object:  Schema [staging] ******/\nIF NOT EXISTS (SELECT name FROM sys.schemas WHERE name = N'staging')\nEXEC sp_executesql @stmt = N'CREATE SCHEMA [staging]'\n\n/****** Object:  Table [orchestration].[ProcessingControl] ******/\nIF OBJECT_ID('[orchestration].[ProcessingControl]', 'U') IS NULL\nEXEC sp_executesql @stmt = N'\nCREATE TABLE [orchestration].[ProcessingControl](\n\t[TableId] [int] IDENTITY(1,1) NOT NULL,\n\t[SourceSchema] [nvarchar](128) NOT NULL,\n\t[SourceTable] [nvarchar](128) NOT NULL,\n\t[TargetSchema] [nvarchar](128) NOT NULL,\n\t[TargetTable] [nvarchar](128) NOT NULL,\n\t[KeyColumns] [nvarchar](4000) NULL,\n\t[IsIncremental] [bit] NOT NULL,\n\t[TableGroup] [smallint] NOT NULL DEFAULT (1),\n\t[IsEnabled] [bit] NOT NULL DEFAULT (1)\n CONSTRAINT [PK_ProcessingControl] PRIMARY KEY CLUSTERED \n(\n\t[TableId] ASC\n)WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]\n) ON [PRIMARY]\n'\n/****** Object:  Table [orchestration].[ProcessingLog] ******/\nIF OBJECT_ID('[orchestration].[ProcessingLog]', 'U') IS NULL\nEXEC sp_executesql @stmt = N'\nCREATE TABLE [orchestration].[ProcessingLog](\n\t[TableId] [int] NOT NULL,\n\t[PipelineRunId] [varchar](50) NOT NULL,\n\t[ProcessingStarted] [datetime2](7) NOT NULL,\n\t[ProcessingEnded] [datetime2](7) NULL,\n\t[LowWatermark] [datetime2](7) NULL,\n\t[HighWatermark] [datetime2](7) NULL,\n\t[RowsCopied] int NULL,\n\t[IsSuccessful] bit NULL\n CONSTRAINT [PK_ProcessingLog_1] PRIMARY KEY CLUSTERED \n(\n\t[TableId] ASC,\n\t[PipelineRunId] ASC\n)WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]\n) ON [PRIMARY]\n\nCREATE UNIQUE NONCLUSTERED INDEX [IX_ProcessingControl_TableNames] ON [orchestration].[ProcessingControl]\n(\n\t[SourceSchema] ASC,\n\t[SourceTable] ASC\n)WITH (STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, OPTIMIZE_FOR_SEQUENTIAL_KEY = OFF) ON [PRIMARY]\n'\n\n/****** Object:  StoredProcedure [orchestration].[GetTablesToProcess] ******/\nIF OBJECT_ID('[orchestration].[GetTablesToProcess]', 'P') IS NULL\nEXEC sp_executesql @stmt = N'\nCREATE PROCEDURE [orchestration].[GetTablesToProcess]\n\t@TableGroup smallint\n\nAS\n\nSELECT PC.TableId,\n\tPC.SourceSchema,\n\tPC.SourceTable,\n\tPC.TargetSchema,\n\tPC.TargetTable,\n\tPC.KeyColumns,\n\tISNULL(PC.IsIncremental, 0) AS IsIncremental,\n\tISNULL(MAX(PL.HighWatermark), ''1900-01-01'') AS LowWatermark,\n\tGETUTCDATE() AS HighWatermark,\n\tCAST(CASE WHEN PC.IsIncremental = 1 AND PK.CONSTRAINT_NAME IS NULL THEN 1 ELSE 0 END AS BIT) AS NeedsPrimaryKey\nFROM orchestration.ProcessingControl PC\n\tLEFT OUTER JOIN orchestration.ProcessingLog PL\n\t\tON PC.TableId= PL.TableId\n\t\tAND PL.ProcessingEnded IS NOT NULL\n\t\tAND PL.IsSuccessful = 1\n\tLEFT OUTER JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS PK\n\t\tON PC.TargetSchema = PK.TABLE_SCHEMA\n\t\tAND PC.TargetTable = PK.TABLE_NAME\n\t\tAND PK.CONSTRAINT_TYPE = ''Primary Key''\nWHERE PC.TableGroup = @TableGroup\n\tAND PC.IsEnabled = 1\nGROUP BY PC.TableId,\n\tPC.SourceSchema,\n\tPC.SourceTable,\n\tPC.TargetSchema,\n\tPC.TargetTable,\n\tPC.KeyColumns,\n\tPC.IsIncremental,\n\tPK.CONSTRAINT_NAME\n'\n\n/****** Object:  StoredProcedure [orchestration].[GeneratePrimaryKey] ******/\nIF OBJECT_ID('[orchestration].[GeneratePrimaryKey]', 'P') IS NULL\nEXEC sp_executesql @stmt = N'\nCREATE OR ALTER PROCEDURE [orchestration].[GeneratePrimaryKey]\n\t@Schema nvarchar(128), \n\t@Table nvarchar(128), \n\t@KeyColumns nvarchar(MAX)\n\nAS\n\n\n--Make sure that a Primary Key constraint does not exist on the specified table\nIF NOT EXISTS (\n\tSELECT * \n\tFROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS\n\tWHERE CONSTRAINT_TYPE = ''Primary Key''\n\tAND TABLE_SCHEMA = @Schema\n\tAND TABLE_NAME = @Table\n)\n\nBEGIN\n\t--Parse and reformat the list of key columns in a manner that will be suitable for subsequent processing\n\tDECLARE @KeyColumnsFormatted nvarchar(MAX) \n\tSELECT @KeyColumnsFormatted =  STRING_AGG(QUOTENAME([value]), '','') FROM STRING_SPLIT(@KeyColumns, '','')\n\n\tDECLARE @SQL nvarchar(MAX) = ''''\n\n\t--Make key columns non-nullable\n\tSELECT @SQL = @SQL + ''ALTER TABLE [''+ @Schema + ''].['' + @Table + ''] ALTER COLUMN ['' + name + ''] '' + system_type_name + '' NOT NULL; ''\n\tFROM sys.dm_exec_describe_first_result_set(''SELECT '' + @KeyColumnsFormatted + '' FROM  [''+ @Schema + ''].['' + @Table + '']'' , NULL, NULL)\n\tEXEC sp_executesql @stmt = @SQL\n\t\n\t--Add primary key\n\tSET @SQL = ''ALTER TABLE [''+ @Schema + ''].['' + @Table + ''] ADD CONSTRAINT [PK_'' + @Schema + ''_'' + @Table + ''] PRIMARY KEY CLUSTERED ('' + @KeyColumnsFormatted + '')''\n\tEXEC sp_executesql @stmt = @SQL\nEND\n'\n\n/****** Object:  StoredProcedure [orchestration].[SchemaDriftHandler] ******/\nIF OBJECT_ID('[orchestration].[SchemaDriftHandler]', 'P') IS NULL\nEXEC sp_executesql @stmt = N'\nCREATE OR ALTER PROCEDURE [orchestration].[SchemaDriftHandler] \n\t@Schema nvarchar(128), \n\t@Table nvarchar(128), \n\t@ColumnDefinitions nvarchar(MAX)\n\nAS\n\nDECLARE @AddSQL nvarchar(MAX), @AlterSQL nvarchar(MAX)\n\n--Parse the column definition parameter passed from the source, identify columns that do not exist in the target, and generate a SQL statement for adding the columns\nIF OBJECT_ID(''[''+ @Schema + ''].['' + @Table + '']'', ''U'') IS NOT NULL\nBEGIN\n\tSELECT @AddSQL = ''ALTER TABLE [''+ @Schema + ''].['' + @Table + ''] ADD '' + STRING_AGG(CAST(QUOTENAME([Source].ColumnName) + '' '' + [Source].Type AS nvarchar(MAX)), '', '')\n\tFROM OPENJSON(@ColumnDefinitions)\n\t\tWITH (\n\t\t\tColumnName varchar(128) ''$.ColumnName'',\n\t\t\tType varchar(128) ''$.Type''\n\t\t)  AS [Source]\n\t\tLEFT JOIN INFORMATION_SCHEMA.COLUMNS AS [Target]\n\t\t\tON [Target].TABLE_SCHEMA = @Schema\n\t\t\tAND [Target].TABLE_NAME = @Table\n\t\t\tAND [Source].ColumnName = [Target].COLUMN_NAME\n\tWHERE [Target].COLUMN_NAME IS NULL\nEND\n\n--If the SQL statement is not null and the target table exists, execute the SQL statement\nIF @AddSQL IS NOT NULL AND OBJECT_ID(''[''+ @Schema + ''].['' + @Table + '']'', ''U'') IS NOT NULL\nBEGIN\n\tEXEC sp_executesql @stmt = @AddSQL\nEND\n\n--Update the data type of the SyncedToSqlDbOn column added to the destination tables\nSELECT @AlterSQL = ''ALTER TABLE [''+ @Schema + ''].['' + @Table + ''] ALTER COLUMN SyncedToSqlDbOn DateTime2''\n\tFROM INFORMATION_SCHEMA.COLUMNS AS [Target]\n\tWHERE [Target].TABLE_SCHEMA = @Schema\n\t\tAND [Target].TABLE_NAME = @Table\n\t\tAND [Target].COLUMN_NAME = ''SyncedToSqlDbOn''\n\t\tAND [Target].DATA_TYPE <> ''datetime2''\n\n--If the SQL statement is not null\nIF @AlterSQL IS NOT NULL\nBEGIN\n\tEXEC sp_executesql @stmt = @AlterSQL\nEND\n'"}]}}],"policy":{"elapsedTimeMetric":{},"cancelAfter":{}},"parameters":{"AutoPopulateListOfTablesToSync":{"type":"bool","defaultValue":true},"AutoGeneratePrimaryKeys":{"type":"bool","defaultValue":true},"TableGroupToSync":{"type":"int","defaultValue":1},"UsePartitionedTables":{"type":"bool","defaultValue":false},"DefaultTargetSchema":{"type":"string","defaultValue":"dbo"}},"variables":{"Structure":{"type":"Array"}},"folder":{"name":"Dataverse - Synapse Serverless to SQLDB"},"annotations":[],"lastPublishTime":"2022-09-16T13:55:00Z"},"dependsOn":["[concat(variables('workspaceId'), '/datasets/DataverseSQLDB')]","[concat(variables('workspaceId'), '/pipelines/Handle Schema Drift')]","[concat(variables('workspaceId'), '/pipelines/Populate List of Tables to Sync')]","[concat(variables('workspaceId'), '/pipelines/Table Full Copy')]","[concat(variables('workspaceId'), '/pipelines/Table Incremental Copy')]"]},{"name":"[concat(parameters('workspaceName'), '/DataverseSQLDB')]","type":"Microsoft.Synapse/workspaces/datasets","apiVersion":"2019-06-01-preview","properties":{"linkedServiceName":{"referenceName":"[parameters('DestinationSQLDB')]","type":"LinkedServiceReference"},"parameters":{"SchemaName":{"type":"string"},"TableName":{"type":"string"}},"annotations":[],"type":"AzureSqlTable","schema":[],"typeProperties":{"schema":{"value":"@dataset().SchemaName","type":"Expression"},"table":{"value":"@dataset().TableName","type":"Expression"}}},"dependsOn":[]},{"name":"[concat(parameters('workspaceName'), '/Handle Schema Drift')]","type":"Microsoft.Synapse/workspaces/pipelines","apiVersion":"2019-06-01-preview","properties":{"description":"Detect schema drift and, if necessary, add new columns to the target table.","activities":[{"name":"Execute stored procedure","description":"Execute the orchestration.SchemaDriftAddColumns procedure","type":"Script","dependsOn":[{"activity":"For each column","dependencyConditions":["Succeeded"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"linkedServiceName":{"referenceName":"[parameters('DestinationSQLDB')]","type":"LinkedServiceReference"},"typeProperties":{"scripts":[{"type":"Query","text":{"value":"@concat('EXEC [orchestration].[SchemaDriftHandler] \n\t@Schema = ''', pipeline().parameters.TargetSchema,''',\n\t@Table = ''',pipeline().parameters.TargetTable,''',\n\t@ColumnDefinitions =''', string(variables('ColumnDefinitions')), '''')","type":"Expression"}}]}},{"name":"Get source table definition","type":"Lookup","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"typeProperties":{"source":{"type":"AzureSqlSource","sqlReaderQuery":{"value":"@concat('EXEC sp_describe_first_result_set @tsql = N''SELECT TOP 1 * FROM [',pipeline().parameters.SourceSchema,'].[',pipeline().parameters.SourceTable,']''')","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"dataset":{"referenceName":"DataverseSynapseServerlessDB","type":"DatasetReference","parameters":{"TableName":"n/a","SchemaName":"n/a"}},"firstRowOnly":false}},{"name":"For each column","description":"Iterate over each column in the source table","type":"ForEach","dependsOn":[{"activity":"Get source table definition","dependencyConditions":["Succeeded"]}],"userProperties":[],"typeProperties":{"items":{"value":"@activity('Get source table definition').output.value","type":"Expression"},"batchCount":50,"activities":[{"name":"Append column definition","description":"Append column name and data type to an array variable","type":"AppendVariable","dependsOn":[],"userProperties":[],"typeProperties":{"variableName":"ColumnDefinitions","value":{"value":"@json(concat('{\"ColumnName\":\"', item().name, '\", \"Type\":\"', item().system_type_name, '\"}'))","type":"Expression"}}}]}}],"policy":{"elapsedTimeMetric":{},"cancelAfter":{}},"parameters":{"SourceSchema":{"type":"string"},"SourceTable":{"type":"string"},"TargetSchema":{"type":"string"},"TargetTable":{"type":"string"}},"variables":{"Structure":{"type":"Array"},"ColumnDefinitions":{"type":"Array"}},"folder":{"name":"Dataverse - Synapse Serverless to SQLDB"},"annotations":[],"lastPublishTime":"2022-09-16T13:54:37Z"},"dependsOn":["[concat(variables('workspaceId'), '/datasets/DataverseSynapseServerlessDB')]"]},{"name":"[concat(parameters('workspaceName'), '/Populate List of Tables to Sync')]","type":"Microsoft.Synapse/workspaces/pipelines","apiVersion":"2019-06-01-preview","properties":{"description":"Identify tables in the source database that appear to be suitable for incremental or full copy behavior and load them into the orchestration.ProcessingControl table.\n\nNote: review the list of tables in the orchestration.ProcessingControl table to ensure that the list matches your requirements.","activities":[{"name":"Copy list of tables to process","description":"Identify tables in the source database that have the suffixes of \"_partitioned\" and certain metadata tables, determine whether they appear to be suitable for incremental or full copy behavior and upsert them into the orchestration.ProcessingControl table.","type":"Copy","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"typeProperties":{"source":{"type":"AzureSqlSource","sqlReaderQuery":{"value":"@replace(replace('--Specify whether the solution should use the \"_partitioned\" tables as its source\nDECLARE @UsePartitionedTables BIT = UsePartitionedTablesParameter\n\n--Get a list of tables with the suffixes of \"_partitioned\" and certain metadata tables from the dbo schema\n; WITH DiscoveredTables AS (\nSELECT T.TABLE_SCHEMA AS SourceSchema,\n\tT.TABLE_NAME AS SourceTable,\n\tIIF(T.TABLE_NAME LIKE ''%_partitioned'', LEFT(T.TABLE_NAME, LEN(T.TABLE_NAME)-12), T.TABLE_NAME) AS TargetTable,\n\t--If the table contains columns Id, IsDeleted, SinkModifiedOn and versionnumber treat it as an incrementally-updated data table\n\tIIF(SUM(IIF(C.COLUMN_NAME IN (''Id'', ''IsDelete'', ''SinkModifiedOn'', ''versionnumber''), 1, 0)) = 4, 1, 0) AS IsIncrementalDataTable\nFROM INFORMATION_SCHEMA.TABLES T\n\tJOIN INFORMATION_SCHEMA.COLUMNS C\n\t\tON T.TABLE_SCHEMA= C.TABLE_SCHEMA\n\t\tAND T.TABLE_NAME = C.TABLE_NAME\nWHERE T.TABLE_SCHEMA = ''dbo''\n\tAND (\n\t\t(@UsePartitionedTables = 0 AND T.TABLE_NAME NOT LIKE ''%_partitioned'')\n\t\tOR (@UsePartitionedTables = 1 AND T.TABLE_NAME LIKE ''%_partitioned'')\n\t\tOR T.TABLE_NAME IN (''StateMetadata'', ''StatusMetadata'', ''TargetMetadata'', ''OptionsetMetadata'', ''GlobalOptionsetMetadata'')\n\t\t)\nGROUP BY T.TABLE_SCHEMA,\n\tT.TABLE_NAME\n)\n\nSELECT  T.SourceSchema,\n\tT.SourceTable,\n\t''TargetSchemaParameter'' AS TargetSchema,\n\tT.TargetTable,\n\t--If the table contains columns Id and IsDeleted, treat it as an incrementally-updated table with a KeyColumnName of Id\n\t--Otherwise, assume that it requires a full refresh\n\tCASE WHEN \n\t\tT.IsIncrementalDataTable = 1 THEN ''Id'' \n\t\tWHEN T.SourceTable = ''GlobalOptionsetMetadata'' THEN ''GlobalOptionSetName,Option,LocalizedLabelLanguageCode'' \n\t\tWHEN T.SourceTable = ''OptionsetMetadata'' THEN ''EntityName,OptionSetName,Option,LocalizedLabelLanguageCode'' \n\t\tWHEN T.SourceTable = ''StateMetadata'' THEN ''EntityName,State,LocalizedLabelLanguageCode'' \n\t\tWHEN T.SourceTable = ''StatusMetadata'' THEN ''EntityName,Status,LocalizedLabelLanguageCode'' \n\t\tWHEN T.SourceTable = ''TargetMetadata'' THEN ''EntityName,AttributeName,ReferencedEntity,ReferencedAttribute'' \n\t\tEND AS KeyColumns,\n\tCAST(IIF(T.IsIncrementalDataTable = 1, 1, 0) AS BIT) AS IsIncremental\nFROM DiscoveredTables T\nWHERE T.IsIncrementalDataTable = 1 OR T.SourceTable IN (''StateMetadata'', ''StatusMetadata'', ''TargetMetadata'', ''OptionsetMetadata'', ''GlobalOptionsetMetadata'')\n', 'UsePartitionedTablesParameter', if(pipeline().parameters.UsePartitionedTables, '1', '0')), 'TargetSchemaParameter', pipeline().parameters.TargetSchema)","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"sink":{"type":"AzureSqlSink","writeBehavior":"upsert","upsertSettings":{"useTempDB":false,"keys":["SourceSchema","SourceTable"],"interimSchemaName":"staging"},"sqlWriterUseTableLock":false,"disableMetricsCollection":false},"enableStaging":false,"translator":{"type":"TabularTranslator","typeConversion":true,"typeConversionSettings":{"allowDataTruncation":true,"treatBooleanAsNumber":false}}},"inputs":[{"referenceName":"DataverseSynapseServerlessDB","type":"DatasetReference","parameters":{"TableName":"n/a","SchemaName":"n/a"}}],"outputs":[{"referenceName":"DataverseSQLDB","type":"DatasetReference","parameters":{"SchemaName":"orchestration","TableName":"ProcessingControl"}}]}],"policy":{"elapsedTimeMetric":{},"cancelAfter":{}},"parameters":{"UsePartitionedTables":{"type":"bool","defaultValue":true},"TargetSchema":{"type":"string","defaultValue":"dbo"}},"folder":{"name":"Dataverse - Synapse Serverless to SQLDB"},"annotations":[],"lastPublishTime":"2022-09-16T13:54:44Z"},"dependsOn":["[concat(variables('workspaceId'), '/datasets/DataverseSynapseServerlessDB')]","[concat(variables('workspaceId'), '/datasets/DataverseSQLDB')]"]},{"name":"[concat(parameters('workspaceName'), '/Table Full Copy')]","type":"Microsoft.Synapse/workspaces/pipelines","apiVersion":"2019-06-01-preview","properties":{"description":"Perform a full copy of a table (truncate and reload the destination table)","activities":[{"name":"Copy and Upsert Data","description":"Retrieve all data from the source table and upsert it into the destination table. Create a new table (if it does not exist). Log the start of a sync process.\n\nNote: the read behavior involves reading all data from source. The write behavior is an upsert.","type":"Copy","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":3,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"typeProperties":{"source":{"type":"AzureSqlSource","additionalColumns":[{"name":"SyncedToSqlDbOn","value":{"value":"@pipeline().TriggerTime","type":"Expression"}}],"queryTimeout":"02:00:00","partitionOption":"None"},"sink":{"type":"AzureSqlSink","preCopyScript":{"value":"@{concat('INSERT INTO [orchestration].[ProcessingLog]\n           ([TableId],\n           [PipelineRunId],\n           [ProcessingStarted])\n     VALUES\n           (''', string(pipeline().parameters.TableId), ''',\n           ''', pipeline().RunId, ''',\n           ''', string(pipeline().TriggerTime), ''')\n           ')}","type":"Expression"},"writeBehavior":"upsert","upsertSettings":{"useTempDB":true,"keys":{"value":"@split(pipeline().parameters.KeyColumns, ',')","type":"Expression"}},"sqlWriterUseTableLock":false,"tableOption":"autoCreate","disableMetricsCollection":false},"enableStaging":false,"translator":{"type":"TabularTranslator","typeConversion":true,"typeConversionSettings":{"allowDataTruncation":true,"treatBooleanAsNumber":false}}},"inputs":[{"referenceName":"DataverseSynapseServerlessDB","type":"DatasetReference","parameters":{"TableName":{"value":"@pipeline().parameters.SourceTable","type":"Expression"},"SchemaName":{"value":"@pipeline().parameters.SourceSchema","type":"Expression"}}}],"outputs":[{"referenceName":"DataverseSQLDB","type":"DatasetReference","parameters":{"SchemaName":{"value":"@pipeline().parameters.TargetSchema","type":"Expression"},"TableName":{"value":"@pipeline().parameters.TargetTable","type":"Expression"}}}]},{"name":"Log successful processing end","description":"Update a log record to indicate successful completion of the sync process.","type":"Script","dependsOn":[{"activity":"Copy and Upsert Data","dependencyConditions":["Completed"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"linkedServiceName":{"referenceName":"[parameters('DestinationSQLDB')]","type":"LinkedServiceReference"},"typeProperties":{"scripts":[{"type":"Query","text":{"value":"@concat('\nUPDATE orchestration.ProcessingLog\nSET ProcessingEnded = GETUTCDATE(),\n    IsSuccessful = ', string(if(equals(activity('Copy and Upsert Data').output.executionDetails[0].status, 'Succeeded'), 1, 0)), ',\n    RowsCopied = ', string(activity('Copy and Upsert Data').output.rowsCopied), '\nWHERE TableId = ', string(pipeline().parameters.TableId),' \n    AND PipelineRunId = ''', pipeline().RunId, '''')","type":"Expression"}}]}},{"name":"Fail pipeline","description":"Fail pipeline execution in the event of copy activity failure.","type":"Fail","dependsOn":[{"activity":"Log successful processing end","dependencyConditions":["Completed"]},{"activity":"Copy and Upsert Data","dependencyConditions":["Failed"]}],"userProperties":[],"typeProperties":{"message":"1000","errorCode":"Failed pipeline execution due to copy activity failure."}}],"policy":{"elapsedTimeMetric":{},"cancelAfter":{}},"parameters":{"SourceSchema":{"type":"string","defaultValue":"dbo"},"SourceTable":{"type":"string"},"TargetSchema":{"type":"string","defaultValue":"dbo"},"TargetTable":{"type":"string"},"TableId":{"type":"int"},"KeyColumns":{"type":"string"}},"folder":{"name":"Dataverse - Synapse Serverless to SQLDB"},"annotations":[],"lastPublishTime":"2022-09-16T13:54:51Z"},"dependsOn":["[concat(variables('workspaceId'), '/datasets/DataverseSynapseServerlessDB')]","[concat(variables('workspaceId'), '/datasets/DataverseSQLDB')]"]},{"name":"[concat(parameters('workspaceName'), '/Table Incremental Copy')]","type":"Microsoft.Synapse/workspaces/pipelines","apiVersion":"2019-06-01-preview","properties":{"description":"Perform an incremental copy of a table (upsert)","activities":[{"name":"Copy and Upsert Data","description":"Retrieve and de-duplicate data from the source table and upsert it into the destination table. Create the destination table if it does not exist. Log the start of a sync process.","type":"Copy","dependsOn":[],"policy":{"timeout":"7.00:00:00","retry":3,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"typeProperties":{"source":{"type":"AzureSqlSource","additionalColumns":[{"name":"SyncedToSqlDbOn","value":{"value":"@pipeline().TriggerTime","type":"Expression"}}],"sqlReaderQuery":{"value":"@replace(replace(replace(replace(replace('\n--Get a set of latest records from a table modified between specific watermark dates\n\n; WITH CurrentRecord AS (\nSELECT [KeyColumn], \n\tMAX(versionnumber) AS versionnumber,\n\tMAX(SinkModifiedOn) AS SinkModifiedOn\nFROM [SourceSchema].[SourceTable]\nWHERE SinkModifiedOn >= ''1900-01-01'' \n\tAND SinkModifiedOn < ''2999-12-31''\nGROUP BY [KeyColumn]\n)\n\nSELECT BaseTable.*\nFROM [SourceSchema].[SourceTable] AS BaseTable\n\tJOIN CurrentRecord\n\t\tON BaseTable.[KeyColumn] = CurrentRecord.[KeyColumn]\n\t\tAND BaseTable.versionnumber = CurrentRecord.versionnumber\n\t\tAND BaseTable.SinkModifiedOn = CurrentRecord.SinkModifiedOn\n', \n'SourceSchema', pipeline().parameters.SourceSchema),\n'SourceTable', pipeline().parameters.SourceTable),\n\n'1900-01-01', pipeline().parameters.LowWatermark),\n'2099-12-31', pipeline().parameters.HighWatermark),\n'KeyColumn', pipeline().parameters.KeyColumn)","type":"Expression"},"queryTimeout":"02:00:00","partitionOption":"None"},"sink":{"type":"AzureSqlSink","preCopyScript":{"value":"@{concat('INSERT INTO [orchestration].[ProcessingLog]\n           ([TableId],\n           [PipelineRunId],\n           [ProcessingStarted],\n           [LowWatermark],\n           [HighWatermark])\n     VALUES\n           (''', string(pipeline().parameters.TableId), ''',\n           ''', pipeline().RunId, ''',\n           ''', string(pipeline().TriggerTime), ''',\n           ''', pipeline().parameters.LowWatermark, ''',\n           ''', pipeline().parameters.HighWatermark, ''')')}","type":"Expression"},"writeBehavior":"upsert","upsertSettings":{"useTempDB":false,"keys":{"value":"@split(pipeline().parameters.KeyColumn, ',')","type":"Expression"},"interimSchemaName":"staging"},"sqlWriterUseTableLock":false,"tableOption":"autoCreate","disableMetricsCollection":false},"enableStaging":false,"translator":{"type":"TabularTranslator","typeConversion":true,"typeConversionSettings":{"allowDataTruncation":true,"treatBooleanAsNumber":false}}},"inputs":[{"referenceName":"DataverseSynapseServerlessDB","type":"DatasetReference","parameters":{"TableName":"n/a","SchemaName":"n/a"}}],"outputs":[{"referenceName":"DataverseSQLDB","type":"DatasetReference","parameters":{"SchemaName":{"value":"@pipeline().parameters.TargetSchema","type":"Expression"},"TableName":{"value":"@pipeline().parameters.TargetTable","type":"Expression"}}}]},{"name":"Log processing end","description":"Update a log record to indicate successful completion of the sync process.","type":"Script","dependsOn":[{"activity":"Copy and Upsert Data","dependencyConditions":["Completed"]}],"policy":{"timeout":"7.00:00:00","retry":0,"retryIntervalInSeconds":30,"secureOutput":false,"secureInput":false},"userProperties":[],"linkedServiceName":{"referenceName":"[parameters('DestinationSQLDB')]","type":"LinkedServiceReference"},"typeProperties":{"scripts":[{"type":"Query","text":{"value":"@concat('\nUPDATE orchestration.ProcessingLog\nSET ProcessingEnded = GETUTCDATE(),\n    IsSuccessful = ', string(if(equals(activity('Copy and Upsert Data').output.executionDetails[0].status, 'Succeeded'), 1, 0)), ',\n    RowsCopied = ', string(activity('Copy and Upsert Data').output.rowsCopied), '\nWHERE TableId = ', string(pipeline().parameters.TableId),' \n    AND PipelineRunId = ''', pipeline().RunId, '''')","type":"Expression"}}]}},{"name":"Fail pipeline","description":"Fail pipeline execution in the event of copy activity failure.","type":"Fail","dependsOn":[{"activity":"Copy and Upsert Data","dependencyConditions":["Failed"]},{"activity":"Log processing end","dependencyConditions":["Completed"]}],"userProperties":[],"typeProperties":{"message":"1000","errorCode":"Failed pipeline execution due to copy activity failure."}}],"policy":{"elapsedTimeMetric":{},"cancelAfter":{}},"parameters":{"SourceSchema":{"type":"string","defaultValue":"dbo"},"SourceTable":{"type":"string"},"TargetSchema":{"type":"string","defaultValue":"dbo"},"TargetTable":{"type":"string"},"LowWatermark":{"type":"string","defaultValue":"1900-01-01"},"HighWatermark":{"type":"string","defaultValue":"2999-12-31"},"KeyColumn":{"type":"string","defaultValue":"Id"},"TableId":{"type":"int"},"GeneratePrimaryKey":{"type":"bool"}},"folder":{"name":"Dataverse - Synapse Serverless to SQLDB"},"annotations":[],"lastPublishTime":"2022-09-16T13:54:55Z"},"dependsOn":["[concat(variables('workspaceId'), '/datasets/DataverseSynapseServerlessDB')]","[concat(variables('workspaceId'), '/datasets/DataverseSQLDB')]"]},{"name":"[concat(parameters('workspaceName'), '/DataverseSynapseServerlessDB')]","type":"Microsoft.Synapse/workspaces/datasets","apiVersion":"2019-06-01-preview","properties":{"linkedServiceName":{"referenceName":"[parameters('SourceSynapseServerlessDB')]","type":"LinkedServiceReference"},"parameters":{"TableName":{"type":"string"},"SchemaName":{"type":"string"}},"annotations":[],"type":"AzureSqlTable","schema":[],"typeProperties":{"schema":{"value":"@dataset().SchemaName","type":"Expression"},"table":{"value":"@dataset().TableName","type":"Expression"}}},"dependsOn":[]}]}