Skip to content

Commit a72c50d

Browse files
sjohanssonsjohansson
committed
updates to end sp logic
- add more fulsome processing in EndModule and EndBatch - add tests for these sp's - tweaked approach for parameter logging to add single json of parameters into log - tweaked row number logging to be more conditionally automatic in end module
1 parent f1ea0fe commit a72c50d

11 files changed

Lines changed: 1082 additions & 197 deletions

‎Direct_Framework/Stored Procedures/omd.EndBatchInstance.sql‎

Lines changed: 70 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ CREATE PROCEDURE [omd].[EndBatchInstance]
4747
-- Required parameters
4848
@BatchInstanceId BIGINT = NULL
4949
-- Optional parameters
50-
,@EventCode NVARCHAR(100) = NULL
50+
,@EventCode NVARCHAR(100) = 'Failure'
5151
,@Debug CHAR(1) = 'N'
5252
-- Output parameters
5353
,@SuccessIndicator CHAR(1) = 'N' OUTPUT
@@ -64,6 +64,7 @@ BEGIN
6464
SET @SuccessIndicator = 'N';
6565
SET @MessageLog = N'[]';
6666
DECLARE @LogMessage NVARCHAR(2048);
67+
DECLARE @ProcessDescription NVARCHAR(4000) = N'End Batch Instance process';
6768

6869
/* Event and return codes */
6970
DECLARE @ReturnCode INT = 0;
@@ -80,7 +81,7 @@ BEGIN
8081
DECLARE @FailBatchOnModuleFailure CHAR(1) = [omd_metadata].[GetSettingFlag]('FAIL_BATCH_ON_MODULE_FAILURE');
8182
DECLARE @EndModuleOnBatchEnd CHAR(1) = [omd_metadata].[GetSettingFlag]('END_MODULE_ON_BATCH_END');
8283

83-
/* ----- Validate input parameters ------------------------------------------ */
84+
/* ----- Validate input parameters -------------------------------------- */
8485

8586
IF @BatchInstanceId IS NULL OR @BatchInstanceId <= 0
8687
BEGIN
@@ -144,13 +145,21 @@ BEGIN
144145
[omd_metadata].[GetFrameworkVersion](), @MessageLog);
145146
SET @MessageLog = [omd].[AddLogMessage]('DEBUG', DEFAULT, N'Start Timestamp',
146147
@StartTimestampString, @MessageLog);
147-
/* Log parameters */
148-
SET @MessageLog = [omd].[AddLogMessage]('INFO', DEFAULT, N'Parameter @BatchInstanceId',
149-
@BatchInstanceId, @MessageLog);
150-
SET @MessageLog = [omd].[AddLogMessage]('INFO', DEFAULT, N'Parameter @EventCode',
151-
@EventCode, @MessageLog);
148+
149+
/* Log parameters in a single JSON block */
150+
DECLARE @paramsJson NVARCHAR(MAX) = (
151+
SELECT
152+
@BatchInstanceId AS BatchInstanceId,
153+
@EventCode AS EventCode,
154+
@Debug AS Debug
155+
FOR JSON PATH, WITHOUT_ARRAY_WRAPPER);
156+
157+
SET @MessageLog = [omd].[AddLogMessage]('INFO', DEFAULT, N'Parameters',
158+
@paramsJson, @MessageLog);
152159
END
153160

161+
DECLARE @RowsAffected INT = 0;
162+
154163
/* ----- Start of main process ---------------------------------------------- */
155164
BEGIN TRY
156165
BEGIN TRANSACTION;
@@ -189,120 +198,69 @@ BEGIN
189198
) SET @EventCode = N'Failure';
190199
END
191200

192-
-- Abort event
193-
-- This is an end-state event (no further processing)
194-
IF @EventCode = N'Abort'
201+
SET @LogMessage = CONCAT(N'Setting the Batch Instance ', @BatchInstanceId,
202+
N' to ', @EventCode, N'.');
203+
SET @MessageLog = [omd].[AddLogMessage](DEFAULT, DEFAULT, N'Status Update',
204+
@LogMessage, @MessageLog);
205+
206+
;WITH StatusMap AS (
207+
SELECT *
208+
FROM (VALUES
209+
(N'Abort', N'Aborted', N'Abort', N'Proceed', 1),
210+
(N'Cancel', N'Cancelled', N'Cancel', N'Proceed', 1),
211+
(N'Success', N'Succeeded', N'Proceed', N'Proceed', 1),
212+
(N'Failure', N'Failed', NULL, N'Proceed', 1),
213+
(N'Rollback', NULL, N'Rollback', NULL, 0),
214+
(N'Proceed', NULL, N'Proceed', NULL, 0)
215+
) AS map(EventCode, ExecutionStatusCode, InternalProcessingCode,
216+
NextRunStatusCode, ApplyEndTimestamp)
217+
)
218+
UPDATE bi
219+
SET
220+
EXECUTION_STATUS_CODE = COALESCE(sm.ExecutionStatusCode, bi.EXECUTION_STATUS_CODE),
221+
INTERNAL_PROCESSING_CODE = COALESCE(sm.InternalProcessingCode, bi.INTERNAL_PROCESSING_CODE),
222+
NEXT_RUN_STATUS_CODE = COALESCE(sm.NextRunStatusCode, bi.NEXT_RUN_STATUS_CODE),
223+
END_TIMESTAMP = CASE WHEN sm.ApplyEndTimestamp = 1 THEN @StartTimestamp ELSE bi.END_TIMESTAMP END
224+
FROM [omd].[BATCH_INSTANCE] AS bi
225+
INNER JOIN StatusMap AS sm
226+
ON sm.EventCode = @EventCode
227+
WHERE bi.BATCH_INSTANCE_ID = @BatchInstanceId;
228+
229+
SET @RowsAffected = @@ROWCOUNT;
230+
231+
IF @RowsAffected = 0
195232
BEGIN
196-
SET @LogMessage = CONCAT(N'Setting the Batch Instance ', @BatchInstanceId,
197-
N' to ', @EventCode + N'.');
198-
SET @MessageLog = [omd].[AddLogMessage](DEFAULT, DEFAULT, N'Status Update',
199-
@LogMessage, @MessageLog);
200-
201-
UPDATE [omd].[BATCH_INSTANCE]
202-
SET
203-
EXECUTION_STATUS_CODE = N'Aborted',
204-
INTERNAL_PROCESSING_CODE = N'Abort',
205-
NEXT_RUN_STATUS_CODE = N'Proceed',
206-
END_TIMESTAMP = @StartTimestamp
207-
WHERE BATCH_INSTANCE_ID = @BatchInstanceId;
208-
END
209-
210-
-- Skip / Cancel event
211-
-- This is an end-state event (no further processing)
212-
ELSE IF @EventCode = N'Cancel'
213-
BEGIN
214-
SET @LogMessage = CONCAT(N'Setting the Batch Instance ', @BatchInstanceId,
215-
N' to ', @EventCode, N'.');
216-
SET @MessageLog = [omd].[AddLogMessage](DEFAULT, DEFAULT, N'Status Update',
217-
@LogMessage, @MessageLog);
218-
219-
UPDATE [omd].[BATCH_INSTANCE]
220-
SET
221-
EXECUTION_STATUS_CODE = N'Cancelled',
222-
INTERNAL_PROCESSING_CODE = N'Cancel',
223-
NEXT_RUN_STATUS_CODE = N'Proceed',
224-
END_TIMESTAMP = @StartTimestamp
225-
WHERE BATCH_INSTANCE_ID = @BatchInstanceId;
226-
END
227-
228-
-- Success event
229-
-- This is an end-state event (no further processing)
230-
ELSE IF @EventCode = N'Success'
231-
BEGIN
232-
SET @LogMessage = CONCAT(N'Setting the Batch Instance ', @BatchInstanceId,
233-
N' to ', @EventCode, N'.');
234-
SET @MessageLog = [omd].[AddLogMessage](DEFAULT, DEFAULT, N'Status Update',
235-
@LogMessage, @MessageLog);
236-
237-
UPDATE [omd].[BATCH_INSTANCE]
238-
SET
239-
EXECUTION_STATUS_CODE = N'Succeeded',
240-
NEXT_RUN_STATUS_CODE = N'Proceed',
241-
INTERNAL_PROCESSING_CODE = N'Proceed',
242-
END_TIMESTAMP = @StartTimestamp
243-
WHERE BATCH_INSTANCE_ID = @BatchInstanceId;
244-
245-
GOTO EndOfProcedureSuccess;
246-
END
247-
248-
-- Failure event
249-
-- This is an end-state event (no further processing)
250-
ELSE IF @EventCode = N'Failure'
251-
BEGIN
252-
SET @LogMessage = CONCAT('Setting the Batch Instance ', @BatchInstanceId,
253-
N' to ', @EventCode, N'.');
254-
SET @MessageLog = [omd].[AddLogMessage](DEFAULT, DEFAULT, N'Status Update',
255-
@LogMessage, @MessageLog)
256-
257-
-- Note that the default behavior is that Next Run Indicator at Batch level is 'Proceed'.
258-
-- This will only skip/cancel already successfully completed Modules when a failed Batch is rerun.
259-
UPDATE [omd].[BATCH_INSTANCE]
260-
SET
261-
EXECUTION_STATUS_CODE = N'Failed',
262-
NEXT_RUN_STATUS_CODE = N'Proceed',
263-
END_TIMESTAMP = @StartTimestamp
264-
WHERE BATCH_INSTANCE_ID = @BatchInstanceId
265-
END
266-
267-
-- Rollback event
268-
ELSE IF @EventCode = N'Rollback'
269-
BEGIN
270-
SET @LogMessage = CONCAT(N'Setting the Batch Instance ', @BatchInstanceId,
271-
N' to ', @EventCode, N'.');
272-
SET @MessageLog = [omd].[AddLogMessage](DEFAULT, DEFAULT, N'Status Update',
273-
@LogMessage, @MessageLog)
274-
275-
UPDATE [omd].[BATCH_INSTANCE]
276-
SET
277-
INTERNAL_PROCESSING_CODE = N'Rollback'
278-
WHERE BATCH_INSTANCE_ID = @BatchInstanceId;
279-
END
280-
281-
-- Proceed event
282-
ELSE IF @EventCode = N'Proceed'
283-
BEGIN
284-
SET @LogMessage = CONCAT('Setting the Batch Instance ', @BatchInstanceId,
285-
N' to ', @EventCode, N'.');
286-
SET @MessageLog = [omd].[AddLogMessage](DEFAULT, DEFAULT, N'Status Update',
287-
@LogMessage, @MessageLog);
288-
289-
UPDATE [omd].[BATCH_INSTANCE]
290-
SET INTERNAL_PROCESSING_CODE = 'Proceed'
291-
WHERE BATCH_INSTANCE_ID = @BatchInstanceId;
292-
END
233+
IF XACT_STATE() <> 0 ROLLBACK TRANSACTION;
234+
SET @SuccessIndicator = 'N';
235+
SET @LogMessage = CONCAT('No batch instance row updated for BatchInstanceId ',
236+
@BatchInstanceId, ' and EventCode ''', @EventCode, '''.');
237+
IF @ProcessMessageLog = 'Y'
238+
SET @MessageLog = [omd].[AddLogMessage]('ERROR', DEFAULT,
239+
N'Transaction Error', @LogMessage, @MessageLog);
240+
IF @ThrowOnFailure = 'Y' THROW 50000, @LogMessage, 1;
241+
GOTO EndOfProcedureFailure;
242+
END;
293243

294244
COMMIT TRANSACTION;
295245
GOTO EndOfProcedureSuccess;
296246
END TRY
297247
BEGIN CATCH
248+
DECLARE
249+
@TxnErrorMessage NVARCHAR(4000) = ERROR_MESSAGE(),
250+
@TxnErrorNumber INT = ERROR_NUMBER(),
251+
@TxnErrorSeverity INT = ERROR_SEVERITY(),
252+
@TxnErrorState INT = ERROR_STATE(),
253+
@TxnErrorLine INT = ERROR_LINE();
254+
298255
IF XACT_STATE() <> 0 ROLLBACK TRANSACTION;
299256
SET @SuccessIndicator = 'N';
300257
IF COALESCE(TRIM(@MessageLog), '') = '' SET @MessageLog = N'[]';
301258

302-
SET @LogMessage = 'Unknown Transaction Processing Error';
259+
SET @LogMessage = CONCAT('Transaction error (', @TxnErrorNumber, '/', @TxnErrorState,
260+
') at line ', @TxnErrorLine, ': ', COALESCE(@TxnErrorMessage, 'No additional details.'));
303261
IF @ProcessMessageLog = 'Y' SET @MessageLog =
304-
[omd].[AddLogMessage]('ERROR', DEFAULT, 'Process Output', @LogMessage, @MessageLog);
305-
IF @ThrowOnFailure = 'Y' THROW 50000, @LogMessage, 1;
262+
[omd].[AddLogMessage]('ERROR', DEFAULT, 'Transaction Error', @LogMessage, @MessageLog);
263+
IF @ThrowOnFailure = 'Y' THROW;
306264
GOTO EndOfProcedureFailure;
307265
END CATCH;
308266

@@ -311,7 +269,7 @@ BEGIN
311269
EndOfProcedureFailure:
312270

313271
SET @SuccessIndicator = 'N';
314-
SET @LogMessage = N'End Batch Instance process encountered errors.';
272+
SET @LogMessage = CONCAT(@ProcessDescription, N' encountered errors.');
315273
IF @ProcessMessageLog = 'Y' SET @MessageLog =
316274
[omd].[AddLogMessage]('ERROR', DEFAULT, 'Processing Error', @LogMessage, @MessageLog);
317275
SET @ReturnCode = -1;
@@ -321,7 +279,7 @@ BEGIN
321279
EndOfProcedureSuccess:
322280

323281
SET @SuccessIndicator = 'Y';
324-
SET @LogMessage = N'End Batch Instance process completed successfully.';
282+
SET @LogMessage = CONCAT(@ProcessDescription, N' completed successfully.');
325283
IF @ProcessMessageLog = 'Y' SET @MessageLog =
326284
[omd].[AddLogMessage]('SUCCESS', DEFAULT, 'Processing Completion', @LogMessage, @MessageLog);
327285
SET @ReturnCode = 0;

‎Direct_Framework/Stored Procedures/omd.EndModuleInstance.sql‎

Lines changed: 58 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,12 @@ CREATE PROCEDURE [omd].[EndModuleInstance]
6969
,@EventCode NVARCHAR(100) = 'Failure'
7070
,@EventDetail NVARCHAR(4000) = NULL
7171
-- Optional row count updates
72-
,@RowCountInput BIGINT = 0
73-
,@RowCountInserted BIGINT = 0
74-
,@RowCountUpdated BIGINT = 0
75-
,@RowCountDeleted BIGINT = 0
76-
,@RowCountDiscarded BIGINT = 0
77-
,@RowCountRejected BIGINT = 0
72+
,@RowCountInput BIGINT = NULL
73+
,@RowCountInserted BIGINT = NULL
74+
,@RowCountUpdated BIGINT = NULL
75+
,@RowCountDeleted BIGINT = NULL
76+
,@RowCountDiscarded BIGINT = NULL
77+
,@RowCountRejected BIGINT = NULL
7878
-- Optional parameters
7979
,@EndTimestamp DATETIME2 = NULL
8080
,@Debug CHAR(1) = 'N'
@@ -176,13 +176,13 @@ BEGIN
176176

177177
/* Normalize all other input parameters */
178178
SELECT
179-
@EndTimestamp = COALESCE(@EndTimestamp, @StartTimestamp),
180-
@RowCountInput = CASE WHEN @RowCountInput >= 0 THEN @RowCountInput ELSE 0 END,
181-
@RowCountInserted = CASE WHEN @RowCountInserted >= 0 THEN @RowCountInserted ELSE 0 END,
182-
@RowCountUpdated = CASE WHEN @RowCountUpdated >= 0 THEN @RowCountUpdated ELSE 0 END,
183-
@RowCountDeleted = CASE WHEN @RowCountDeleted >= 0 THEN @RowCountDeleted ELSE 0 END,
184-
@RowCountDiscarded = CASE WHEN @RowCountDiscarded >= 0 THEN @RowCountDiscarded ELSE 0 END,
185-
@RowCountRejected = CASE WHEN @RowCountRejected >= 0 THEN @RowCountRejected ELSE 0 END;
179+
@EndTimestamp = COALESCE(@EndTimestamp, @StartTimestamp)
180+
-- @RowCountInput = CASE WHEN @RowCountInput >= 0 THEN @RowCountInput ELSE 0 END,
181+
-- @RowCountInserted = CASE WHEN @RowCountInserted >= 0 THEN @RowCountInserted ELSE 0 END,
182+
-- @RowCountUpdated = CASE WHEN @RowCountUpdated >= 0 THEN @RowCountUpdated ELSE 0 END,
183+
-- @RowCountDeleted = CASE WHEN @RowCountDeleted >= 0 THEN @RowCountDeleted ELSE 0 END,
184+
-- @RowCountDiscarded = CASE WHEN @RowCountDiscarded >= 0 THEN @RowCountDiscarded ELSE 0 END,
185+
-- @RowCountRejected = CASE WHEN @RowCountRejected >= 0 THEN @RowCountRejected ELSE 0 END;
186186

187187
/* Add normalized parameters log here if required */
188188

@@ -202,12 +202,12 @@ BEGIN
202202
;WITH StatusMap AS (
203203
SELECT *
204204
FROM (VALUES
205-
('Abort', 'Aborted', 'Abort', 'Proceed', 1, 0),
206-
('Cancel', 'Cancelled', 'Cancel', 'Proceed', 1, 0),
205+
('Abort', 'Aborted', 'Abort', 'Proceed', 1, 1),
206+
('Cancel', 'Cancelled', 'Cancel', 'Proceed', 1, 1),
207207
('Success', 'Succeeded', 'Proceed', 'Proceed', 1, 1),
208-
('Failure', 'Failed', NULL, 'Rollback', 1, 0),
209-
('Rollback', NULL, 'Rollback', NULL, 1, 0),
210-
('Proceed', NULL, 'Proceed', NULL, 1, 0)
208+
('Failure', 'Failed', NULL, 'Rollback', 1, 1),
209+
('Rollback', NULL, 'Rollback', NULL, 1, 1),
210+
('Proceed', NULL, 'Proceed', NULL, 1, 1)
211211
) AS map(EventCode, ExecutionStatusCode, InternalProcessingCode,
212212
NextRunStatusCode, ApplyEndTimestamp, ApplyRowCounts)
213213
)
@@ -217,12 +217,46 @@ BEGIN
217217
INTERNAL_PROCESSING_CODE = COALESCE(sm.InternalProcessingCode, mi.INTERNAL_PROCESSING_CODE),
218218
NEXT_RUN_STATUS_CODE = COALESCE(sm.NextRunStatusCode, mi.NEXT_RUN_STATUS_CODE),
219219
END_TIMESTAMP = CASE WHEN sm.ApplyEndTimestamp = 1 THEN @EndTimestamp ELSE mi.END_TIMESTAMP END,
220-
ROWS_INPUT = CASE WHEN sm.ApplyRowCounts = 1 THEN @RowCountInput ELSE mi.ROWS_INPUT END,
221-
ROWS_INSERTED = CASE WHEN sm.ApplyRowCounts = 1 THEN @RowCountInserted ELSE mi.ROWS_INSERTED END,
222-
ROWS_UPDATED = CASE WHEN sm.ApplyRowCounts = 1 THEN @RowCountUpdated ELSE mi.ROWS_UPDATED END,
223-
ROWS_DELETED = CASE WHEN sm.ApplyRowCounts = 1 THEN @RowCountDeleted ELSE mi.ROWS_DELETED END,
224-
ROWS_DISCARDED = CASE WHEN sm.ApplyRowCounts = 1 THEN @RowCountDiscarded ELSE mi.ROWS_DISCARDED END,
225-
ROWS_REJECTED = CASE WHEN sm.ApplyRowCounts = 1 THEN @RowCountRejected ELSE mi.ROWS_REJECTED END
220+
/* Update row counts if settings are enabled for it and
221+
values are provided as parameters. If not, keep existing values.
222+
If both are null, set them to 0 */
223+
ROWS_INPUT = CASE
224+
WHEN sm.ApplyRowCounts = 0 THEN mi.ROWS_INPUT
225+
WHEN @RowCountInput >= 0 THEN @RowCountInput
226+
WHEN @RowCountInput IS NULL AND mi.ROWS_INPUT IS NULL THEN 0
227+
ELSE mi.ROWS_INPUT
228+
END,
229+
230+
ROWS_INSERTED = CASE
231+
WHEN sm.ApplyRowCounts = 0 THEN mi.ROWS_INSERTED
232+
WHEN @RowCountInserted >= 0 THEN @RowCountInserted
233+
WHEN @RowCountInserted IS NULL AND mi.ROWS_INSERTED IS NULL THEN 0
234+
ELSE mi.ROWS_INSERTED
235+
END,
236+
ROWS_UPDATED = CASE
237+
WHEN sm.ApplyRowCounts = 0 THEN mi.ROWS_UPDATED
238+
WHEN @RowCountUpdated >= 0 THEN @RowCountUpdated
239+
WHEN @RowCountUpdated IS NULL AND mi.ROWS_UPDATED IS NULL THEN 0
240+
ELSE mi.ROWS_UPDATED
241+
END,
242+
ROWS_DELETED = CASE
243+
WHEN sm.ApplyRowCounts = 0 THEN mi.ROWS_DELETED
244+
WHEN @RowCountDeleted >= 0 THEN @RowCountDeleted
245+
WHEN @RowCountDeleted IS NULL AND mi.ROWS_DELETED IS NULL THEN 0
246+
ELSE mi.ROWS_DELETED
247+
END,
248+
ROWS_DISCARDED = CASE
249+
WHEN sm.ApplyRowCounts = 0 THEN mi.ROWS_DISCARDED
250+
WHEN @RowCountDiscarded >= 0 THEN @RowCountDiscarded
251+
WHEN @RowCountDiscarded IS NULL AND mi.ROWS_DISCARDED IS NULL THEN 0
252+
ELSE mi.ROWS_DISCARDED
253+
END,
254+
ROWS_REJECTED = CASE
255+
WHEN sm.ApplyRowCounts = 0 THEN mi.ROWS_REJECTED
256+
WHEN @RowCountRejected >= 0 THEN @RowCountRejected
257+
WHEN @RowCountRejected IS NULL AND mi.ROWS_REJECTED IS NULL THEN 0
258+
ELSE mi.ROWS_REJECTED
259+
END
226260
FROM [omd].[MODULE_INSTANCE] AS mi
227261
INNER JOIN StatusMap AS sm
228262
ON sm.EventCode = @EventCode

‎Releases.Direct_Framework/next/db/Direct_Framework.StaticCodeAnalysis.Results.xml‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,19 @@
11
<?xml version="1.0" encoding="utf-8"?>
2-
<Problems />
2+
<Problems>
3+
<Problem>
4+
<Rule>Microsoft.Rules.Data.SR0001</Rule>
5+
<ProblemDescription>The shape of the result set produced by a SELECT * statement will change if the underlying table or view structure changes.</ProblemDescription>
6+
<SourceFile>c:\repos\github\data-solution-automation-engine\DIRECT\Direct_Framework\Stored Procedures\omd.EndModuleInstance.sql</SourceFile>
7+
<Line>203</Line>
8+
<Column>18</Column>
9+
<Severity>Warning</Severity>
10+
</Problem>
11+
<Problem>
12+
<Rule>Microsoft.Rules.Data.SR0001</Rule>
13+
<ProblemDescription>The shape of the result set produced by a SELECT * statement will change if the underlying table or view structure changes.</ProblemDescription>
14+
<SourceFile>c:\repos\github\data-solution-automation-engine\DIRECT\Direct_Framework\Stored Procedures\omd.EndBatchInstance.sql</SourceFile>
15+
<Line>207</Line>
16+
<Column>16</Column>
17+
<Severity>Warning</Severity>
18+
</Problem>
19+
</Problems>

‎Releases.Direct_Framework/next/db/Direct_Framework.dacpac‎

998 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)