fix(EAV-737): prevent latency when running long callbacks after updating the timeline

moves timeline generation to `saveAllToDatabase`

Author: ianshade

packages/job-worker/src/playout/model/PlayoutModel.ts

@@ -387,6 +387,12 @@ export interface PlayoutModel extends PlayoutModelReadonly, StudioPlayoutModelBa
 	 */
 	getNowInPlayout(): Time
 
+	/**
+	 * Mark the playlist as needing a timeline update.
+	 * The timeline will be generated and published when model is ready to be saved.
+	 */
+	markTimelineNeedsUpdate(): void
+
 	/** Lifecycle */
 
 	/**

packages/job-worker/src/playout/model/implementation/PlayoutModelImpl.ts

@@ -71,6 +71,14 @@ import { PieceInstanceWithTimings } from '@sofie-automation/corelib/dist/playout
 import { NotificationsModelHelper } from '../../../notifications/NotificationsModelHelper.js'
 import { getExpectedLatency } from '@sofie-automation/corelib/dist/studio/playout'
 import { ExpectedPackage } from '@sofie-automation/blueprints-integration'
+import { stringifyError } from '@sofie-automation/shared-lib/dist/lib/stringifyError'
+import {
+	getTimelineRundown,
+	flattenAndProcessTimelineObjects,
+	preserveOrReplaceNowTimesInObjects,
+	logAnyRemainingNowTimes,
+} from '../../timeline/generate.js'
+import { deNowifyMultiGatewayTimeline } from '../../timeline/multi-gateway.js'
 
 export class PlayoutModelReadonlyImpl implements PlayoutModelReadonly {
 	public readonly playlistId: RundownPlaylistId
@@ -315,6 +323,7 @@ export class PlayoutModelImpl extends PlayoutModelReadonlyImpl implements Playou
 
 	#playlistHasChanged = false
 	#timelineHasChanged = false
+	#timelineNeedsRegeneration = false
 
 	#pendingPartInstanceTimingEvents = new Set<PartInstanceId>()
 
@@ -694,6 +703,12 @@ export class PlayoutModelImpl extends PlayoutModelReadonlyImpl implements Playou
 		}
 		this.#deferredBeforeSaveFunctions.length = 0 // clear the array
 
+		// Generate timeline if needed
+		if (this.#timelineNeedsRegeneration) {
+			await this.#regenerateTimeline()
+			this.#timelineNeedsRegeneration = false
+		}
+
 		// Prioritise the timeline for publication reasons
 		if (this.#timelineHasChanged && this.timelineImpl) {
 			// Do a fast-track for the timeline to be published faster:
@@ -889,6 +904,10 @@ export class PlayoutModelImpl extends PlayoutModelReadonlyImpl implements Playou
 		return result
 	}
 
+	markTimelineNeedsUpdate(): void {
+		this.#timelineNeedsRegeneration = true
+	}
+
 	/** Notifications */
 
 	async getAllNotifications(
@@ -925,6 +944,38 @@ export class PlayoutModelImpl extends PlayoutModelReadonlyImpl implements Playou
 
 	/** BaseModel */
 
+	async #regenerateTimeline(): Promise<void> {
+		const span = this.context.startSpan('PlayoutModelImpl.regenerateTimeline')
+		logger.debug('Regenerating timeline...')
+
+		try {
+			const {
+				versions,
+				objs: timelineObjs,
+				timingContext: timingInfo,
+				regenerateTimelineToken
+			} = await getTimelineRundown(this.context, this as PlayoutModel)
+
+			flattenAndProcessTimelineObjects(this.context, timelineObjs)
+
+			preserveOrReplaceNowTimesInObjects(this, timelineObjs)
+
+			if (this.isMultiGatewayMode) {
+				deNowifyMultiGatewayTimeline(this as PlayoutModel, timelineObjs, timingInfo)
+
+				logAnyRemainingNowTimes(this.context, timelineObjs)
+			}
+
+			const timelineHash = this.setTimeline(timelineObjs, versions, regenerateTimelineToken).timelineHash
+			logger.verbose(`Timeline regeneration done, hash: "${timelineHash}"`)
+		} catch (err) {
+			logger.error(`Error regenerating timeline: ${stringifyError(err)}`)
+			throw err
+		} finally {
+			if (span) span.end()
+		}
+	}
+
 	/**
 	 * Assert that no changes should have been made to the model, will throw an Error otherwise. This can be used in
 	 * place of `saveAllToDatabase()`, when the code controlling the model expects no changes to have been made and any

packages/job-worker/src/playout/timeline/generate.ts

@@ -37,7 +37,6 @@ import { deserializePieceTimelineObjectsBlob } from '@sofie-automation/corelib/d
 import { convertResolvedPieceInstanceToBlueprints } from '../../blueprints/context/lib.js'
 import { buildTimelineObjsForRundown, RundownTimelineTimingContext } from './rundown.js'
 import { SourceLayers } from '@sofie-automation/corelib/dist/dataModel/ShowStyleBase'
-import { deNowifyMultiGatewayTimeline } from './multi-gateway.js'
 import { validateTimeline } from 'superfly-timeline'
 import { getPartTimingsOrDefaults, PartCalculatedTimings } from '@sofie-automation/corelib/dist/playout/timings'
 import { applyAbPlaybackForTimeline } from '../abPlayback/index.js'
@@ -140,39 +139,21 @@ export async function updateStudioTimeline(
 
 export async function updateTimeline(context: JobContext, playoutModel: PlayoutModel): Promise<void> {
 	const span = context.startSpan('updateTimeline')
-	logger.debug('updateTimeline running...')
+	logger.debug('updateTimeline: marking playlist as needing timeline update')
 
 	if (!playoutModel.playlist.activationId) {
 		throw new Error(`RundownPlaylist ("${playoutModel.playlist._id}") is not active")`)
 	}
 
-	const {
-		versions,
-		objs: timelineObjs,
-		timingContext: timingInfo,
-		regenerateTimelineToken,
-	} = await getTimelineRundown(context, playoutModel)
-
-	flattenAndProcessTimelineObjects(context, timelineObjs)
-
-	preserveOrReplaceNowTimesInObjects(playoutModel, timelineObjs)
-
-	if (playoutModel.isMultiGatewayMode) {
-		deNowifyMultiGatewayTimeline(playoutModel, timelineObjs, timingInfo)
-
-		logAnyRemainingNowTimes(context, timelineObjs)
-	}
-
-	const timelineHash = playoutModel.setTimeline(timelineObjs, versions, regenerateTimelineToken).timelineHash
-	logger.verbose(`updateTimeline done, hash: "${timelineHash}"`)
+	playoutModel.markTimelineNeedsUpdate()
 
 	if (span) span.end()
 }
 
-function preserveOrReplaceNowTimesInObjects(
+export function preserveOrReplaceNowTimesInObjects(
 	studioPlayoutModel: StudioPlayoutModelBase,
 	timelineObjs: Array<TimelineObjGeneric>
-) {
+): void {
 	const timeline = studioPlayoutModel.timeline
 	const oldTimelineObjsMap = normalizeArray(
 		(timeline?.timelineBlob !== undefined && deserializeTimelineBlob(timeline.timelineBlob)) || [],
@@ -202,7 +183,7 @@ function preserveOrReplaceNowTimesInObjects(
 	})
 }
 
-function logAnyRemainingNowTimes(_context: JobContext, timelineObjs: Array<TimelineObjGeneric>): void {
+export function logAnyRemainingNowTimes(_context: JobContext, timelineObjs: Array<TimelineObjGeneric>): void {
 	const badTimelineObjs: any[] = []
 
 	for (const obj of timelineObjs) {
@@ -287,7 +268,7 @@ function getPartInstanceTimelineInfo(
 /**
  * Returns timeline objects related to rundowns in a studio
  */
-async function getTimelineRundown(
+export async function getTimelineRundown(
 	context: JobContext,
 	playoutModel: PlayoutModel
 ): Promise<{
@@ -541,7 +522,7 @@ function createRegenerateTimelineObj(
  * @param context
  * @param timelineObjs Array of timeline objects
  */
-function flattenAndProcessTimelineObjects(context: JobContext, timelineObjs: Array<TimelineObjGeneric>): void {
+export function flattenAndProcessTimelineObjects(context: JobContext, timelineObjs: Array<TimelineObjGeneric>): void {
 	const span = context.startSpan('processTimelineObjects')
 
 	// first, split out any grouped objects, to make the timeline shallow: