Fix pinned workflow trampolining via revision-based signal suppression (#9895)

Shivs11 · claude · web-flow · commit 089fba53e16d · 2026-04-21T20:15:49.000-07:00
## Summary - Matching now sends the real `targetDeploymentRevisionNumber` for pinned workflows instead of hardcoded 0 - Added `revision_number` field to `LastNotifiedTargetVersion` (server-internal) and `DeclinedTargetVersionUpgrade` (public API) - Case 4 in the target version change switch now uses `targetRevisionNumber <= declined.RevisionNumber` instead of version string comparison, preventing trampolining when stale matching partitions send outdated target versions ## Problem When a pinned workflow CaNs and declines a target version upgrade, a stale matching partition can send an older target version. The old case 4 compared version strings (`declined.buildId == target.buildId`), which didn't match the stale version — causing the workflow to re-signal, CaN again, and trampoline indefinitely between stale and up-to-date partitions. ## Test plan - [x] `TestStalePartition_RevisionSuppressesTrampolining` — integration test that simulates a stale partition via `rollbackTaskQueueToVersion` and verifies: - Stale partition (revision 0) is suppressed after declining at a higher revision - Genuinely new version (v4 at higher revision) correctly fires the signal - [x] Verified test **fails** when matching + history changes are reverted (matching sends 0, old string comparison) ## API repo dependency - api: `temporalio/api@trampolining-rev-number` - api-go: `temporalio/api-go@trampolining-rev-number` 🤖 Generated with [Claude Code](https://claude.com/claude-code)  --- > [!NOTE] > **Medium Risk** > Touches pinned-workflow versioning logic across matching, history, and persistence; a bug here could change when workflows are signaled to continue-as-new or upgrade, affecting routing behavior. > > **Overview** > Prevents pinned workflows from repeatedly continue-as-new “trampolining” when task queue partitions have stale routing data by **tracking and comparing target routing-config revision numbers**. > > Matching now propagates the real `targetDeploymentRevisionNumber` for pinned workflow tasks, and history threads this through `AddWorkflowTaskStartedEvent` to persist `LastNotifiedTargetVersion.revision_number` and carry it into `DeclinedTargetVersionUpgrade` on continue-as-new. The pinned target-change decision in `workflow_task_state_machine` switches case-4 suppression from deployment version string equality to `targetRevisionNumber <= declined.RevisionNumber`, and adds an integration test (`TestStalePartition_RevisionSuppressesTrampolining`) covering stale vs fresh partition behavior. > > <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 4114662. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>  --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
diff --git a/api/persistence/v1/executions.pb.go b/api/persistence/v1/executions.pb.go
diff --git a/go.mod b/go.mod
@@ -63,7 +63,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.43.0
 	go.opentelemetry.io/otel/sdk/metric v1.43.0
 	go.opentelemetry.io/otel/trace v1.43.0
-	go.temporal.io/api v1.62.10-0.20260420202918-975b82732988
+	go.temporal.io/api v1.62.10-0.20260421204157-0617d4e3bba2
 	go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2
 	go.temporal.io/sdk v1.41.1
 	go.uber.org/fx v1.24.0
diff --git a/go.sum b/go.sum
@@ -469,8 +469,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R
 go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U=
 go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0=
 go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4=
-go.temporal.io/api v1.62.10-0.20260420202918-975b82732988 h1:ZJ1SLhzqMz62LR0nIcle5MSHJzqERBGj/tsxQfX8YTo=
-go.temporal.io/api v1.62.10-0.20260420202918-975b82732988/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM=
+go.temporal.io/api v1.62.10-0.20260421204157-0617d4e3bba2 h1:9s2PjMyiiRg49fmjgWDo5RIx2MuWu5K4S8a7pThNpzU=
+go.temporal.io/api v1.62.10-0.20260421204157-0617d4e3bba2/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM=
 go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 h1:1hKeH3GyR6YD6LKMHGCZ76t6h1Sgha0hXVQBxWi3dlQ=
 go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2/go.mod h1:T8dnzVPeO+gaUTj9eDgm/lT2lZH4+JXNvrGaQGyVi50=
 go.temporal.io/sdk v1.41.1 h1:yOpvsHyDD1lNuwlGBv/SUodCPhjv9nDeC9lLHW/fJUA=
diff --git a/proto/internal/temporal/server/api/persistence/v1/executions.proto b/proto/internal/temporal/server/api/persistence/v1/executions.proto
@@ -333,6 +333,10 @@ message TimeSkippingInfo {
 // Used only within server persistence; never flows to the public API.
 message LastNotifiedTargetVersion {
   temporal.api.deployment.v1.WorkerDeploymentVersion deployment_version = 1;
+  // Revision number of the task queue routing config at the time the
+  // notification was sent. Carried forward to DeclinedTargetVersionUpgrade
+  // at continue-as-new time.
+  int64 revision_number = 2;
 }
 
 message ExecutionStats {
diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go
@@ -113,6 +113,7 @@ func NewWorkflowWithSignal(
 			nil,
 			false,
 			nil,
+			0,
 		)
 		if err != nil {
 			// Unable to add WorkflowTaskStarted event to history
diff --git a/service/history/api/recordworkflowtaskstarted/api.go b/service/history/api/recordworkflowtaskstarted/api.go
@@ -170,6 +170,7 @@ func Invoke(
 				workflowLease.GetContext().UpdateRegistry(ctx),
 				false,
 				req.TargetDeploymentVersion,
+				req.TaskDispatchRevisionNumber,
 			)
 			if err != nil {
 				// Unable to add WorkflowTaskStarted event to history
diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go
@@ -603,6 +603,7 @@ func (handler *WorkflowTaskCompletedHandler) Invoke(
 				workflowLease.GetContext().UpdateRegistry(ctx),
 				false,
 				nil,
+				0,
 			)
 			if err != nil {
 				return nil, err
@@ -728,6 +729,7 @@ func (handler *WorkflowTaskCompletedHandler) Invoke(
 			workflowLease.GetContext().UpdateRegistry(ctx),
 			false,
 			nil,
+			0,
 		)
 		if err != nil {
 			return nil, err
diff --git a/service/history/api/respondworkflowtaskcompleted/api_test.go b/service/history/api/respondworkflowtaskcompleted/api_test.go
@@ -728,6 +728,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) createSentUpdate(tv *testvars.TestVa
 		nil,
 		false,
 		nil,
+		0,
 	)
 	taskToken := &tokenspb.Task{
 		Attempt:          1,
@@ -807,6 +808,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) createPausedWorkflowWithWFT(tv *test
 		nil,
 		false,
 		nil,
+		0,
 	)
 	_, _ = ms.AddWorkflowTaskCompletedEvent(wt, &workflowservice.RespondWorkflowTaskCompletedRequest{
 		Identity: tv.Any().String(),
@@ -824,6 +826,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) createPausedWorkflowWithWFT(tv *test
 		nil,
 		false,
 		nil,
+		0,
 	)
 	taskToken := &tokenspb.Task{
 		Attempt:          1,
diff --git a/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go b/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go
@@ -238,6 +238,7 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched
 		nil,
 		false,
 		nil,
+		0,
 	)
 	wt.StartedEventID = workflowTasksStartEvent.GetEventId()
 
diff --git a/service/history/history_engine_test.go b/service/history/history_engine_test.go
@@ -6618,6 +6618,7 @@ func addWorkflowTaskStartedEventWithRequestID(ms historyi.MutableState, schedule
 		nil,
 		false,
 		nil,
+		0,
 	)
 
 	return event
diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go
@@ -76,7 +76,7 @@ type (
 		AddFirstWorkflowTaskScheduled(parentClock *clockspb.VectorClock, event *historypb.HistoryEvent, bypassTaskGeneration bool) (int64, error)
 		AddWorkflowTaskScheduledEvent(bypassTaskGeneration bool, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error)
 		AddWorkflowTaskScheduledEventAsHeartbeat(bypassTaskGeneration bool, originalScheduledTimestamp *timestamppb.Timestamp, workflowTaskType enumsspb.WorkflowTaskType) (*WorkflowTaskInfo, error)
-		AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string, *commonpb.WorkerVersionStamp, *taskqueuespb.BuildIdRedirectInfo, update.Registry, bool, *deploymentpb.WorkerDeploymentVersion) (*historypb.HistoryEvent, *WorkflowTaskInfo, error)
+		AddWorkflowTaskStartedEvent(int64, string, *taskqueuepb.TaskQueue, string, *commonpb.WorkerVersionStamp, *taskqueuespb.BuildIdRedirectInfo, update.Registry, bool, *deploymentpb.WorkerDeploymentVersion, int64) (*historypb.HistoryEvent, *WorkflowTaskInfo, error)
 		AddWorkflowTaskTimedOutEvent(workflowTask *WorkflowTaskInfo) (*historypb.HistoryEvent, error)
 		AddExternalWorkflowExecutionCancelRequested(int64, namespace.Name, namespace.ID, string, string) (*historypb.HistoryEvent, error)
 		AddExternalWorkflowExecutionSignaled(int64, namespace.Name, namespace.ID, string, string, string) (*historypb.HistoryEvent, error)
diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go
diff --git a/service/history/ndc/workflow_resetter.go b/service/history/ndc/workflow_resetter.go
@@ -541,6 +541,7 @@ func (r *workflowResetterImpl) failWorkflowTask(
 			// skipping versioning checks because this task is not actually dispatched but will fail immediately.
 			true,
 			nil,
+			0,
 		)
 		if err != nil {
 			return err
diff --git a/service/history/ndc/workflow_resetter_test.go b/service/history/ndc/workflow_resetter_test.go
@@ -397,6 +397,7 @@ func (s *workflowResetterSuite) TestFailWorkflowTask_WorkflowTaskScheduled() {
 		nil,
 		true,
 		nil,
+		gomock.Any(),
 	).Return(&historypb.HistoryEvent{}, workflowTaskStart, nil)
 	mutableState.EXPECT().AddWorkflowTaskFailedEvent(
 		workflowTaskStart,
@@ -1553,6 +1554,7 @@ func (s *workflowResetterSuite) TestWorkflowRestartAfterExecutionTimeout() {
 		nil,
 		true,
 		nil,
+		gomock.Any(),
 	).Return(&historypb.HistoryEvent{}, workflowTaskStart, nil)
 
 	resetMutableState.EXPECT().AddWorkflowTaskFailedEvent(
diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go
@@ -2679,6 +2679,7 @@ func computeDeclinedTargetVersionUpgrade(info *persistencespb.WorkflowExecutionI
 	if lastNotified := info.GetLastNotifiedTargetVersion(); lastNotified != nil {
 		return &historypb.DeclinedTargetVersionUpgrade{
 			DeploymentVersion: lastNotified.GetDeploymentVersion(),
+			RevisionNumber:    lastNotified.GetRevisionNumber(),
 		}
 	}
 	return info.GetDeclinedTargetVersionUpgrade()
@@ -3300,12 +3301,13 @@ func (ms *MutableStateImpl) AddWorkflowTaskStartedEvent(
 	updateReg update.Registry,
 	skipVersioningCheck bool,
 	targetDeploymentVersion *deploymentpb.WorkerDeploymentVersion,
+	targetRevisionNumber int64,
 ) (*historypb.HistoryEvent, *historyi.WorkflowTaskInfo, error) {
 	opTag := tag.WorkflowActionWorkflowTaskStarted
 	if err := ms.checkMutability(opTag); err != nil {
 		return nil, nil, err
 	}
-	return ms.workflowTaskManager.AddWorkflowTaskStartedEvent(scheduledEventID, requestID, taskQueue, identity, versioningStamp, redirectInfo, skipVersioningCheck, updateReg, targetDeploymentVersion)
+	return ms.workflowTaskManager.AddWorkflowTaskStartedEvent(scheduledEventID, requestID, taskQueue, identity, versioningStamp, redirectInfo, skipVersioningCheck, updateReg, targetDeploymentVersion, targetRevisionNumber)
 }
 
 func (ms *MutableStateImpl) ApplyWorkflowTaskStartedEvent(
diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go
@@ -332,6 +332,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_Valid() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.Equal("b2", wft.BuildId)
@@ -357,6 +358,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_Invalid() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
 	s.ErrorAs(err, &expectedErr)
@@ -380,6 +382,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_EmptyRedirectInfo() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
 	s.ErrorAs(err, &expectedErr)
@@ -403,6 +406,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_EmptyStamp() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
 	s.ErrorAs(err, &expectedErr)
@@ -428,6 +432,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_Sticky() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.Equal("", wft.BuildId)
@@ -455,6 +460,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_StickyInvalid() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
 	s.ErrorAs(err, &expectedErr)
@@ -479,6 +485,7 @@ func (s *mutableStateSuite) TestRedirectInfoValidation_UnexpectedSticky() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	expectedErr := &serviceerror2.ObsoleteDispatchBuildId{}
 	s.ErrorAs(err, &expectedErr)
@@ -541,6 +548,7 @@ func (s *mutableStateSuite) TestPopulateDeleteTasks_WithWorkflowTaskTimeouts() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 
@@ -595,6 +603,7 @@ func (s *mutableStateSuite) TestPopulateDeleteTasks_LongTimeout_NotIncluded() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 
@@ -664,6 +673,7 @@ func (s *mutableStateSuite) createVersionedMutableStateWithCompletedWFT(tq *task
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.Equal("b1", wft.BuildId)
@@ -947,6 +957,7 @@ func (s *mutableStateSuite) createMutableStateWithVersioningBehavior(
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.verifyEffectiveDeployment(deployment, enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE)
@@ -998,6 +1009,7 @@ func (s *mutableStateSuite) TestUnpinnedTransition() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.verifyEffectiveDeployment(deployment2, behavior)
@@ -1038,6 +1050,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionFailed() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.verifyEffectiveDeployment(deployment2, behavior)
@@ -1081,6 +1094,7 @@ func (s *mutableStateSuite) TestUnpinnedTransitionTimeout() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.verifyEffectiveDeployment(deployment2, behavior)
@@ -1276,6 +1290,7 @@ func (s *mutableStateSuite) TestOverride_BaseDeploymentUpdatedOnCompletion() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.verifyEffectiveDeployment(deployment3, overrideBehavior)
@@ -1731,6 +1746,7 @@ func (s *mutableStateSuite) TestAddWorkflowExecutionPausedEvent() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent(
@@ -1794,6 +1810,7 @@ func (s *mutableStateSuite) TestAddWorkflowExecutionUnpausedEvent() {
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	completedEvent, err := s.mutableState.AddWorkflowTaskCompletedEvent(
@@ -2017,6 +2034,7 @@ func (s *mutableStateSuite) TestTransientWorkflowTaskStart_CurrentVersionChanged
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	s.Equal(0, s.mutableState.hBuilder.NumBufferedEvents())
@@ -6185,6 +6203,7 @@ func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerControlTa
 		nil,
 		false,
 		nil,
+		0,
 	)
 	s.NoError(err)
 	_, err = s.mutableState.AddWorkflowTaskCompletedEvent(
diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go
diff --git a/service/history/workflow/workflow_test/mutable_state_impl_test.go b/service/history/workflow/workflow_test/mutable_state_impl_test.go
diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go
diff --git a/tests/versioning_3_test.go b/tests/versioning_3_test.go

Original file line number	Diff line number	Diff line change
`@@ -113,6 +113,7 @@ func NewWorkflowWithSignal(`
`113`	`113`	`nil,`
`114`	`114`	`false,`
`115`	`115`	`nil,`
	`116`	`+ 0,`
`116`	`117`	`)`
`117`	`118`	`if err != nil {`
`118`	`119`	`// Unable to add WorkflowTaskStarted event to history`
Original file line number	Diff line number	Diff line change
`@@ -170,6 +170,7 @@ func Invoke(`
`170`	`170`	`workflowLease.GetContext().UpdateRegistry(ctx),`
`171`	`171`	`false,`
`172`	`172`	`req.TargetDeploymentVersion,`
	`173`	`+ req.TaskDispatchRevisionNumber,`
`173`	`174`	`)`
`174`	`175`	`if err != nil {`
`175`	`176`	`// Unable to add WorkflowTaskStarted event to history`
Original file line number	Diff line number	Diff line change
`@@ -238,6 +238,7 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched`
`238`	`238`	`nil,`
`239`	`239`	`false,`
`240`	`240`	`nil,`
	`241`	`+ 0,`
`241`	`242`	`)`
`242`	`243`	`wt.StartedEventID = workflowTasksStartEvent.GetEventId()`
`243`	`244`
Original file line number	Diff line number	Diff line change
`@@ -6618,6 +6618,7 @@ func addWorkflowTaskStartedEventWithRequestID(ms historyi.MutableState, schedule`
`6618`	`6618`	`nil,`
`6619`	`6619`	`false,`
`6620`	`6620`	`nil,`
	`6621`	`+ 0,`
`6621`	`6622`	`)`
`6622`	`6623`
`6623`	`6624`	`return event`