From de872cc06f9cfff037aab3b9c550199668a4365a Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Tue, 16 Jun 2026 12:36:22 -0700 Subject: [PATCH 01/13] timeskipping poc with saa --- chasm/context.go | 18 ++ chasm/context_mock.go | 29 ++- chasm/lib/activity/activity.go | 47 +++++ chasm/lib/activity/config.go | 2 + chasm/lib/activity/handler.go | 4 + chasm/node_backend_mock.go | 65 ++++-- chasm/timeskipping_test.go | 103 ++++++++++ chasm/timeskippping.go | 164 +++++++++++++++ chasm/tree.go | 189 ++++++++++++++++++ go.mod | 2 +- go.sum | 4 + service/history/interfaces/mutable_state.go | 3 + .../history/interfaces/mutable_state_mock.go | 14 ++ .../timer_queue_active_task_executor.go | 11 +- .../history/workflow/mutable_state_impl.go | 3 + service/history/workflow/timeskipping.go | 143 +++++++++++++ service/history/workflow/timeskipping_test.go | 50 +++++ tests/activity_standalone_test.go | 125 ++++++++++++ 18 files changed, 950 insertions(+), 26 deletions(-) create mode 100644 chasm/timeskipping_test.go create mode 100644 chasm/timeskippping.go diff --git a/chasm/context.go b/chasm/context.go index 8eb17021408..78b291615fe 100644 --- a/chasm/context.go +++ b/chasm/context.go @@ -97,6 +97,12 @@ type MutableContext interface { // SetUserMetadata replaces the user metadata attached to the given component. SetUserMetadata(Component, *sdkpb.UserMetadata) error + // TimeSkippingController is the framework-provided surface for managing this execution's + // time-skipping configuration (Init/Update/Get). A root component opts in by calling + // InitTimeSkippingConfig when the execution is created. These take no Component argument because + // the configuration is execution-scoped, like the ExecutionKey()/ExecutionInfo() accessors. + TimeSkippingController + // Get a Ref for the component // This ref to the component state at the end of the transition // Same as Ref(Component) method in Context, @@ -277,6 +283,18 @@ func (c *mutableCtx) SetUserMetadata(component Component, md *sdkpb.UserMetadata return c.root.setComponentUserMetadata(component, md) } +func (c *mutableCtx) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + c.root.backend.InitTimeSkippingConfig(config) +} + +func (c *mutableCtx) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + c.root.backend.UpdateTimeSkippingConfig(config) +} + +func (c *mutableCtx) GetTimeSkippingConfig() *commonpb.TimeSkippingConfig { + return c.root.backend.GetTimeSkippingInfo().GetConfig() +} + func (c *mutableCtx) withValue(key any, value any) Context { return &mutableCtx{ immutableCtx: ContextWithValue(c.immutableCtx, key, value), diff --git a/chasm/context_mock.go b/chasm/context_mock.go index 92b05f780b7..74eae6e61d1 100644 --- a/chasm/context_mock.go +++ b/chasm/context_mock.go @@ -183,10 +183,31 @@ func (c *MockContext) withValue(key any, value any) Context { type MockMutableContext struct { MockContext - mu sync.Mutex - Tasks []MockTask - LinksByRequest map[Component]map[string][]*commonpb.Link - UserMetadataByComponent map[Component]*sdkpb.UserMetadata + mu sync.Mutex + Tasks []MockTask + LinksByRequest map[Component]map[string][]*commonpb.Link + UserMetadataByComponent map[Component]*sdkpb.UserMetadata + TimeSkippingConfig *commonpb.TimeSkippingConfig + TimeSkippingConfigInited bool +} + +func (c *MockMutableContext) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + c.mu.Lock() + defer c.mu.Unlock() + c.TimeSkippingConfig = config + c.TimeSkippingConfigInited = true +} + +func (c *MockMutableContext) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + c.mu.Lock() + defer c.mu.Unlock() + c.TimeSkippingConfig = config +} + +func (c *MockMutableContext) GetTimeSkippingConfig() *commonpb.TimeSkippingConfig { + c.mu.Lock() + defer c.mu.Unlock() + return c.TimeSkippingConfig } func (c *MockMutableContext) AddTask(component Component, attributes TaskAttributes, payload any) { diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index 3efce2027c9..d498bceb6e9 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -124,6 +124,53 @@ func (a *Activity) LifecycleState(_ chasm.Context) chasm.LifecycleState { } } +// HasInflightWork implements chasm.TimeSkippable. It reports whether advancing virtual time would +// skip past real work, gating the framework's per-transaction time-skipping decision. +// +// Only the activity attempt is considered: it is in flight unless the activity is SCHEDULED and +// still waiting for its next dispatch — out of a start delay or retry backoff with the next-attempt +// time strictly in the (virtual) future. Once that time arrives (dispatch pushed to matching, +// waiting for a poller), the attempt is running (STARTED) or being cancelled (CANCEL_REQUESTED), or +// the activity has closed, this returns true. This mirrors the workflow time-skipping rule for +// activities, and means a closed activity is never time-skipped. +func (a *Activity) HasInflightWork(ctx chasm.Context) bool { + if a.GetStatus() != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { + return true + } + nextDispatch := a.attemptScheduleTime(a.LastAttempt.Get(ctx)) + if nextDispatch == nil || !ctx.Now(a).Before(nextDispatch.AsTime()) { + return true + } + return false +} + +// FindNextTargetTime implements chasm.TimeSkippable. The framework calls it after HasInflightWork has +// reported the activity is idle; the activity contributes its candidate virtual time by calling +// targetTime, leaving it untouched if it has nothing of its own to skip to. Today that is only the next +// attempt dispatch (start-delay end on the first attempt, retry-backoff end on a retry) while the +// activity is SCHEDULED-and-waiting, gated by the schedule-to-close deadline (the execution timeout) so +// we never skip past it. +// +// The framework pre-seeds targetTime with the configured fast-forward target, so the activity only +// contributes its own next-wake time; CompareAndSet keeps the minimum and the framework decides whether +// the fast-forward budget was reached. +// +// TODO(time-skipping): completion callback retry backoff +func (a *Activity) FindNextTargetTime( + ctx chasm.Context, + targetTime *chasm.TimeSkippingTargetTime, +) { + nextDispatch := a.attemptScheduleTime(a.LastAttempt.Get(ctx)) + if nextDispatch == nil || !ctx.Now(a).Before(nextDispatch.AsTime()) { + return + } + targetTime.CompareAndSet(nextDispatch.AsTime()) + // Gate by the schedule-to-close deadline (execution timeout) so we never skip past it. + if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { + targetTime.GateByExecutionTimeout(deadline) + } +} + func (a *Activity) ContextMetadata(_ chasm.Context) map[string]string { md := make(map[string]string, 2) if actType := a.GetActivityType().GetName(); actType != "" { diff --git a/chasm/lib/activity/config.go b/chasm/lib/activity/config.go index b21d7df65d8..ce01b5a69be 100644 --- a/chasm/lib/activity/config.go +++ b/chasm/lib/activity/config.go @@ -53,6 +53,7 @@ type Config struct { MaxCallbacksPerExecution dynamicconfig.IntPropertyFnWithNamespaceFilter DefaultActivityRetryPolicy dynamicconfig.TypedPropertyFnWithNamespaceFilter[retrypolicy.DefaultRetrySettings] StartDelayEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter + TimeSkippingEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter VisibilityMaxPageSize dynamicconfig.IntPropertyFnWithNamespaceFilter } @@ -68,6 +69,7 @@ func ConfigProvider(dc *dynamicconfig.Collection) *Config { LongPollTimeout: LongPollTimeout.Get(dc), MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc), StartDelayEnabled: StartDelayEnabled.Get(dc), + TimeSkippingEnabled: dynamicconfig.TimeSkippingEnabled.Get(dc), MaxCallbacksPerExecution: callback.MaxPerExecution.Get(dc), VisibilityMaxPageSize: dynamicconfig.FrontendVisibilityMaxPageSize.Get(dc), } diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 33e7ed15296..d8e7a938fb9 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -95,6 +95,10 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } } + if request.GetTimeSkippingConfig() != nil { + mutableContext.InitTimeSkippingConfig(request.GetTimeSkippingConfig()) + } + err = TransitionScheduled.Apply(newActivity, mutableContext, nil) if err != nil { return nil, err diff --git a/chasm/node_backend_mock.go b/chasm/node_backend_mock.go index 0fcc7faba92..80e833f0091 100644 --- a/chasm/node_backend_mock.go +++ b/chasm/node_backend_mock.go @@ -5,6 +5,7 @@ import ( "sync" "time" + commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" enumsspb "go.temporal.io/server/api/enums/v1" @@ -22,23 +23,27 @@ var _ NodeBackend = (*MockNodeBackend)(nil) // fields (thread-safe). type MockNodeBackend struct { // Optional function overrides. If nil, methods return zero-values. - HandleGetExecutionState func() *persistencespb.WorkflowExecutionState - HandleGetExecutionInfo func() *persistencespb.WorkflowExecutionInfo - HandleGetCurrentVersion func() int64 - HandleNextTransitionCount func() int64 - HandleGetApproximatePersistedSize func() int - HandleCurrentVersionedTransition func() *persistencespb.VersionedTransition - HandleGetWorkflowKey func() definition.WorkflowKey - HandleUpdateWorkflowStateStatus func(state enumsspb.WorkflowExecutionState, status enumspb.WorkflowExecutionStatus) (bool, error) - HandleIsWorkflow func() bool - HandleGetNexusCompletion func(ctx context.Context, requestID string) (nexusrpc.CompleteOperationOptions, error) - HandleGetNexusUpdateCompletion func(ctx context.Context, updateID string, requestID string) (nexusrpc.CompleteOperationOptions, error) - HandleAddHistoryEvent func(t enumspb.EventType, setAttributes func(*historypb.HistoryEvent)) *historypb.HistoryEvent - HandleLoadHistoryEvent func(ctx context.Context, token []byte) (*historypb.HistoryEvent, error) - HandleGenerateEventLoadToken func(event *historypb.HistoryEvent) ([]byte, error) - HandleHasAnyBufferedEvent func(filter func(*historypb.HistoryEvent) bool) bool - HandleGetNamespaceEntry func() *namespace.Namespace - HandleEndpointRegistry func() EndpointRegistry + HandleGetExecutionState func() *persistencespb.WorkflowExecutionState + HandleGetExecutionInfo func() *persistencespb.WorkflowExecutionInfo + HandleGetCurrentVersion func() int64 + HandleNextTransitionCount func() int64 + HandleGetApproximatePersistedSize func() int + HandleCurrentVersionedTransition func() *persistencespb.VersionedTransition + HandleGetWorkflowKey func() definition.WorkflowKey + HandleUpdateWorkflowStateStatus func(state enumsspb.WorkflowExecutionState, status enumspb.WorkflowExecutionStatus) (bool, error) + HandleIsWorkflow func() bool + HandleGetNexusCompletion func(ctx context.Context, requestID string) (nexusrpc.CompleteOperationOptions, error) + HandleGetNexusUpdateCompletion func(ctx context.Context, updateID string, requestID string) (nexusrpc.CompleteOperationOptions, error) + HandleAddHistoryEvent func(t enumspb.EventType, setAttributes func(*historypb.HistoryEvent)) *historypb.HistoryEvent + HandleLoadHistoryEvent func(ctx context.Context, token []byte) (*historypb.HistoryEvent, error) + HandleGenerateEventLoadToken func(event *historypb.HistoryEvent) ([]byte, error) + HandleHasAnyBufferedEvent func(filter func(*historypb.HistoryEvent) bool) bool + HandleGetNamespaceEntry func() *namespace.Namespace + HandleEndpointRegistry func() EndpointRegistry + HandleInitTimeSkippingConfig func(config *commonpb.TimeSkippingConfig) + HandleUpdateTimeSkippingConfig func(config *commonpb.TimeSkippingConfig) + HandleGetTimeSkippingInfo func() *persistencespb.TimeSkippingInfo + HandleRecordTimeSkippingTransition func(ctx context.Context, transition TimeSkippingTransition, archetype ArchetypeID) error // Recorded calls (protected by mu). mu sync.Mutex @@ -111,6 +116,32 @@ func (m *MockNodeBackend) AddTasks(ts ...tasks.Task) { } } +func (m *MockNodeBackend) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + if m.HandleInitTimeSkippingConfig != nil { + m.HandleInitTimeSkippingConfig(config) + } +} + +func (m *MockNodeBackend) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + if m.HandleUpdateTimeSkippingConfig != nil { + m.HandleUpdateTimeSkippingConfig(config) + } +} + +func (m *MockNodeBackend) GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo { + if m.HandleGetTimeSkippingInfo != nil { + return m.HandleGetTimeSkippingInfo() + } + return nil +} + +func (m *MockNodeBackend) RecordTimeSkippingTransition(ctx context.Context, transition TimeSkippingTransition, archetype ArchetypeID) error { + if m.HandleRecordTimeSkippingTransition != nil { + return m.HandleRecordTimeSkippingTransition(ctx, transition, archetype) + } + return nil +} + func (m *MockNodeBackend) DeleteCHASMPureTasks(maxScheduledTime time.Time) { m.mu.Lock() defer m.mu.Unlock() diff --git a/chasm/timeskipping_test.go b/chasm/timeskipping_test.go new file mode 100644 index 00000000000..cf7f8971486 --- /dev/null +++ b/chasm/timeskipping_test.go @@ -0,0 +1,103 @@ +package chasm + +import ( + "time" + + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/service/history/tasks" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// timeSkippingTaskNodes builds a tree with three CategoryTimer tasks already materialized +// (PhysicalTaskStatus == Created): a side-effect timer on the root, a side-effect timer on a child, +// and a pure timer on the root. The scheduled times are in the (virtual) future and carry no +// destination, so taskCategory classifies the side-effect tasks as CategoryTimer. +func (s *nodeSuite) timeSkippingTaskNodes() map[string]*persistencespb.ChasmNode { + now := s.timeSource.Now() + return map[string]*persistencespb.ChasmNode{ + "": { + Metadata: &persistencespb.ChasmNodeMetadata{ + InitialVersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1}, + LastUpdateVersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1}, + Attributes: &persistencespb.ChasmNodeMetadata_ComponentAttributes{ + ComponentAttributes: &persistencespb.ChasmComponentAttributes{ + TypeId: testComponentTypeID, + PureTasks: []*persistencespb.ChasmComponentAttributes_Task{ + { + TypeId: testPureTaskTypeID, + ScheduledTime: timestamppb.New(now.Add(time.Hour)), + VersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1}, + VersionedTransitionOffset: 1, + PhysicalTaskStatus: physicalTaskStatusCreated, + }, + }, + SideEffectTasks: []*persistencespb.ChasmComponentAttributes_Task{ + { + TypeId: testSideEffectTaskTypeID, + ScheduledTime: timestamppb.New(now.Add(time.Hour)), + VersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1}, + VersionedTransitionOffset: 2, + PhysicalTaskStatus: physicalTaskStatusCreated, + }, + }, + }, + }, + }, + }, + "SubComponent1": { + Metadata: &persistencespb.ChasmNodeMetadata{ + InitialVersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1}, + LastUpdateVersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1}, + Attributes: &persistencespb.ChasmNodeMetadata_ComponentAttributes{ + ComponentAttributes: &persistencespb.ChasmComponentAttributes{ + TypeId: testSubComponent1TypeID, + SideEffectTasks: []*persistencespb.ChasmComponentAttributes_Task{ + { + TypeId: testSideEffectTaskTypeID, + ScheduledTime: timestamppb.New(now.Add(2 * time.Hour)), + VersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1}, + VersionedTransitionOffset: 3, + PhysicalTaskStatus: physicalTaskStatusCreated, + }, + }, + }, + }, + }, + }, + } +} + +// TestRegenerateTasksForTimeSkipping_RestampsEachTimerExactlyOnce verifies a time-skipping +// regeneration re-emits exactly one physical task per logical CategoryTimer task — two side-effect +// timers plus a single pure timer (the earliest across the tree) — and not duplicates. This is the +// "generated only once" guarantee: even though every task starts already Created, the re-stamp emits +// each one exactly once. +func (s *nodeSuite) TestRegenerateTasksForTimeSkipping_RestampsEachTimerExactlyOnce() { + root, err := s.newTestTree(s.timeSkippingTaskNodes()) + s.NoError(err) + + err = root.regenerateTasksForTimeSkipping() + s.NoError(err) + + // 2 side-effect CategoryTimer tasks (root + SubComponent1) + 1 pure timer (earliest) = 3, all + // CategoryTimer. Not 6 (would indicate double-emission) and not 2 (would indicate the no-break + // re-stamp skipped already-Created tasks). + s.Equal(3, s.nodeBackend.NumTasksAdded()) + s.Len(s.nodeBackend.TasksByCategory[tasks.CategoryTimer], 3) +} + +// TestCloseTransaction_DoesNotRegenerateAlreadyMaterializedTimers verifies the complementary +// property: a plain CloseTransaction (no time-skipping transition, no user-state change) does NOT +// re-emit timer tasks that are already materialized. This is what keeps the active cluster from +// generating new timer tasks on every transaction — only a skip (via regenerateTasksForTimeSkipping) +// re-stamps them. +func (s *nodeSuite) TestCloseTransaction_DoesNotRegenerateAlreadyMaterializedTimers() { + root, err := s.newTestTree(s.timeSkippingTaskNodes()) + s.NoError(err) + + // Time skipping is not enabled (the mock backend's GetTimeSkippingInfo returns nil) and nothing was + // mutated, so the normal generation loop must skip the already-Created timer tasks. + _, err = root.CloseTransaction() + s.NoError(err) + s.Equal(0, s.nodeBackend.NumTasksAdded()) +} diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go new file mode 100644 index 00000000000..03532e03913 --- /dev/null +++ b/chasm/timeskippping.go @@ -0,0 +1,164 @@ +package chasm + +import ( + "time" + + commonpb "go.temporal.io/api/common/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" +) + +// Time skipping is a two-sided contract between the CHASM framework and a root component: +// +// - TimeSkippingController (this interface) is PROVIDED BY the framework and CALLED BY the +// component. It is embedded in MutableContext: the component opts in via InitTimeSkippingConfig, +// adjusts the config with UpdateTimeSkippingConfig, and reads it back with GetTimeSkippingConfig. +// - TimeSkippable (below) is IMPLEMENTED BY the root component and CALLED BY the framework, so the +// framework can ask whether the execution is idle enough to fast-forward, and where to skip to. +// +// A root component that wants time skipping does both: implement TimeSkippable, and call +// InitTimeSkippingConfig when the execution is created. + +// TimeSkippingController is the framework-provided surface through which a component controls time +// skipping for the whole execution. +// The interface is embedded in MutableContext (the component-facing surface). +// The backing mutable state (NodeBackend) implements the mutators and exposes the full TimeSkippingInfo +// to the framework via GetTimeSkippingInfo. +type TimeSkippingController interface { + InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) + UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) + GetTimeSkippingConfig() *commonpb.TimeSkippingConfig +} + +// TimeSkippable is the component-implemented half of the time-skipping contract (see +// TimeSkippingController above for the framework-provided half). A root component that opts into time +// skipping (by calling MutableContext.InitTimeSkippingConfig) implements this so the CHASM framework +// can call HasInflightWork during CloseTransaction to decide whether the execution is idle enough to +// fast-forward virtual time. +// +// ROOT COMPONENT ONLY. Time skipping is a per-execution concern decided once per transaction, so the +// framework only ever consults the ROOT component's TimeSkippable implementation (resolved via the +// root ComponentRef in closeTransactionHandleTimeSkipping). Implementing this interface on a non-root +// (child) component has NO effect — its methods are never called. Most components therefore should not +// implement it; only the root archetype of an execution that wants time skipping does. +// +// Time skipping only advances virtual time while the execution is idle and there is a fast-forward target time. +// When virtual time is advanced, the framework will regenerate all scheduled tasks so that their visibility timestamps +// are updated to the new virtual time. +// +// The framework drives two calls per transaction, in order, at the END of CloseTransaction if the execution has +// time skipping enabled: +// 1. HasInflightWork — the mandatory idle gate. It must return true whenever advancing virtual time +// would skip past real work (e.g. a standalone activity whose attempt has been dispatched to a +// worker or is currently running). When it returns true the framework does not skip and does not +// consult NextTimePoint. +// HasInflightWork is kept as a distinct method (rather than folded into FindNextTargetTime) to make the +// idle check an explicit, mandatory part of the contract for implementors. +// 2. FindNextTargetTime — the mandatory method to find the next target time to skip to. +// The input parameter is pre-seeded with the configured fast-forward target if there is a valid one. +// The implementation should directly call CompareAndSet to set a new possible target time point. +// A timeout time point is not a good target time point to skip to, but it is usually a good practice +// to call GateByExecutionTimeout to cap the target time point at the entire execution timeout if time skipping +// is not expected to jump past the execution timeout. +type TimeSkippable interface { + HasInflightWork(ctx Context) bool + FindNextTargetTime(ctx Context, nextTargetTime *TimeSkippingTargetTime) +} + +type TimeSkippingTargetTime time.Time + +func (n *TimeSkippingTargetTime) CompareAndSet(other time.Time) { + if other.IsZero() { + return + } + // Initialize when there is no candidate yet (current is zero), otherwise keep the earlier one. + current := n.GetTime() + if current.IsZero() || other.Before(current) { + *n = TimeSkippingTargetTime(other) + } +} + +func (n *TimeSkippingTargetTime) GateByExecutionTimeout(executionTimeout time.Time) { + current := n.GetTime() + if current.IsZero() || executionTimeout.IsZero() { + return + } + if current.After(executionTimeout) { + *n = TimeSkippingTargetTime(executionTimeout) + } +} + +func (n *TimeSkippingTargetTime) GetTime() time.Time { + if n == nil { + return time.Time{} + } + return time.Time(*n) +} + +func (n *TimeSkippingTargetTime) IsZero() bool { + return n.GetTime().IsZero() +} + +// TimeSkippingTransition is the result of a time-skipping decision: the virtual time the transition +// is recorded at (CurrentTime), the virtual time to advance to (TargetTime), and whether reaching it +// should disable time skipping because the fast-forward budget was reached (DisabledAfterFastForward). +// The applied skip delta is TargetTime - CurrentTime. It is an internal struct shared between the +// CHASM framework and the mutable-state backend — it is NOT part of the component-facing TimeSkippable +// contract. A component only returns a candidate time point (from NextTimePoint); the framework combines +// that with the configured fast-forward target/bound into this struct when recording the skip. The +// workflow (event-based) time-skipping path uses the same struct. +type TimeSkippingTransition struct { + // CurrentTime is the virtual "now" at which the transition is recorded (virtual frame, not wall + // clock). The skip delta added to the accumulated skipped duration is TargetTime - CurrentTime. + CurrentTime time.Time + TargetTime time.Time + DisabledAfterFastForward bool +} + +// IsValid reports whether the transition represents something to apply: a non-zero target to skip +// to, or a disable-after-fast-forward signal. It is nil-safe — a nil transition is not valid. +func (t *TimeSkippingTransition) IsValid() bool { + if t == nil { + return false + } + return !t.TargetTime.IsZero() || t.DisabledAfterFastForward +} + +// buildTimeSkippingTransition turns the component's resolved candidate (targetTime, already seeded with and +// min'd against the fast-forward target) into a TimeSkippingTransition to record, or nil if there is nothing +// to do. currentTime is virtual "now"; ffTarget is the configured fast-forward target (zero if none). +func buildTimeSkippingTransition( + currentTime time.Time, + targetTime *TimeSkippingTargetTime, + ffTarget time.Time, +) *TimeSkippingTransition { + target := targetTime.GetTime() + + // No valid future target to skip to. + if target.IsZero() || !target.After(currentTime) { + // Even with nothing to skip to, if the fast-forward target has already been reached, signal + // that time skipping should be disabled. + if !ffTarget.IsZero() && !ffTarget.After(currentTime) { + return &TimeSkippingTransition{ + CurrentTime: currentTime, + DisabledAfterFastForward: true, + } + } + return nil + } + + // We have a future target. Reaching it disables time skipping when it lands at or past the + // fast-forward target (the seed guarantees target never exceeds ffTarget when ffTarget is set). + disabledAfterFastForward := false + if !ffTarget.IsZero() { + disabledAfterFastForward = !target.Before(ffTarget) + } + return &TimeSkippingTransition{ + CurrentTime: currentTime, + TargetTime: target, + DisabledAfterFastForward: disabledAfterFastForward, + } +} + +func IsActiveFastForward(ff *persistencespb.FastForwardInfo) bool { + return ff != nil && !ff.HasReached && ff.TargetTime != nil && !ff.TargetTime.AsTime().IsZero() +} diff --git a/chasm/tree.go b/chasm/tree.go index b9076ff179f..874c61ccb45 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -234,6 +234,30 @@ type ( requestID string, ) (nexusrpc.CompleteOperationOptions, error) EndpointRegistry() EndpointRegistry + + // Time-skipping backend surface. The component-facing config control (GetTimeSkippingConfig) + // is exposed to executions through MutableContext, which delegates the mutators here; the + // framework itself reads the full TimeSkippingInfo (config AND fast-forward info) via + // GetTimeSkippingInfo. + InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) + UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) + GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo + // RecordTimeSkippingTransition records a single time-skipping transition for the execution: it + // advances the virtual clock to transition.TargetTime (and disables time skipping when the + // fast-forward target was reached). The caller (the framework's closeTransactionHandleTimeSkipping, + // or the workflow event path) is responsible for computing the final transition — including the + // min of the component candidate and the fast-forward target. archetype selects how the skip is + // recorded: the workflow archetype records a history event, others apply it directly to the TSI. + RecordTimeSkippingTransition(ctx context.Context, transition TimeSkippingTransition, archetype ArchetypeID) error + } + + // nowProvider is an optional capability of a NodeBackend: a backend that owns its own clock + // (mutable state, whose clock reflects the execution's time-skipping offset) implements it so + // that Node.Now() reads a single, time-skipping-aware clock shared with the rest of the + // execution. Backends that do not implement it (e.g. bare test mocks) cause Node.Now() to fall + // back to the tree's own timeSource. + nowProvider interface { + Now() time.Time } // NodePathEncoder is an interface for encoding and decoding node paths. @@ -1642,7 +1666,15 @@ func (n *Node) dataNodePath( func (n *Node) Now( _ Component, ) time.Time { + // Delegate to the backend when it owns a (time-skipping-aware) clock so that CHASM and the rest + // of the execution share a single clock; mutable state's clock reflects the execution's + // time-skipping offset. Fall back to the tree's own timeSource for backends that don't provide + // one (e.g. bare test mocks). + // // TODO: Now() could be different for components after we support Pause for CHASM components. + if np, ok := n.backend.(nowProvider); ok { + return np.Now() + } return n.timeSource.Now() } @@ -1716,6 +1748,10 @@ func (n *Node) CloseTransaction() (NodesMutation, error) { return NodesMutation{}, err } + if err := n.closeTransactionHandleTimeSkipping(); err != nil { + return NodesMutation{}, err + } + // Both user & system data mutation need to be returned and persisted. maps.Copy(n.mutation.UpdatedNodes, n.systemMutation.UpdatedNodes) maps.Copy(n.mutation.DeletedNodes, n.systemMutation.DeletedNodes) @@ -2085,6 +2121,152 @@ func (n *Node) closeTransactionUpdateComponentTasks( ) } +func (n *Node) closeTransactionHandleTimeSkipping() error { + // step1: time skipping only works on the root node + // todo@time-skipping: it seems CloseTransaction is only a method reasonable for the root node + // but currently the framework doesn't enforce this contract + if !softassert.That(n.logger, n.parent == nil, "closeTransactionHandleTimeSkipping must run on the root node") { + return nil + } + if n.ArchetypeID() == WorkflowArchetypeID { + // todo: removed when workflows are migrated to use chasm + return nil + } + tsi := n.backend.GetTimeSkippingInfo() + if !tsi.GetConfig().GetEnabled() { + return nil + } + + // The chasm framework distinguishes active vs passive via isActiveStateDirty: it is set only for + // active-cluster user-data mutations and is never set by replication (ApplyMutation/ApplySnapshot) + // — see its declaration and the same "active cluster" gate in closeTransactionUpdateComponentTasks. + // + // NOTE: isActiveStateDirty tracks chasm component user-data changes only. It does NOT track + // TimeSkippingInfo changes — TimeSkippingInfo lives in mutable state's WorkflowExecutionInfo (not a + // chasm node) and is tracked separately by MutableStateImpl.timeSkippingInfoUpdated. That is fine + // here: the decision is driven by the component work that schedules a future task worth skipping to + // (which sets isActiveStateDirty), and the skip the decision records marks the TimeSkippingInfo + // dirty via its own flag. The one implication is that a transaction which only changes + // TimeSkippingInfo (e.g. UpdateTimeSkippingConfig) without touching chasm state won't trigger the + // decision; it is picked up on the next active transaction. + if !n.isActiveStateDirty { + // Passive cluster: the active cluster already made the time-skipping decision and replicated the + // resulting TimeSkippingInfo (including the new accumulatedSkippedDuration). We must NOT re-run + // the decision here — recording another skip on top of the replicated one would diverge from + // the active cluster. + // + // TODO(time-skipping/chasm): on the passive cluster, detect that the replicated + // accumulatedSkippedDuration changed since the physical timer tasks were last generated and, if + // so, re-stamp them via regenerateTasksForTimeSkipping (the same no-break CategoryTimer + // regeneration the active path uses) — comparing the current accumulatedSkippedDuration against + // a baseline captured at tree load and refreshed each CloseTransaction. Until then, passive + // timer tasks are only re-stamped on the full-refresh replication path (taskRefresher.Refresh -> + // ChasmTree().RefreshTasks()); the incremental path (PartialRefresh, a no-op for CHASM) leaves + // them stale. State-based time-skipping replication is currently out of scope. + return nil + } + + // Active cluster: make and apply the time-skipping decision. + immutableContext := NewContext(context.Background(), n) + rootComponent, err := n.Component(immutableContext, ComponentRef{}) + if err != nil { + return err + } + tsRoot, ok := rootComponent.(TimeSkippable) + if !ok { + n.logger.Error( + "root component does not implement TimeSkippable when executions have turned on time skipping", + tag.Error(fmt.Errorf("type: %s", reflect.TypeOf(rootComponent).Name()))) + return nil + } + + // step2: call the component to find the next target time + now := n.Now(nil) + ff := tsi.GetFastForwardInfo() + var transition *TimeSkippingTransition + if !tsRoot.HasInflightWork(immutableContext) { + nextTargetTime := &TimeSkippingTargetTime{} + if IsActiveFastForward(ff) { + nextTargetTime.CompareAndSet(ff.GetTargetTime().AsTime()) + } + tsRoot.FindNextTargetTime(immutableContext, nextTargetTime) + if !nextTargetTime.IsZero() { + transition = buildTimeSkippingTransition(now, nextTargetTime, ff.TargetTime.AsTime()) + } + } + + // step3: if a time skipping transition is needed + if transition.IsValid() { + // record the transition + if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { + return err + } + // regenerate timer tasks + if err := n.regenerateTasksForTimeSkipping(); err != nil { + return err + } + } + return nil +} + +// regenerateTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition +// so their wall-clock visibility reflects the new accumulated skip. +// +// "Regenerate all" by default: every CategoryTimer task (pure and side-effect) across the tree is +// reset to physicalTaskStatusNone and re-generated. BOTH pure and side-effect tasks can be +// future-scheduled (CategoryTimer) tasks that need re-stamping, for example: +// - a side-effect activity retry timer task — the ActivityDispatchTask that fires to dispatch the +// next attempt after a retry backoff (or initial start delay); and +// - a pure start-workflow task — a pure timer that fires to schedule the workflow task after a +// start delay/backoff. +// +// Immediate Transfer/Outbound/Visibility tasks are left untouched — they fire at wall-clock now and +// are unaffected by accumulated skip. This default is deliberately broad so the process stays correct +// even as the set of skippable/in-flight tasks evolves; narrowing to only the tasks that actually need +// re-stamping is a future optimization. +// +// Physical tasks generated for the same logical tasks earlier this transaction (pre-skip, with the old +// accumulated skip) become stale; they fire early and no-op when their executor re-checks state. This +// matches the workflow RegenerateTimerTasksForTimeSkipping model (re-stamp, tolerate stale duplicates). +func (n *Node) regenerateTasksForTimeSkipping() error { + archetypeID := n.ArchetypeID() + + var firstPureTask *persistencespb.ChasmComponentAttributes_Task + var firstPureTaskNode *Node + + for nodePath, node := range n.andAllChildren() { + componentAttr := node.serializedNode.GetMetadata().GetComponentAttributes() + if componentAttr == nil { + continue + } + + // Side-effect timer tasks: reset and re-generate each one. Unlike the normal generation loop + // in closeTransactionUpdateComponentTasks, there is no early break on physicalTaskStatusCreated + // — we re-stamp every CategoryTimer task regardless of prior status. + for _, sideEffectTask := range componentAttr.GetSideEffectTasks() { + if taskCategory(sideEffectTask) != tasks.CategoryTimer { + continue + } + sideEffectTask.PhysicalTaskStatus = physicalTaskStatusNone + node.closeTransactionGeneratePhysicalSideEffectTask(sideEffectTask, nodePath, archetypeID) + } + + // Pure tasks are always future-scheduled (CategoryTimer). Reset them all, and track the earliest + // across the tree; closeTransactionGeneratePhysicalPureTask emits the single earliest one. + pureTasks := componentAttr.GetPureTasks() + for _, pureTask := range pureTasks { + pureTask.PhysicalTaskStatus = physicalTaskStatusNone + } + if len(pureTasks) > 0 && + (firstPureTask == nil || comparePureTasks(pureTasks[0], firstPureTask) < 0) { + firstPureTask = pureTasks[0] + firstPureTaskNode = node + } + } + + return n.closeTransactionGeneratePhysicalPureTask(firstPureTask, firstPureTaskNode, archetypeID) +} + func (n *Node) deserializeComponentTask( componentTask *persistencespb.ChasmComponentAttributes_Task, ) (any, error) { @@ -2701,6 +2883,13 @@ func (n *Node) applyUpdates( return nil } +// RefreshTasks resets the physical task status of every task in the tree so they are re-generated on +// the next CloseTransaction. It needs no time-skipping-specific logic: regeneration re-emits each task +// through MutableState.AddTasks, which applies ToRealTime to scheduled (CategoryTimer) tasks — i.e. it +// subtracts the execution's current accumulatedSkippedDuration (already in mutable state). So a refresh +// naturally re-stamps timer tasks at the correct wall-clock fire time for time skipping, just as the +// scoped closeTransactionRegenerateTimerTasks does after a skip; both rely on the same AddTasks -> +// ToRealTime conversion. func (n *Node) RefreshTasks() error { for _, node := range n.andAllChildren() { // Only reset task status here, the actual task generation will be done when diff --git a/go.mod b/go.mod index 64032ba8611..a0cd5ba178c 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 - go.temporal.io/api v1.63.0 + go.temporal.io/api v1.62.15-0.20260701034110-a7b499228361 go.temporal.io/auto-scaled-workers v0.0.0-20260622220320-9b1e3849116d go.temporal.io/sdk v1.41.1 go.uber.org/fx v1.24.0 diff --git a/go.sum b/go.sum index 36513b04183..b987aff262d 100644 --- a/go.sum +++ b/go.sum @@ -471,6 +471,10 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= +go.temporal.io/api v1.62.15-0.20260617232320-bf832cc7a07b h1:m4476YxxwyyK2v7+lCvXo4Fea22wXbpyvxKnfwN9gHU= +go.temporal.io/api v1.62.15-0.20260617232320-bf832cc7a07b/go.mod h1:0k75tRljEuELWGeXjEZZO7zYqBln4+1FrG6+IMOMy7Q= +go.temporal.io/api v1.62.15-0.20260701034110-a7b499228361 h1:51Ol1sND0K6BQi+Cxtk2pk0woTTUCXem8ZE8a7tIyPI= +go.temporal.io/api v1.62.15-0.20260701034110-a7b499228361/go.mod h1:0k75tRljEuELWGeXjEZZO7zYqBln4+1FrG6+IMOMy7Q= go.temporal.io/api v1.63.0 h1:YZFOTA0/thRUIUC4qunAWdHhPh/IG4vy/+WjfEvT+ZE= go.temporal.io/api v1.63.0/go.mod h1:0k75tRljEuELWGeXjEZZO7zYqBln4+1FrG6+IMOMy7Q= go.temporal.io/auto-scaled-workers v0.0.0-20260622220320-9b1e3849116d h1:f7+FCJHSrYWz9zvJp2OxKo8Fu/dsBUdnZZA+m5CEOS0= diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index a0887a58bd3..463c73b3a95 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -424,5 +424,8 @@ type ( AddWorkflowExecutionTimeSkippingTransitionedEvent( ctx context.Context, targetTime time.Time, disabledAfterFastForward bool) (*historypb.HistoryEvent, error) ApplyWorkflowExecutionTimeSkippingTransitionedEvent(ctx context.Context, event *historypb.HistoryEvent) error + // RecordTimeSkippingTransition is the archetype-aware time-skipping sink: workflows record a + // history event, other archetypes apply the transition to the TimeSkippingInfo directly. + RecordTimeSkippingTransition(ctx context.Context, transition chasm.TimeSkippingTransition, archetype chasm.ArchetypeID) error } ) diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index 7b12ad207dd..4d4478c7708 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -797,6 +797,20 @@ func (mr *MockMutableStateMockRecorder) AddWorkflowExecutionTimeSkippingTransiti return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkflowExecutionTimeSkippingTransitionedEvent", reflect.TypeOf((*MockMutableState)(nil).AddWorkflowExecutionTimeSkippingTransitionedEvent), ctx, targetTime, disabledAfterFastForward) } +// RecordTimeSkippingTransition mocks base method. +func (m *MockMutableState) RecordTimeSkippingTransition(ctx context.Context, transition chasm.TimeSkippingTransition, archetype chasm.ArchetypeID) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecordTimeSkippingTransition", ctx, transition, archetype) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecordTimeSkippingTransition indicates an expected call of RecordTimeSkippingTransition. +func (mr *MockMutableStateMockRecorder) RecordTimeSkippingTransition(ctx, transition, archetype any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecordTimeSkippingTransition", reflect.TypeOf((*MockMutableState)(nil).RecordTimeSkippingTransition), ctx, transition, archetype) +} + // AddWorkflowExecutionUnpausedEvent mocks base method. func (m *MockMutableState) AddWorkflowExecutionUnpausedEvent(identity, reason, requestID string) (*history.HistoryEvent, error) { m.ctrl.T.Helper() diff --git a/service/history/timer_queue_active_task_executor.go b/service/history/timer_queue_active_task_executor.go index dc07530e4fd..82c44f964db 100644 --- a/service/history/timer_queue_active_task_executor.go +++ b/service/history/timer_queue_active_task_executor.go @@ -3,7 +3,6 @@ package history import ( "context" "fmt" - "time" "github.com/google/uuid" commonpb "go.temporal.io/api/common/v1" @@ -937,9 +936,13 @@ func (t *timerQueueActiveTaskExecutor) executeTimeSkippingTimerTask( return errNoTimerFired } - _, err = mutableState.AddWorkflowExecutionTimeSkippingTransitionedEvent( - ctx, time.Time{}, true) - if err != nil { + // Disable time skipping through the archetype-aware sink: workflows record a history event; CHASM + // (and other archetypes) apply the disable-only transition directly to the TimeSkippingInfo. + if err = mutableState.RecordTimeSkippingTransition( + ctx, + chasm.TimeSkippingTransition{DisabledAfterFastForward: true}, + mutableState.ChasmTree().ArchetypeID(), + ); err != nil { return err } return t.updateWorkflowExecution(ctx, weContext, mutableState, false) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index a5fa564e6fc..11038e8ac2f 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -7688,6 +7688,8 @@ func (ms *MutableStateImpl) closeTransaction( ms.chasmNodeSizes[nodePath] = newSize } + // todo@time-skipping: chasm close transaction logic is after isStateDirty, + // need to check if this impacts chasm time skipping logic if isStateDirty { if err := ms.closeTransactionUpdateTransitionHistory( transactionPolicy, @@ -8140,6 +8142,7 @@ func (ms *MutableStateImpl) closeTransactionPrepareTasks( if err := ms.closeTransactionHandleActivityUserTimerTasks(transactionPolicy); err != nil { return err } + // todo: this is only for workflows not for chasm-based executions if regenerateTimerTasksForTimeSkipping { if err := ms.closeTransactionRegenTimerTasksForWorkflowTimeSkipping(transactionPolicy); err != nil { return err diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index 2953d69cc02..91594cee77d 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -9,6 +9,7 @@ import ( historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log/tag" @@ -478,3 +479,145 @@ func (ms *MutableStateImpl) closeTransactionRegenTimerTasksForWorkflowTimeSkippi return serviceerror.NewInternalf("unknown transaction policy: %v", transactionPolicy) } } + +// ============================================================================= +// Time Skipping for CHASM-based Executions +// ============================================================================= + +// applyTimeSkippingTransition mutates the execution's TimeSkippingInfo for a single skip transition: +// it advances AccumulatedSkippedDuration by (TargetTime - CurrentTime), toggles Config.Enabled, and +// marks the fast-forward target reached. It is the archetype-agnostic core shared by the workflow +// event path (ApplyWorkflowExecutionTimeSkippingTransitionedEvent) and the CHASM close-transaction +// path (RecordTimeSkippingTransition); the workflow path records the transition as a history event while +// CHASM (which has no history) applies it directly, but the TSI mutation is identical. +// +// A zero TargetTime means "no skip" (only valid together with DisabledAfterFastForward, e.g. a bound +// hit exactly). transition.CurrentTime must be in the virtual frame (ms.Now() / the event's virtual +// EventTime). +func (ms *MutableStateImpl) applyTimeSkippingTransition(transition chasm.TimeSkippingTransition) error { + opTag := tag.WorkflowActionWorkflowExecutionTimeSkippingTransitioned + invalidTransitionError := serviceerror.NewInternal("TimeSkippingTransitionedEvent failed to apply") + tsi := ms.executionInfo.GetTimeSkippingInfo() + if tsi == nil { + ms.logError("TimeSkippingTransitionedEvent failed to apply: TimeSkippingInfo is nil", opTag) + return invalidTransitionError + } + if transition.TargetTime.IsZero() && !transition.DisabledAfterFastForward { + ms.logError("TimeSkippingTransitionedEvent failed to apply: TargetTime is nil and disabled after fast-forward is false", opTag) + return invalidTransitionError + } + + if tsi.GetAccumulatedSkippedDuration() == nil { + tsi.AccumulatedSkippedDuration = durationpb.New(0) + } + accumulatedSkippedDuration := tsi.GetAccumulatedSkippedDuration().AsDuration() + if !transition.TargetTime.IsZero() { + accumulatedSkippedDuration += transition.TargetTime.Sub(transition.CurrentTime) + } + tsi.AccumulatedSkippedDuration = durationpb.New(accumulatedSkippedDuration) + tsi.Config.Enabled = !transition.DisabledAfterFastForward + + if transition.DisabledAfterFastForward && tsi.GetFastForwardInfo() != nil { + tsi.FastForwardInfo.HasReached = true + } + + ms.timeSkippingInfoUpdated = true + + // Re-stamp the CHASM fast-forward wake so its wall-clock fire time tracks the new accumulated skip + // (real fire = TargetTime - accumulatedSkippedDuration). Bundled with the accumulated update here + // rather than routed through the chasm task pipeline; no-op for workflows and once the target is + // reached. + ms.regenerateChasmFastForwardWakeTask() + return nil +} + +// chasmNoEventID is the sentinel event ID used when (re)computing time-skipping info for CHASM +// executions, which have no history events to anchor the configuration to. Real event IDs start at +// 1, so -1 is unambiguous. For CHASM the fast-forward wake is emitted as a CHASM task during +// CloseTransaction (anchored on the VersionedTransition), so this stub only ends up in the unused +// FastForwardInfo.SourceEventId field. +// +// TODO(time-skipping/chasm): emit the fast-forward wake as a ChasmTaskPure on the root component and +// anchor its staleness check on the VersionedTransition (as ChasmTaskInfo does) rather than an +// event ID. +const chasmNoEventID int64 = -1 + +// InitTimeSkippingConfig sets the execution's time-skipping configuration for a CHASM execution. It +// backs chasm.MutableContext.InitTimeSkippingConfig (via the chasm.NodeBackend interface). +// +// It shares initTimeSkippingInfo with the workflow path; that path is archetype-aware, so it does +// not emit a physical workflow timer task for CHASM executions (see applyFastForward). CHASM +// executions have no history events, so there is no event ID to anchor to (see chasmNoEventID) and +// no propagated state in this path. +func (ms *MutableStateImpl) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + ms.initTimeSkippingInfo(config, nil, chasmNoEventID) +} + +// UpdateTimeSkippingConfig updates the execution's time-skipping configuration for a CHASM +// execution. It backs chasm.MutableContext.UpdateTimeSkippingConfig (via the chasm.NodeBackend +// interface) and shares updateTimeSkippingInfo with the workflow path. +func (ms *MutableStateImpl) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + ms.updateTimeSkippingInfo(config, chasmNoEventID) +} + +// GetTimeSkippingConfig returns the execution's current time-skipping configuration, or nil if time +// skipping was never initialized. It backs chasm.MutableContext.GetTimeSkippingConfig (via the +// chasm.NodeBackend interface) so a root component can read back the config it set. +func (ms *MutableStateImpl) GetTimeSkippingConfig() *commonpb.TimeSkippingConfig { + return ms.GetExecutionInfo().GetTimeSkippingInfo().GetConfig() +} + +// GetTimeSkippingInfo returns the execution's full TimeSkippingInfo (config + fast-forward info). It +// is the CHASM framework's window into time-skipping state: GetTimeSkippingConfig is the projection +// exposed to executions, while the framework reads the fast-forward target/reached flag through here. +func (ms *MutableStateImpl) GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo { + return ms.GetExecutionInfo().GetTimeSkippingInfo() +} + +// RecordTimeSkippingTransition records a single time-skipping transition for the execution. It is the +// archetype-agnostic apply sink shared by both time-skipping front-ends: the CHASM framework's +// closeTransactionHandleTimeSkipping (which builds the transition for non-workflow CHASM executions) +// and the workflow event path (closeTransactionHandleTimeSkipping -> shouldExecuteTimeSkipping). The +// caller is responsible for computing the final transition — including the min of the component +// candidate and the fast-forward target; this method only records it. +// +// archetype selects how the skip is recorded: +// - the workflow archetype records a history event (AddWorkflowExecutionTimeSkippingTransitionedEvent), +// which in turn applies the transition to the TimeSkippingInfo; +// - all other archetypes have no history, so the transition is applied directly to the TSI. +// +// For CHASM executions, after this returns the physical timer tasks regenerated later in the same +// CloseTransaction pick up the new accumulated offset via AddTasks -> ToRealTime, so no separate +// timer-task regeneration is needed. +func (ms *MutableStateImpl) RecordTimeSkippingTransition( + ctx context.Context, + transition chasm.TimeSkippingTransition, + archetype chasm.ArchetypeID, +) error { + if !transition.IsValid() { + return nil + } + switch archetype { + case chasm.WorkflowArchetypeID: + // workflows require applying the transition through a history event + _, err := ms.AddWorkflowExecutionTimeSkippingTransitionedEvent( + ctx, transition.TargetTime, transition.DisabledAfterFastForward) + return err + default: + return ms.applyTimeSkippingTransition(transition) + } +} +func (ms *MutableStateImpl) regenerateChasmFastForwardWakeTask() { + if ms.IsWorkflow() { + return + } + ff := ms.executionInfo.GetTimeSkippingInfo().GetFastForwardInfo() + if ff == nil || ff.GetHasReached() || ff.GetTargetTime() == nil { + return + } + ms.AddTasks(&tasks.TimeSkippingTimerTask{ + WorkflowKey: ms.GetWorkflowKey(), + VisibilityTimestamp: ff.GetTargetTime().AsTime(), // virtual; AddTasks -> ToRealTime shifts to wall clock + EventID: chasmNoEventID, + }) +} diff --git a/service/history/workflow/timeskipping_test.go b/service/history/workflow/timeskipping_test.go index fb438df8abe..44b7f5b4460 100644 --- a/service/history/workflow/timeskipping_test.go +++ b/service/history/workflow/timeskipping_test.go @@ -11,10 +11,12 @@ import ( historypb "go.temporal.io/api/history/v1" enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/components/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" + "go.temporal.io/server/service/history/tasks" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -143,6 +145,54 @@ func (s *mutableStateSuite) TestPropagateTimeSkippingToNextRun_FastForwardInfo() }) } +// TestChasmFastForwardWake_ShortenedBySkip verifies that, for a CHASM (non-workflow) execution, the +// fast-forward wake timer is kept in mutable state and its wall-clock fire time is shortened by each +// skip: init emits the wake at the fast-forward target (no skip yet); a 10m skip re-emits it 10m +// sooner (real fire = target - accumulatedSkippedDuration). +func (s *mutableStateSuite) TestChasmFastForwardWake_ShortenedBySkip() { + // Force a CHASM (non-workflow) archetype so the wake stays in mutable state and is re-stamped here + // (rather than via the workflow event path). + mockChasmTree := historyi.NewMockChasmTree(s.controller) + s.mutableState.chasmTree = mockChasmTree + mockChasmTree.EXPECT().ArchetypeID().Return(chasm.WorkflowArchetypeID + 1).AnyTimes() + + // Init the fast-forward — emits the wake at the target (accumulated == 0, so real fire == target). + s.mutableState.InitTimeSkippingConfig(&commonpb.TimeSkippingConfig{ + Enabled: true, + FastForward: durationpb.New(time.Hour), + }) + ffTarget := s.mutableState.executionInfo.GetTimeSkippingInfo().GetFastForwardInfo().GetTargetTime().AsTime() + + initialWake := s.lastTimeSkippingTimerTask() + s.Require().NotNil(initialWake) + s.Equal(ffTarget, initialWake.GetVisibilityTime(), "initial wake fires at the fast-forward target") + + // Skip 10m of virtual time. + const skipped = 10 * time.Minute + base := s.mutableState.timeSource.Now() + s.NoError(s.mutableState.applyTimeSkippingTransition(chasm.TimeSkippingTransition{ + CurrentTime: base, + TargetTime: base.Add(skipped), + })) + s.Equal(skipped, s.mutableState.executionInfo.GetTimeSkippingInfo().GetAccumulatedSkippedDuration().AsDuration()) + + // The wake is re-stamped 10m sooner — the fast-forward timer is shortened by the skip. + skippedWake := s.lastTimeSkippingTimerTask() + s.Require().NotNil(skippedWake) + s.Equal(ffTarget.Add(-skipped), skippedWake.GetVisibilityTime(), "wake shortened by the skipped duration") + s.Equal(skipped, initialWake.GetVisibilityTime().Sub(skippedWake.GetVisibilityTime())) +} + +func (s *mutableStateSuite) lastTimeSkippingTimerTask() *tasks.TimeSkippingTimerTask { + var last *tasks.TimeSkippingTimerTask + for _, t := range s.mutableState.InsertTasks[tasks.CategoryTimer] { + if tst, ok := t.(*tasks.TimeSkippingTimerTask); ok { + last = tst + } + } + return last +} + func (s *mutableStateSuite) TestSnapshotTimeSkippingInfo_ForChildWorkflows() { newSource := func() *persistencespb.WorkflowExecutionInfo { s.mutableState.timeSource = clock.NewEventTimeSource() diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 7d893028672..b2c0a164b99 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -23,6 +23,7 @@ import ( "go.temporal.io/sdk/temporal" "go.temporal.io/server/chasm/lib/activity" "go.temporal.io/server/chasm/lib/callback" + "go.temporal.io/server/common" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" @@ -30,6 +31,7 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/payloads" + "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/searchattribute/sadefs" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/testing/parallelsuite" @@ -116,6 +118,129 @@ func (s *standaloneActivityTestSuite) newTestEnv(opts ...testcore.TestOption) *s return env } +// getMutableState loads the persisted mutable state of a standalone activity execution, so tests can +// inspect server-internal state such as the time-skipping info. +func (env *standaloneActivityEnv) getMutableState(t *testing.T, activityID, runID string) *persistence.GetWorkflowExecutionResponse { + shardID := common.WorkflowIDToHistoryShard( + env.NamespaceID().String(), + activityID, + env.GetTestClusterConfig().HistoryConfig.NumHistoryShards, + ) + ms, err := env.GetTestCluster().ExecutionManager().GetWorkflowExecution(testcore.NewContext(), &persistence.GetWorkflowExecutionRequest{ + ShardID: shardID, + NamespaceID: env.NamespaceID().String(), + WorkflowID: activityID, + RunID: runID, + ArchetypeID: activity.ArchetypeID, + }) + require.NoError(t, err) + return ms +} + +// TestTimeSkipping_StartDelayAndRetryBackoff verifies that a standalone activity, which opts into +// time skipping, fast-forwards both its start delay and its retry backoff while idle. With a 1h +// fast-forward budget, a 20m start delay plus a 20m retry backoff (~40m of virtual time) are skipped, +// so the run completes in seconds of wall clock and the accumulated skipped duration reflects the +// skipped delays. +func (s *standaloneActivityTestSuite) TestTimeSkipping_StartDelayAndRetryBackoff() { + env := s.newTestEnv() + t := s.T() + + // Standalone activities only opt into time skipping when the namespace flag is on. + env.GetTestCluster().OverrideDynamicConfig(t, dynamicconfig.TimeSkippingEnabled, []dynamicconfig.ConstrainedValue{ + {Constraints: dynamicconfig.Constraints{Namespace: env.Namespace().String()}, Value: true}, + }) + + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + + const startDelay = 20 * time.Minute + const retryBackoff = 20 * time.Minute + + wallStart := time.Now() + + startResp, err := env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + Input: defaultInput, + StartToCloseTimeout: durationpb.New(time.Minute), + ScheduleToCloseTimeout: durationpb.New(2 * time.Hour), // comfortably exceeds delay + backoff + StartDelay: durationpb.New(startDelay), + RetryPolicy: &commonpb.RetryPolicy{ + InitialInterval: durationpb.New(retryBackoff), + BackoffCoefficient: 1.0, + MaximumAttempts: 2, + }, + TimeSkippingConfig: &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForward: durationpb.New(time.Hour), + }, + }) + require.NoError(t, err) + runID := startResp.RunId + + pollTaskQueue := func() *workflowservice.PollActivityTaskQueueResponse { + resp, pollErr := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, + }) + require.NoError(t, pollErr) + return resp + } + + // Attempt 1's dispatch is deferred 20m by the start delay. Time skipping fast-forwards virtual + // time to the dispatch, so the task becomes pollable within seconds of wall clock rather than + // after 20m. (Without skipping this poll would block out its long-poll window and return empty.) + attempt1 := pollTaskQueue() + require.EqualValues(t, 1, attempt1.Attempt) + + // Fail attempt 1 retryably; the 20m retry backoff is likewise fast-forwarded. + _, err = env.FrontendClient().RespondActivityTaskFailed(s.Context(), &workflowservice.RespondActivityTaskFailedRequest{ + Namespace: env.Namespace().String(), + TaskToken: attempt1.TaskToken, + Failure: &failurepb.Failure{ + Message: "retryable failure", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{NonRetryable: false}}, + }, + }) + require.NoError(t, err) + + attempt2 := pollTaskQueue() + require.EqualValues(t, 2, attempt2.Attempt) + + _, err = env.FrontendClient().RespondActivityTaskCompleted(s.Context(), &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: attempt2.TaskToken, + Result: defaultResult, + }) + require.NoError(t, err) + + wallElapsed := time.Since(wallStart) + + // The run spanned ~40m of virtual time (start delay + retry backoff) but finished in seconds of + // wall clock — proof that both delays were skipped rather than waited out. + require.Less(t, wallElapsed, startDelay, "run should finish in well under the virtual start delay; took %v", wallElapsed) + + // The activity completed on its second attempt. + desc, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: runID, + }) + require.NoError(t, err) + require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED, desc.GetInfo().GetStatus()) + require.EqualValues(t, 2, desc.GetInfo().GetAttempt()) + + // The accumulated skipped duration reflects the skipped start delay + retry backoff (~40m). Time + // skipping then ends because the activity is closed (no further pending timers to skip to). + ms := env.getMutableState(t, activityID, runID) + accumulated := ms.State.GetExecutionInfo().GetTimeSkippingInfo().GetAccumulatedSkippedDuration().AsDuration() + require.InDelta(t, float64(startDelay+retryBackoff), float64(accumulated), float64(time.Minute), + "accumulated skipped duration %v should be ~%v", accumulated, startDelay+retryBackoff) +} + func (s *standaloneActivityTestSuite) TestIDReusePolicy() { env := s.newTestEnv() t := s.T() From 75ef32379ea293c356c36ce7ce51c7e808f6b90b Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Tue, 23 Jun 2026 16:33:57 -0700 Subject: [PATCH 02/13] merge init and update to set, and delete get TSC --- chasm/context.go | 18 +++------ chasm/context_mock.go | 14 +------ chasm/lib/activity/handler.go | 2 +- chasm/node_backend_mock.go | 15 ++----- chasm/timeskippping.go | 15 +++---- chasm/tree.go | 12 +++--- service/history/workflow/timeskipping.go | 40 +++++++------------ service/history/workflow/timeskipping_test.go | 2 +- 8 files changed, 40 insertions(+), 78 deletions(-) diff --git a/chasm/context.go b/chasm/context.go index 78b291615fe..bc200b38344 100644 --- a/chasm/context.go +++ b/chasm/context.go @@ -98,9 +98,9 @@ type MutableContext interface { SetUserMetadata(Component, *sdkpb.UserMetadata) error // TimeSkippingController is the framework-provided surface for managing this execution's - // time-skipping configuration (Init/Update/Get). A root component opts in by calling - // InitTimeSkippingConfig when the execution is created. These take no Component argument because - // the configuration is execution-scoped, like the ExecutionKey()/ExecutionInfo() accessors. + // time-skipping configuration. A root component opts in by calling SetTimeSkippingConfig when the + // execution is created (and again to adjust it). It takes no Component argument because the + // configuration is execution-scoped, like the ExecutionKey()/ExecutionInfo() accessors. TimeSkippingController // Get a Ref for the component @@ -283,16 +283,8 @@ func (c *mutableCtx) SetUserMetadata(component Component, md *sdkpb.UserMetadata return c.root.setComponentUserMetadata(component, md) } -func (c *mutableCtx) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { - c.root.backend.InitTimeSkippingConfig(config) -} - -func (c *mutableCtx) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { - c.root.backend.UpdateTimeSkippingConfig(config) -} - -func (c *mutableCtx) GetTimeSkippingConfig() *commonpb.TimeSkippingConfig { - return c.root.backend.GetTimeSkippingInfo().GetConfig() +func (c *mutableCtx) SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + c.root.backend.SetTimeSkippingConfig(config) } func (c *mutableCtx) withValue(key any, value any) Context { diff --git a/chasm/context_mock.go b/chasm/context_mock.go index 74eae6e61d1..9a6a0219d04 100644 --- a/chasm/context_mock.go +++ b/chasm/context_mock.go @@ -191,25 +191,13 @@ type MockMutableContext struct { TimeSkippingConfigInited bool } -func (c *MockMutableContext) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { +func (c *MockMutableContext) SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { c.mu.Lock() defer c.mu.Unlock() c.TimeSkippingConfig = config c.TimeSkippingConfigInited = true } -func (c *MockMutableContext) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { - c.mu.Lock() - defer c.mu.Unlock() - c.TimeSkippingConfig = config -} - -func (c *MockMutableContext) GetTimeSkippingConfig() *commonpb.TimeSkippingConfig { - c.mu.Lock() - defer c.mu.Unlock() - return c.TimeSkippingConfig -} - func (c *MockMutableContext) AddTask(component Component, attributes TaskAttributes, payload any) { c.mu.Lock() defer c.mu.Unlock() diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index d8e7a938fb9..3ffe750da25 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -96,7 +96,7 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St } if request.GetTimeSkippingConfig() != nil { - mutableContext.InitTimeSkippingConfig(request.GetTimeSkippingConfig()) + mutableContext.SetTimeSkippingConfig(request.GetTimeSkippingConfig()) } err = TransitionScheduled.Apply(newActivity, mutableContext, nil) diff --git a/chasm/node_backend_mock.go b/chasm/node_backend_mock.go index 80e833f0091..5d4e1587936 100644 --- a/chasm/node_backend_mock.go +++ b/chasm/node_backend_mock.go @@ -40,8 +40,7 @@ type MockNodeBackend struct { HandleHasAnyBufferedEvent func(filter func(*historypb.HistoryEvent) bool) bool HandleGetNamespaceEntry func() *namespace.Namespace HandleEndpointRegistry func() EndpointRegistry - HandleInitTimeSkippingConfig func(config *commonpb.TimeSkippingConfig) - HandleUpdateTimeSkippingConfig func(config *commonpb.TimeSkippingConfig) + HandleSetTimeSkippingConfig func(config *commonpb.TimeSkippingConfig) HandleGetTimeSkippingInfo func() *persistencespb.TimeSkippingInfo HandleRecordTimeSkippingTransition func(ctx context.Context, transition TimeSkippingTransition, archetype ArchetypeID) error @@ -116,15 +115,9 @@ func (m *MockNodeBackend) AddTasks(ts ...tasks.Task) { } } -func (m *MockNodeBackend) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { - if m.HandleInitTimeSkippingConfig != nil { - m.HandleInitTimeSkippingConfig(config) - } -} - -func (m *MockNodeBackend) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { - if m.HandleUpdateTimeSkippingConfig != nil { - m.HandleUpdateTimeSkippingConfig(config) +func (m *MockNodeBackend) SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + if m.HandleSetTimeSkippingConfig != nil { + m.HandleSetTimeSkippingConfig(config) } } diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 03532e03913..7e70d8e0362 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -10,13 +10,14 @@ import ( // Time skipping is a two-sided contract between the CHASM framework and a root component: // // - TimeSkippingController (this interface) is PROVIDED BY the framework and CALLED BY the -// component. It is embedded in MutableContext: the component opts in via InitTimeSkippingConfig, -// adjusts the config with UpdateTimeSkippingConfig, and reads it back with GetTimeSkippingConfig. +// component. It is embedded in MutableContext: the component opts in / adjusts the config via +// SetTimeSkippingConfig. (The framework reads the config back via NodeBackend.GetTimeSkippingInfo, +// not through this component-facing surface.) // - TimeSkippable (below) is IMPLEMENTED BY the root component and CALLED BY the framework, so the // framework can ask whether the execution is idle enough to fast-forward, and where to skip to. // // A root component that wants time skipping does both: implement TimeSkippable, and call -// InitTimeSkippingConfig when the execution is created. +// SetTimeSkippingConfig when the execution is created. // TimeSkippingController is the framework-provided surface through which a component controls time // skipping for the whole execution. @@ -24,14 +25,14 @@ import ( // The backing mutable state (NodeBackend) implements the mutators and exposes the full TimeSkippingInfo // to the framework via GetTimeSkippingInfo. type TimeSkippingController interface { - InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) - UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) - GetTimeSkippingConfig() *commonpb.TimeSkippingConfig + // SetTimeSkippingConfig sets the execution's time-skipping config: the first call establishes it, + // later calls update it in place (preserving the accumulated skipped duration). + SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) } // TimeSkippable is the component-implemented half of the time-skipping contract (see // TimeSkippingController above for the framework-provided half). A root component that opts into time -// skipping (by calling MutableContext.InitTimeSkippingConfig) implements this so the CHASM framework +// skipping (by calling MutableContext.SetTimeSkippingConfig) implements this so the CHASM framework // can call HasInflightWork during CloseTransaction to decide whether the execution is idle enough to // fast-forward virtual time. // diff --git a/chasm/tree.go b/chasm/tree.go index 874c61ccb45..fd7bd1c1626 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -235,12 +235,10 @@ type ( ) (nexusrpc.CompleteOperationOptions, error) EndpointRegistry() EndpointRegistry - // Time-skipping backend surface. The component-facing config control (GetTimeSkippingConfig) - // is exposed to executions through MutableContext, which delegates the mutators here; the - // framework itself reads the full TimeSkippingInfo (config AND fast-forward info) via - // GetTimeSkippingInfo. - InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) - UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) + // Time-skipping backend surface. The component-facing config control (SetTimeSkippingConfig) is + // exposed to executions through MutableContext, which delegates the mutator here; the framework + // itself reads the full TimeSkippingInfo (config AND fast-forward info) via GetTimeSkippingInfo. + SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo // RecordTimeSkippingTransition records a single time-skipping transition for the execution: it // advances the virtual clock to transition.TargetTime (and disables time skipping when the @@ -2147,7 +2145,7 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { // here: the decision is driven by the component work that schedules a future task worth skipping to // (which sets isActiveStateDirty), and the skip the decision records marks the TimeSkippingInfo // dirty via its own flag. The one implication is that a transaction which only changes - // TimeSkippingInfo (e.g. UpdateTimeSkippingConfig) without touching chasm state won't trigger the + // TimeSkippingInfo (e.g. SetTimeSkippingConfig) without touching chasm state won't trigger the // decision; it is picked up on the next active transaction. if !n.isActiveStateDirty { // Passive cluster: the active cluster already made the time-skipping decision and replicated the diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index 91594cee77d..9ef80394fc8 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -542,34 +542,24 @@ func (ms *MutableStateImpl) applyTimeSkippingTransition(transition chasm.TimeSki // event ID. const chasmNoEventID int64 = -1 -// InitTimeSkippingConfig sets the execution's time-skipping configuration for a CHASM execution. It -// backs chasm.MutableContext.InitTimeSkippingConfig (via the chasm.NodeBackend interface). -// -// It shares initTimeSkippingInfo with the workflow path; that path is archetype-aware, so it does -// not emit a physical workflow timer task for CHASM executions (see applyFastForward). CHASM -// executions have no history events, so there is no event ID to anchor to (see chasmNoEventID) and -// no propagated state in this path. -func (ms *MutableStateImpl) InitTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { - ms.initTimeSkippingInfo(config, nil, chasmNoEventID) -} - -// UpdateTimeSkippingConfig updates the execution's time-skipping configuration for a CHASM -// execution. It backs chasm.MutableContext.UpdateTimeSkippingConfig (via the chasm.NodeBackend -// interface) and shares updateTimeSkippingInfo with the workflow path. -func (ms *MutableStateImpl) UpdateTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { - ms.updateTimeSkippingInfo(config, chasmNoEventID) -} - -// GetTimeSkippingConfig returns the execution's current time-skipping configuration, or nil if time -// skipping was never initialized. It backs chasm.MutableContext.GetTimeSkippingConfig (via the -// chasm.NodeBackend interface) so a root component can read back the config it set. -func (ms *MutableStateImpl) GetTimeSkippingConfig() *commonpb.TimeSkippingConfig { - return ms.GetExecutionInfo().GetTimeSkippingInfo().GetConfig() +// SetTimeSkippingConfig sets the execution's time-skipping configuration for a CHASM execution. It +// backs chasm.MutableContext.SetTimeSkippingConfig (via the chasm.NodeBackend interface): the first +// call initializes the TimeSkippingInfo, and subsequent calls update the config in place (preserving +// the accumulated skipped duration). It reuses the workflow path's initTimeSkippingInfo / +// updateTimeSkippingInfo, which are archetype-aware (no physical workflow timer task for CHASM — see +// applyFastForward). CHASM executions have no history events to anchor to (see chasmNoEventID) and no +// propagated state on this path. +func (ms *MutableStateImpl) SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { + if ms.executionInfo.GetTimeSkippingInfo() == nil { + ms.initTimeSkippingInfo(config, nil, chasmNoEventID) + } else { + ms.updateTimeSkippingInfo(config, chasmNoEventID) + } } // GetTimeSkippingInfo returns the execution's full TimeSkippingInfo (config + fast-forward info). It -// is the CHASM framework's window into time-skipping state: GetTimeSkippingConfig is the projection -// exposed to executions, while the framework reads the fast-forward target/reached flag through here. +// is the CHASM framework's window into time-skipping state: the framework reads the config and the +// fast-forward target/reached flag through here (NodeBackend), not through a component-facing getter. func (ms *MutableStateImpl) GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo { return ms.GetExecutionInfo().GetTimeSkippingInfo() } diff --git a/service/history/workflow/timeskipping_test.go b/service/history/workflow/timeskipping_test.go index 44b7f5b4460..17b5e7fe9b9 100644 --- a/service/history/workflow/timeskipping_test.go +++ b/service/history/workflow/timeskipping_test.go @@ -157,7 +157,7 @@ func (s *mutableStateSuite) TestChasmFastForwardWake_ShortenedBySkip() { mockChasmTree.EXPECT().ArchetypeID().Return(chasm.WorkflowArchetypeID + 1).AnyTimes() // Init the fast-forward — emits the wake at the target (accumulated == 0, so real fire == target). - s.mutableState.InitTimeSkippingConfig(&commonpb.TimeSkippingConfig{ + s.mutableState.SetTimeSkippingConfig(&commonpb.TimeSkippingConfig{ Enabled: true, FastForward: durationpb.New(time.Hour), }) From 3172935b2809ee262935c1addee42736b07a5a0a Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Tue, 23 Jun 2026 19:10:16 -0700 Subject: [PATCH 03/13] move find next time point to frame --- chasm/lib/activity/activity.go | 27 ------------- chasm/timeskippping.go | 34 +++++------------ chasm/tree.go | 70 +++++++++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 54 deletions(-) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index d498bceb6e9..f6730010bbc 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -144,33 +144,6 @@ func (a *Activity) HasInflightWork(ctx chasm.Context) bool { return false } -// FindNextTargetTime implements chasm.TimeSkippable. The framework calls it after HasInflightWork has -// reported the activity is idle; the activity contributes its candidate virtual time by calling -// targetTime, leaving it untouched if it has nothing of its own to skip to. Today that is only the next -// attempt dispatch (start-delay end on the first attempt, retry-backoff end on a retry) while the -// activity is SCHEDULED-and-waiting, gated by the schedule-to-close deadline (the execution timeout) so -// we never skip past it. -// -// The framework pre-seeds targetTime with the configured fast-forward target, so the activity only -// contributes its own next-wake time; CompareAndSet keeps the minimum and the framework decides whether -// the fast-forward budget was reached. -// -// TODO(time-skipping): completion callback retry backoff -func (a *Activity) FindNextTargetTime( - ctx chasm.Context, - targetTime *chasm.TimeSkippingTargetTime, -) { - nextDispatch := a.attemptScheduleTime(a.LastAttempt.Get(ctx)) - if nextDispatch == nil || !ctx.Now(a).Before(nextDispatch.AsTime()) { - return - } - targetTime.CompareAndSet(nextDispatch.AsTime()) - // Gate by the schedule-to-close deadline (execution timeout) so we never skip past it. - if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { - targetTime.GateByExecutionTimeout(deadline) - } -} - func (a *Activity) ContextMetadata(_ chasm.Context) map[string]string { md := make(map[string]string, 2) if actType := a.GetActivityType().GetName(); actType != "" { diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 7e70d8e0362..4966927653f 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -46,23 +46,17 @@ type TimeSkippingController interface { // When virtual time is advanced, the framework will regenerate all scheduled tasks so that their visibility timestamps // are updated to the new virtual time. // -// The framework drives two calls per transaction, in order, at the END of CloseTransaction if the execution has -// time skipping enabled: -// 1. HasInflightWork — the mandatory idle gate. It must return true whenever advancing virtual time -// would skip past real work (e.g. a standalone activity whose attempt has been dispatched to a -// worker or is currently running). When it returns true the framework does not skip and does not -// consult NextTimePoint. -// HasInflightWork is kept as a distinct method (rather than folded into FindNextTargetTime) to make the -// idle check an explicit, mandatory part of the contract for implementors. -// 2. FindNextTargetTime — the mandatory method to find the next target time to skip to. -// The input parameter is pre-seeded with the configured fast-forward target if there is a valid one. -// The implementation should directly call CompareAndSet to set a new possible target time point. -// A timeout time point is not a good target time point to skip to, but it is usually a good practice -// to call GateByExecutionTimeout to cap the target time point at the entire execution timeout if time skipping -// is not expected to jump past the execution timeout. +// The contract is a single idle gate, consulted at the END of CloseTransaction when time skipping is enabled: +// +// HasInflightWork — the mandatory idle gate. It must return true whenever advancing virtual time would +// skip past real work (e.g. a standalone activity whose attempt has been dispatched to a worker or is +// currently running). When it returns true the framework does not skip. +// +// The component does NOT choose where to skip to. When HasInflightWork reports the execution idle, the +// FRAMEWORK finds the next target by scanning all scheduled (CategoryTimer) tasks across the tree — pure +// and side-effect — and taking the earliest one that still validates (see Node.findNextTargetTime). type TimeSkippable interface { HasInflightWork(ctx Context) bool - FindNextTargetTime(ctx Context, nextTargetTime *TimeSkippingTargetTime) } type TimeSkippingTargetTime time.Time @@ -78,16 +72,6 @@ func (n *TimeSkippingTargetTime) CompareAndSet(other time.Time) { } } -func (n *TimeSkippingTargetTime) GateByExecutionTimeout(executionTimeout time.Time) { - current := n.GetTime() - if current.IsZero() || executionTimeout.IsZero() { - return - } - if current.After(executionTimeout) { - *n = TimeSkippingTargetTime(executionTimeout) - } -} - func (n *TimeSkippingTargetTime) GetTime() time.Time { if n == nil { return time.Time{} diff --git a/chasm/tree.go b/chasm/tree.go index fd7bd1c1626..e4851cb8e5c 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2178,7 +2178,8 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } - // step2: call the component to find the next target time + // step2: while the root reports the execution idle, the framework — not the component — finds the + // next target time by scanning all scheduled timer tasks across the tree (see findNextTargetTime). now := n.Now(nil) ff := tsi.GetFastForwardInfo() var transition *TimeSkippingTransition @@ -2187,7 +2188,9 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { if IsActiveFastForward(ff) { nextTargetTime.CompareAndSet(ff.GetTargetTime().AsTime()) } - tsRoot.FindNextTargetTime(immutableContext, nextTargetTime) + if err := n.findNextTargetTime(nextTargetTime); err != nil { + return err + } if !nextTargetTime.IsZero() { transition = buildTimeSkippingTransition(now, nextTargetTime, ff.TargetTime.AsTime()) } @@ -2207,6 +2210,69 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } +// findNextTargetTime scans every scheduled (CategoryTimer) task across the tree — pure AND +// side-effect — and records the earliest one strictly after now into targetTime. This is the +// framework's "where to skip to": the component only reports idleness (TimeSkippable.HasInflightWork); +// it does not choose the target. +// +// A task is a candidate only if it still validates against its (current) component state. A task that +// has become invalid since it was scheduled must never be a skip target — e.g. a timeout task whose +// activity has already executed; skipping to it would advance virtual time to a wake that will only +// no-op. Component values are hydrated before validation, mirroring closeTransactionCleanupInvalidTasks. +func (n *Node) findNextTargetTime(targetTime *TimeSkippingTargetTime) error { + now := n.Now(nil) + validateContext := NewContext(newContextWithOperationIntent(context.Background(), OperationIntentProgress), n) + + for _, node := range n.andAllChildren() { + componentAttr := node.serializedNode.GetMetadata().GetComponentAttributes() + if componentAttr == nil { + continue + } + + if err := node.prepareComponentValue(validateContext); err != nil { + return err + } + + for _, taskList := range [][]*persistencespb.ChasmComponentAttributes_Task{ + componentAttr.GetPureTasks(), + componentAttr.GetSideEffectTasks(), + } { + for _, task := range taskList { + // Only future-scheduled timer tasks are skip targets; immediate + // transfer/outbound/visibility tasks fire at wall-clock now. + if taskCategory(task) != tasks.CategoryTimer { + continue + } + scheduledTime := task.GetScheduledTime().AsTime() + if !scheduledTime.After(now) { + continue + } + + // Never skip to a task that has become invalid (e.g. a timeout whose activity already + // executed). + taskInstance, err := node.deserializeComponentTask(task) + if err != nil { + return err + } + valid, err := node.validateTask( + validateContext, + TaskAttributes{ScheduledTime: scheduledTime, Destination: task.GetDestination()}, + taskInstance, + ) + if err != nil { + return err + } + if !valid { + continue + } + + targetTime.CompareAndSet(scheduledTime) + } + } + } + return nil +} + // regenerateTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition // so their wall-clock visibility reflects the new accumulated skip. // From 100f696bacc00c40ee55e2de267e003b206d97e5 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Tue, 23 Jun 2026 20:10:54 -0700 Subject: [PATCH 04/13] add TimeSkippingTargetTime back for discussion --- chasm/lib/activity/activity.go | 18 +++++ chasm/timeskippping.go | 105 +++++++++++++++-------------- chasm/tree.go | 118 +++++++++++---------------------- 3 files changed, 107 insertions(+), 134 deletions(-) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index f6730010bbc..2eca5632511 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -144,6 +144,24 @@ func (a *Activity) HasInflightWork(ctx chasm.Context) bool { return false } +// FindNextTargetTime implements chasm.TimeSkippable. The framework calls it once HasInflightWork has +// reported the activity idle. The only thing to skip to is the next attempt dispatch — the start-delay +// end on attempt 1, the retry-backoff end on a retry — capped at the schedule-to-close deadline so we +// never skip past it. The deadline is only a cap, never a target: we skip to the next dispatch, not to +// the timeout. +// +// TODO(time-skipping): completion callback retry backoff (Nexus callback) is also skippable. +func (a *Activity) FindNextTargetTime(ctx chasm.Context, target *chasm.TimeSkippingTargetTime) { + nextDispatch := a.attemptScheduleTime(a.LastAttempt.Get(ctx)) + if nextDispatch == nil || !ctx.Now(a).Before(nextDispatch.AsTime()) { + return + } + target.CompareAndSet(nextDispatch.AsTime()) + if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { + target.GateByExecutionTimeout(deadline) + } +} + func (a *Activity) ContextMetadata(_ chasm.Context) map[string]string { md := make(map[string]string, 2) if actType := a.GetActivityType().GetName(); actType != "" { diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 4966927653f..57064791f80 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -46,32 +46,58 @@ type TimeSkippingController interface { // When virtual time is advanced, the framework will regenerate all scheduled tasks so that their visibility timestamps // are updated to the new virtual time. // -// The contract is a single idle gate, consulted at the END of CloseTransaction when time skipping is enabled: +// The framework consults two methods at the END of CloseTransaction when time skipping is enabled: // // HasInflightWork — the mandatory idle gate. It must return true whenever advancing virtual time would // skip past real work (e.g. a standalone activity whose attempt has been dispatched to a worker or is // currently running). When it returns true the framework does not skip. // -// The component does NOT choose where to skip to. When HasInflightWork reports the execution idle, the -// FRAMEWORK finds the next target by scanning all scheduled (CategoryTimer) tasks across the tree — pure -// and side-effect — and taking the earliest one that still validates (see Node.findNextTargetTime). +// FindNextTargetTime — choose where to skip to. The component contributes its next wake time into +// target (via CompareAndSet, optionally capped by GateByExecutionTimeout); the framework then combines +// the result with the configured fast-forward target. This is component-only by design: only the +// component knows which of its pending timers is a legitimate wake (a retry backoff / next dispatch) +// versus an execution/run timeout that virtual time must never be advanced to. The framework only +// knows a task's category (CategoryTimer), not its meaning, so it cannot choose the target itself and +// provides no default scan. type TimeSkippable interface { HasInflightWork(ctx Context) bool + FindNextTargetTime(ctx Context, target *TimeSkippingTargetTime) } +// TimeSkippingTargetTime is a nil-safe accumulator passed to TimeSkippable.FindNextTargetTime for a +// component to contribute candidate skip targets (virtual time). It is the easy way to implement +// FindNextTargetTime, and is intentionally kept separate from TimeSkippingTransition (which is the +// decision the framework records into the history substrate). A zero value means "no candidate yet": +// CompareAndSet keeps the earliest candidate, GateByExecutionTimeout caps it at an execution deadline. +// It is a defined type over time.Time; the mutating methods use a pointer receiver so the component's +// calls are visible to the framework through the shared pointer. type TimeSkippingTargetTime time.Time +// CompareAndSet keeps the earlier candidate: initializes when unset, otherwise keeps the earlier of the +// current value and other. A zero other is ignored. func (n *TimeSkippingTargetTime) CompareAndSet(other time.Time) { if other.IsZero() { return } - // Initialize when there is no candidate yet (current is zero), otherwise keep the earlier one. current := n.GetTime() if current.IsZero() || other.Before(current) { *n = TimeSkippingTargetTime(other) } } +// GateByExecutionTimeout caps the candidate at executionTimeout so a skip never advances past it. No-op +// if there is no candidate yet or the timeout is unset. +func (n *TimeSkippingTargetTime) GateByExecutionTimeout(executionTimeout time.Time) { + current := n.GetTime() + if current.IsZero() || executionTimeout.IsZero() { + return + } + if current.After(executionTimeout) { + *n = TimeSkippingTargetTime(executionTimeout) + } +} + +// GetTime returns the accumulated candidate (zero if unset). Nil-safe. func (n *TimeSkippingTargetTime) GetTime() time.Time { if n == nil { return time.Time{} @@ -79,18 +105,15 @@ func (n *TimeSkippingTargetTime) GetTime() time.Time { return time.Time(*n) } -func (n *TimeSkippingTargetTime) IsZero() bool { - return n.GetTime().IsZero() -} - -// TimeSkippingTransition is the result of a time-skipping decision: the virtual time the transition -// is recorded at (CurrentTime), the virtual time to advance to (TargetTime), and whether reaching it -// should disable time skipping because the fast-forward budget was reached (DisabledAfterFastForward). -// The applied skip delta is TargetTime - CurrentTime. It is an internal struct shared between the -// CHASM framework and the mutable-state backend — it is NOT part of the component-facing TimeSkippable -// contract. A component only returns a candidate time point (from NextTimePoint); the framework combines -// that with the configured fast-forward target/bound into this struct when recording the skip. The -// workflow (event-based) time-skipping path uses the same struct. +// TimeSkippingTransition is a time-skipping decision: the virtual time the transition is recorded at +// (CurrentTime), the virtual time to advance to (TargetTime), and whether reaching it should disable +// time skipping because the fast-forward budget was reached (DisabledAfterFastForward). The applied +// skip delta is TargetTime - CurrentTime. It is an internal struct shared between the CHASM framework +// and the mutable-state backend — NOT part of the component-facing contract. +// +// The framework builds it directly: it seeds TargetTime with the fast-forward target and folds in each +// valid pending timer via considerTarget (keeping the earliest), then finalizes CurrentTime and +// DisabledAfterFastForward. The workflow (event-based) path uses the same struct. type TimeSkippingTransition struct { // CurrentTime is the virtual "now" at which the transition is recorded (virtual frame, not wall // clock). The skip delta added to the accumulated skipped duration is TargetTime - CurrentTime. @@ -99,6 +122,18 @@ type TimeSkippingTransition struct { DisabledAfterFastForward bool } +// considerTarget folds a candidate skip target into the transition, keeping the earliest non-zero one. +// The framework calls it to combine the fast-forward target and each valid pending timer into a single +// TargetTime; zero candidates are ignored. +func (t *TimeSkippingTransition) considerTarget(candidate time.Time) { + if candidate.IsZero() { + return + } + if t.TargetTime.IsZero() || candidate.Before(t.TargetTime) { + t.TargetTime = candidate + } +} + // IsValid reports whether the transition represents something to apply: a non-zero target to skip // to, or a disable-after-fast-forward signal. It is nil-safe — a nil transition is not valid. func (t *TimeSkippingTransition) IsValid() bool { @@ -108,42 +143,6 @@ func (t *TimeSkippingTransition) IsValid() bool { return !t.TargetTime.IsZero() || t.DisabledAfterFastForward } -// buildTimeSkippingTransition turns the component's resolved candidate (targetTime, already seeded with and -// min'd against the fast-forward target) into a TimeSkippingTransition to record, or nil if there is nothing -// to do. currentTime is virtual "now"; ffTarget is the configured fast-forward target (zero if none). -func buildTimeSkippingTransition( - currentTime time.Time, - targetTime *TimeSkippingTargetTime, - ffTarget time.Time, -) *TimeSkippingTransition { - target := targetTime.GetTime() - - // No valid future target to skip to. - if target.IsZero() || !target.After(currentTime) { - // Even with nothing to skip to, if the fast-forward target has already been reached, signal - // that time skipping should be disabled. - if !ffTarget.IsZero() && !ffTarget.After(currentTime) { - return &TimeSkippingTransition{ - CurrentTime: currentTime, - DisabledAfterFastForward: true, - } - } - return nil - } - - // We have a future target. Reaching it disables time skipping when it lands at or past the - // fast-forward target (the seed guarantees target never exceeds ffTarget when ffTarget is set). - disabledAfterFastForward := false - if !ffTarget.IsZero() { - disabledAfterFastForward = !target.Before(ffTarget) - } - return &TimeSkippingTransition{ - CurrentTime: currentTime, - TargetTime: target, - DisabledAfterFastForward: disabledAfterFastForward, - } -} - func IsActiveFastForward(ff *persistencespb.FastForwardInfo) bool { return ff != nil && !ff.HasReached && ff.TargetTime != nil && !ff.TargetTime.AsTime().IsZero() } diff --git a/chasm/tree.go b/chasm/tree.go index e4851cb8e5c..ed13c92d1b2 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2178,31 +2178,50 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } - // step2: while the root reports the execution idle, the framework — not the component — finds the - // next target time by scanning all scheduled timer tasks across the tree (see findNextTargetTime). + // step2: while the root reports the execution idle, build the transition: seed TargetTime with the + // fast-forward target, fold in the component's chosen wake target, then finalize. + if tsRoot.HasInflightWork(immutableContext) { + return nil + } + now := n.Now(nil) - ff := tsi.GetFastForwardInfo() - var transition *TimeSkippingTransition - if !tsRoot.HasInflightWork(immutableContext) { - nextTargetTime := &TimeSkippingTargetTime{} - if IsActiveFastForward(ff) { - nextTargetTime.CompareAndSet(ff.GetTargetTime().AsTime()) - } - if err := n.findNextTargetTime(nextTargetTime); err != nil { - return err - } - if !nextTargetTime.IsZero() { - transition = buildTimeSkippingTransition(now, nextTargetTime, ff.TargetTime.AsTime()) - } + var ffTarget time.Time + if ff := tsi.GetFastForwardInfo(); IsActiveFastForward(ff) { + ffTarget = ff.GetTargetTime().AsTime() + } + + transition := &TimeSkippingTransition{CurrentTime: now} + transition.considerTarget(ffTarget) + + // The component chooses where to skip to. Only the component knows its task semantics — in + // particular which pending timer is a legitimate wake (e.g. a retry backoff / next dispatch) versus + // an execution/run timeout that virtual time must never be advanced to. The framework cannot make + // that distinction (it only knows a task's category, not its meaning), so there is no framework-side + // default scan — FindNextTargetTime is the sole source of the wake target. + target := &TimeSkippingTargetTime{} + tsRoot.FindNextTargetTime(immutableContext, target) + transition.considerTarget(target.GetTime()) + + // Finalize. The accumulated TargetTime is min(fast-forward target, component's wake target). + switch { + case transition.TargetTime.IsZero(): + // Nothing to skip to and no fast-forward budget. + case !transition.TargetTime.After(now): + // The earliest candidate is the already-reached fast-forward target: disable only, no skip. + transition.TargetTime = time.Time{} + transition.DisabledAfterFastForward = true + case !ffTarget.IsZero() && !transition.TargetTime.Before(ffTarget): + // Skipping to (or past) the fast-forward target: skip and then disable. + transition.DisabledAfterFastForward = true + default: + // A regular skip to the component's wake target, still short of the fast-forward budget. } - // step3: if a time skipping transition is needed + // step3: if a time skipping transition is needed, record it and regenerate timer tasks. if transition.IsValid() { - // record the transition if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { return err } - // regenerate timer tasks if err := n.regenerateTasksForTimeSkipping(); err != nil { return err } @@ -2210,69 +2229,6 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } -// findNextTargetTime scans every scheduled (CategoryTimer) task across the tree — pure AND -// side-effect — and records the earliest one strictly after now into targetTime. This is the -// framework's "where to skip to": the component only reports idleness (TimeSkippable.HasInflightWork); -// it does not choose the target. -// -// A task is a candidate only if it still validates against its (current) component state. A task that -// has become invalid since it was scheduled must never be a skip target — e.g. a timeout task whose -// activity has already executed; skipping to it would advance virtual time to a wake that will only -// no-op. Component values are hydrated before validation, mirroring closeTransactionCleanupInvalidTasks. -func (n *Node) findNextTargetTime(targetTime *TimeSkippingTargetTime) error { - now := n.Now(nil) - validateContext := NewContext(newContextWithOperationIntent(context.Background(), OperationIntentProgress), n) - - for _, node := range n.andAllChildren() { - componentAttr := node.serializedNode.GetMetadata().GetComponentAttributes() - if componentAttr == nil { - continue - } - - if err := node.prepareComponentValue(validateContext); err != nil { - return err - } - - for _, taskList := range [][]*persistencespb.ChasmComponentAttributes_Task{ - componentAttr.GetPureTasks(), - componentAttr.GetSideEffectTasks(), - } { - for _, task := range taskList { - // Only future-scheduled timer tasks are skip targets; immediate - // transfer/outbound/visibility tasks fire at wall-clock now. - if taskCategory(task) != tasks.CategoryTimer { - continue - } - scheduledTime := task.GetScheduledTime().AsTime() - if !scheduledTime.After(now) { - continue - } - - // Never skip to a task that has become invalid (e.g. a timeout whose activity already - // executed). - taskInstance, err := node.deserializeComponentTask(task) - if err != nil { - return err - } - valid, err := node.validateTask( - validateContext, - TaskAttributes{ScheduledTime: scheduledTime, Destination: task.GetDestination()}, - taskInstance, - ) - if err != nil { - return err - } - if !valid { - continue - } - - targetTime.CompareAndSet(scheduledTime) - } - } - } - return nil -} - // regenerateTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition // so their wall-clock visibility reflects the new accumulated skip. // From 0c2158e903e9320309370f85a00838a17fc91759 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Wed, 24 Jun 2026 10:35:00 -0700 Subject: [PATCH 05/13] add validation for both find next target and regen simiplify the time skipping protocol between framework and component users --- chasm/lib/activity/activity.go | 24 ++--- chasm/timeskipping_test.go | 13 ++- chasm/timeskippping.go | 117 +++---------------------- chasm/tree.go | 154 +++++++++++++++++++++++---------- 4 files changed, 139 insertions(+), 169 deletions(-) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index 2eca5632511..b0043ae51fe 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -56,6 +56,12 @@ var ( var _ chasm.VisibilitySearchAttributesProvider = (*Activity)(nil) var _ callback.CompletionSource = (*Activity)(nil) +// The activity opts into time skipping: it gates idleness via TimeSkippable and chooses its own skip +// target via the optional TimeSkippingTargetFinder (it must, because its pending timers include timeout +// tasks the framework's default scan must not target). These compile-time assertions keep the optional +// interface from silently desyncing if a method signature drifts. +var _ chasm.TimeSkippable = (*Activity)(nil) + type ActivityStore interface { // RecordCompleted applies the provided function to record activity completion RecordCompleted(ctx chasm.MutableContext, applyFn func(ctx chasm.MutableContext) error) error @@ -144,24 +150,6 @@ func (a *Activity) HasInflightWork(ctx chasm.Context) bool { return false } -// FindNextTargetTime implements chasm.TimeSkippable. The framework calls it once HasInflightWork has -// reported the activity idle. The only thing to skip to is the next attempt dispatch — the start-delay -// end on attempt 1, the retry-backoff end on a retry — capped at the schedule-to-close deadline so we -// never skip past it. The deadline is only a cap, never a target: we skip to the next dispatch, not to -// the timeout. -// -// TODO(time-skipping): completion callback retry backoff (Nexus callback) is also skippable. -func (a *Activity) FindNextTargetTime(ctx chasm.Context, target *chasm.TimeSkippingTargetTime) { - nextDispatch := a.attemptScheduleTime(a.LastAttempt.Get(ctx)) - if nextDispatch == nil || !ctx.Now(a).Before(nextDispatch.AsTime()) { - return - } - target.CompareAndSet(nextDispatch.AsTime()) - if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { - target.GateByExecutionTimeout(deadline) - } -} - func (a *Activity) ContextMetadata(_ chasm.Context) map[string]string { md := make(map[string]string, 2) if actType := a.GetActivityType().GetName(); actType != "" { diff --git a/chasm/timeskipping_test.go b/chasm/timeskipping_test.go index cf7f8971486..03f031009fb 100644 --- a/chasm/timeskipping_test.go +++ b/chasm/timeskipping_test.go @@ -1,10 +1,12 @@ package chasm import ( + "context" "time" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/service/history/tasks" + "go.uber.org/mock/gomock" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -76,7 +78,16 @@ func (s *nodeSuite) TestRegenerateTasksForTimeSkipping_RestampsEachTimerExactlyO root, err := s.newTestTree(s.timeSkippingTaskNodes()) s.NoError(err) - err = root.regenerateTasksForTimeSkipping() + // regenerateTasksForTimeSkipping validates each task before re-stamping it; all three test timers + // are valid. + s.testLibrary.mockSideEffectTaskHandler.EXPECT(). + Validate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(true, nil).AnyTimes() + s.testLibrary.mockPureTaskHandler.EXPECT(). + Validate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(true, nil).AnyTimes() + + err = root.regenerateTasksForTimeSkipping(NewContext(context.Background(), root)) s.NoError(err) // 2 side-effect CategoryTimer tasks (root + SubComponent1) + 1 pure timer (earliest) = 3, all diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 57064791f80..24d689e035f 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -9,100 +9,25 @@ import ( // Time skipping is a two-sided contract between the CHASM framework and a root component: // -// - TimeSkippingController (this interface) is PROVIDED BY the framework and CALLED BY the -// component. It is embedded in MutableContext: the component opts in / adjusts the config via -// SetTimeSkippingConfig. (The framework reads the config back via NodeBackend.GetTimeSkippingInfo, -// not through this component-facing surface.) -// - TimeSkippable (below) is IMPLEMENTED BY the root component and CALLED BY the framework, so the -// framework can ask whether the execution is idle enough to fast-forward, and where to skip to. -// -// A root component that wants time skipping does both: implement TimeSkippable, and call -// SetTimeSkippingConfig when the execution is created. +// - TimeSkippingController is PROVIDED BY the framework and CALLED BY the +// component. It is embedded in MutableContext and allows the component to opt in / adjust the config via +// SetTimeSkippingConfig. +// - TimeSkippable is the mandatory component-implemented interface that must be implemented by the root +// component of an execution to make time skipping work. -// TimeSkippingController is the framework-provided surface through which a component controls time -// skipping for the whole execution. -// The interface is embedded in MutableContext (the component-facing surface). -// The backing mutable state (NodeBackend) implements the mutators and exposes the full TimeSkippingInfo -// to the framework via GetTimeSkippingInfo. type TimeSkippingController interface { + // SetTimeSkippingConfig sets the execution's time-skipping config: the first call establishes it, // later calls update it in place (preserving the accumulated skipped duration). SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) } -// TimeSkippable is the component-implemented half of the time-skipping contract (see -// TimeSkippingController above for the framework-provided half). A root component that opts into time -// skipping (by calling MutableContext.SetTimeSkippingConfig) implements this so the CHASM framework -// can call HasInflightWork during CloseTransaction to decide whether the execution is idle enough to -// fast-forward virtual time. -// -// ROOT COMPONENT ONLY. Time skipping is a per-execution concern decided once per transaction, so the -// framework only ever consults the ROOT component's TimeSkippable implementation (resolved via the -// root ComponentRef in closeTransactionHandleTimeSkipping). Implementing this interface on a non-root -// (child) component has NO effect — its methods are never called. Most components therefore should not -// implement it; only the root archetype of an execution that wants time skipping does. -// -// Time skipping only advances virtual time while the execution is idle and there is a fast-forward target time. -// When virtual time is advanced, the framework will regenerate all scheduled tasks so that their visibility timestamps -// are updated to the new virtual time. -// -// The framework consults two methods at the END of CloseTransaction when time skipping is enabled: -// -// HasInflightWork — the mandatory idle gate. It must return true whenever advancing virtual time would -// skip past real work (e.g. a standalone activity whose attempt has been dispatched to a worker or is -// currently running). When it returns true the framework does not skip. -// -// FindNextTargetTime — choose where to skip to. The component contributes its next wake time into -// target (via CompareAndSet, optionally capped by GateByExecutionTimeout); the framework then combines -// the result with the configured fast-forward target. This is component-only by design: only the -// component knows which of its pending timers is a legitimate wake (a retry backoff / next dispatch) -// versus an execution/run timeout that virtual time must never be advanced to. The framework only -// knows a task's category (CategoryTimer), not its meaning, so it cannot choose the target itself and -// provides no default scan. type TimeSkippable interface { - HasInflightWork(ctx Context) bool - FindNextTargetTime(ctx Context, target *TimeSkippingTargetTime) -} -// TimeSkippingTargetTime is a nil-safe accumulator passed to TimeSkippable.FindNextTargetTime for a -// component to contribute candidate skip targets (virtual time). It is the easy way to implement -// FindNextTargetTime, and is intentionally kept separate from TimeSkippingTransition (which is the -// decision the framework records into the history substrate). A zero value means "no candidate yet": -// CompareAndSet keeps the earliest candidate, GateByExecutionTimeout caps it at an execution deadline. -// It is a defined type over time.Time; the mutating methods use a pointer receiver so the component's -// calls are visible to the framework through the shared pointer. -type TimeSkippingTargetTime time.Time - -// CompareAndSet keeps the earlier candidate: initializes when unset, otherwise keeps the earlier of the -// current value and other. A zero other is ignored. -func (n *TimeSkippingTargetTime) CompareAndSet(other time.Time) { - if other.IsZero() { - return - } - current := n.GetTime() - if current.IsZero() || other.Before(current) { - *n = TimeSkippingTargetTime(other) - } -} - -// GateByExecutionTimeout caps the candidate at executionTimeout so a skip never advances past it. No-op -// if there is no candidate yet or the timeout is unset. -func (n *TimeSkippingTargetTime) GateByExecutionTimeout(executionTimeout time.Time) { - current := n.GetTime() - if current.IsZero() || executionTimeout.IsZero() { - return - } - if current.After(executionTimeout) { - *n = TimeSkippingTargetTime(executionTimeout) - } -} - -// GetTime returns the accumulated candidate (zero if unset). Nil-safe. -func (n *TimeSkippingTargetTime) GetTime() time.Time { - if n == nil { - return time.Time{} - } - return time.Time(*n) + // HasInflightWork returns true when an execution has in-flight work (e.g. a standalone activity + // whose attempt has been dispatched to a worker or is currently running). Time skipping won't skip + // any time when it returns true so that in-flight work has enough time to complete. + HasInflightWork(ctx Context) bool } // TimeSkippingTransition is a time-skipping decision: the virtual time the transition is recorded at @@ -115,32 +40,14 @@ func (n *TimeSkippingTargetTime) GetTime() time.Time { // valid pending timer via considerTarget (keeping the earliest), then finalizes CurrentTime and // DisabledAfterFastForward. The workflow (event-based) path uses the same struct. type TimeSkippingTransition struct { - // CurrentTime is the virtual "now" at which the transition is recorded (virtual frame, not wall - // clock). The skip delta added to the accumulated skipped duration is TargetTime - CurrentTime. CurrentTime time.Time TargetTime time.Time DisabledAfterFastForward bool } -// considerTarget folds a candidate skip target into the transition, keeping the earliest non-zero one. -// The framework calls it to combine the fast-forward target and each valid pending timer into a single -// TargetTime; zero candidates are ignored. -func (t *TimeSkippingTransition) considerTarget(candidate time.Time) { - if candidate.IsZero() { - return - } - if t.TargetTime.IsZero() || candidate.Before(t.TargetTime) { - t.TargetTime = candidate - } -} - -// IsValid reports whether the transition represents something to apply: a non-zero target to skip -// to, or a disable-after-fast-forward signal. It is nil-safe — a nil transition is not valid. +// IsValid reports whether the transition represents something to record and apply. func (t *TimeSkippingTransition) IsValid() bool { - if t == nil { - return false - } - return !t.TargetTime.IsZero() || t.DisabledAfterFastForward + return t != nil && (!t.TargetTime.IsZero() || t.DisabledAfterFastForward) } func IsActiveFastForward(ff *persistencespb.FastForwardInfo) bool { diff --git a/chasm/tree.go b/chasm/tree.go index ed13c92d1b2..fb276791269 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2184,37 +2184,8 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } - now := n.Now(nil) - var ffTarget time.Time - if ff := tsi.GetFastForwardInfo(); IsActiveFastForward(ff) { - ffTarget = ff.GetTargetTime().AsTime() - } - - transition := &TimeSkippingTransition{CurrentTime: now} - transition.considerTarget(ffTarget) - - // The component chooses where to skip to. Only the component knows its task semantics — in - // particular which pending timer is a legitimate wake (e.g. a retry backoff / next dispatch) versus - // an execution/run timeout that virtual time must never be advanced to. The framework cannot make - // that distinction (it only knows a task's category, not its meaning), so there is no framework-side - // default scan — FindNextTargetTime is the sole source of the wake target. - target := &TimeSkippingTargetTime{} - tsRoot.FindNextTargetTime(immutableContext, target) - transition.considerTarget(target.GetTime()) - - // Finalize. The accumulated TargetTime is min(fast-forward target, component's wake target). - switch { - case transition.TargetTime.IsZero(): - // Nothing to skip to and no fast-forward budget. - case !transition.TargetTime.After(now): - // The earliest candidate is the already-reached fast-forward target: disable only, no skip. - transition.TargetTime = time.Time{} - transition.DisabledAfterFastForward = true - case !ffTarget.IsZero() && !transition.TargetTime.Before(ffTarget): - // Skipping to (or past) the fast-forward target: skip and then disable. - transition.DisabledAfterFastForward = true - default: - // A regular skip to the component's wake target, still short of the fast-forward budget. + if err := n.calculateTimeSkippingTransition(immutableContext, transition); err != nil { + return err } // step3: if a time skipping transition is needed, record it and regenerate timer tasks. @@ -2222,13 +2193,90 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { return err } - if err := n.regenerateTasksForTimeSkipping(); err != nil { + if err := n.regenerateTasksForTimeSkipping(immutableContext); err != nil { return err } } return nil } +// validateSerializedTask deserializes a serialized component task on this node and runs its validator +// against current component state, returning true only if the task is still valid. It mirrors the +// per-task validation in closeTransactionCleanupInvalidTasks and is used by the time-skipping paths +// (findNextTargetTime, regenerateTasksForTimeSkipping) so that an invalidated task — e.g. a timeout +// whose activity already executed — is neither chosen as a skip target nor re-stamped. The component +// value is hydrated first (prepareComponentValue is idempotent). +func (n *Node) validateSerializedTask( + ctx Context, + task *persistencespb.ChasmComponentAttributes_Task, +) (bool, error) { + if err := n.prepareComponentValue(ctx); err != nil { + return false, err + } + taskInstance, err := n.deserializeComponentTask(task) + if err != nil { + return false, err + } + return n.validateTask(ctx, TaskAttributes{ + ScheduledTime: task.GetScheduledTime().AsTime(), + Destination: task.GetDestination(), + }, taskInstance) +} + +// calculateTimeSkippingTransition is the framework's default "where to skip to": it scans every scheduled +// (CategoryTimer) task across the tree — pure AND side-effect — and folds the earliest VALID one +// strictly after now into the transition. It is used only when the root component's FindNextTargetTime +// returns useDefault (see TimeSkippable). It mirrors regenerateTasksForTimeSkipping, which iterates the +// same tasks to re-stamp them. +// +// Each candidate is validated before being considered, so an invalidated task is never chosen as a skip +// target. Validation does NOT exclude timeout timers, though: a timeout is valid while the execution is +// idle. The default scan is therefore only safe for components whose pending timers are all genuine wake +// points — see the caveat on TimeSkippable.FindNextTargetTime. +func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransition, error) { + tsi := n.backend.GetTimeSkippingInfo() + tsc := tsi.GetConfig() + if tsc == nil || !tsc.GetEnabled() { + return nil, nil + } + transition := &TimeSkippingTransition{CurrentTime: n.Now(nil)} + ff := tsi.GetFastForwardInfo() + if ff != nil && !ff.GetHasReached() { + transition.considerTarget(ff.GetTargetTime().AsTime()) + } + + now := n.Now(nil) + for _, node := range n.andAllChildren() { + componentAttr := node.serializedNode.GetMetadata().GetComponentAttributes() + if componentAttr == nil { + continue + } + for _, taskList := range [][]*persistencespb.ChasmComponentAttributes_Task{ + componentAttr.GetPureTasks(), + componentAttr.GetSideEffectTasks(), + } { + for _, task := range taskList { + if taskCategory(task) != tasks.CategoryTimer { + continue + } + scheduledTime := task.GetScheduledTime().AsTime() + if !scheduledTime.After(now) { + continue + } + valid, err := node.validateSerializedTask(ctx, task) + if err != nil { + return err + } + if !valid { + continue + } + transition.considerTarget(scheduledTime) + } + } + } + return nil +} + // regenerateTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition // so their wall-clock visibility reflects the new accumulated skip. // @@ -2248,7 +2296,10 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { // Physical tasks generated for the same logical tasks earlier this transaction (pre-skip, with the old // accumulated skip) become stale; they fire early and no-op when their executor re-checks state. This // matches the workflow RegenerateTimerTasksForTimeSkipping model (re-stamp, tolerate stale duplicates). -func (n *Node) regenerateTasksForTimeSkipping() error { +// +// Invalidated tasks are excluded: each task is validated (validateSerializedTask) before being reset or +// re-stamped, so a task whose validator now rejects it is left alone rather than re-emitted. +func (n *Node) regenerateTasksForTimeSkipping(ctx Context) error { archetypeID := n.ArchetypeID() var firstPureTask *persistencespb.ChasmComponentAttributes_Task @@ -2260,27 +2311,40 @@ func (n *Node) regenerateTasksForTimeSkipping() error { continue } - // Side-effect timer tasks: reset and re-generate each one. Unlike the normal generation loop - // in closeTransactionUpdateComponentTasks, there is no early break on physicalTaskStatusCreated - // — we re-stamp every CategoryTimer task regardless of prior status. + // Side-effect timer tasks: reset and re-generate each valid one. Unlike the normal generation + // loop in closeTransactionUpdateComponentTasks, there is no early break on physicalTaskStatusCreated + // — we re-stamp every valid CategoryTimer task regardless of prior status. for _, sideEffectTask := range componentAttr.GetSideEffectTasks() { if taskCategory(sideEffectTask) != tasks.CategoryTimer { continue } + valid, err := node.validateSerializedTask(ctx, sideEffectTask) + if err != nil { + return err + } + if !valid { + continue + } sideEffectTask.PhysicalTaskStatus = physicalTaskStatusNone node.closeTransactionGeneratePhysicalSideEffectTask(sideEffectTask, nodePath, archetypeID) } - // Pure tasks are always future-scheduled (CategoryTimer). Reset them all, and track the earliest - // across the tree; closeTransactionGeneratePhysicalPureTask emits the single earliest one. - pureTasks := componentAttr.GetPureTasks() - for _, pureTask := range pureTasks { + // Pure tasks are always future-scheduled (CategoryTimer). Reset every valid one, and track the + // earliest valid task across the tree; closeTransactionGeneratePhysicalPureTask emits the single + // earliest one. + for _, pureTask := range componentAttr.GetPureTasks() { + valid, err := node.validateSerializedTask(ctx, pureTask) + if err != nil { + return err + } + if !valid { + continue + } pureTask.PhysicalTaskStatus = physicalTaskStatusNone - } - if len(pureTasks) > 0 && - (firstPureTask == nil || comparePureTasks(pureTasks[0], firstPureTask) < 0) { - firstPureTask = pureTasks[0] - firstPureTaskNode = node + if firstPureTask == nil || comparePureTasks(pureTask, firstPureTask) < 0 { + firstPureTask = pureTask + firstPureTaskNode = node + } } } From 2ff12cd7d706b9026ea405f371d7ca1f06697038 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Wed, 24 Jun 2026 15:00:31 -0700 Subject: [PATCH 06/13] simplify time skipping protocol in chasm --- chasm/context.go | 4 +- chasm/context_mock.go | 12 ++--- chasm/lib/activity/activity.go | 19 ++------ chasm/timeskippping.go | 63 +++++++++++++++++------- chasm/tree.go | 87 +++++++++++++--------------------- 5 files changed, 89 insertions(+), 96 deletions(-) diff --git a/chasm/context.go b/chasm/context.go index bc200b38344..756766258cf 100644 --- a/chasm/context.go +++ b/chasm/context.go @@ -98,9 +98,7 @@ type MutableContext interface { SetUserMetadata(Component, *sdkpb.UserMetadata) error // TimeSkippingController is the framework-provided surface for managing this execution's - // time-skipping configuration. A root component opts in by calling SetTimeSkippingConfig when the - // execution is created (and again to adjust it). It takes no Component argument because the - // configuration is execution-scoped, like the ExecutionKey()/ExecutionInfo() accessors. + // time-skipping configuration. It takes no Component argument as the configuration is execution-scoped. TimeSkippingController // Get a Ref for the component diff --git a/chasm/context_mock.go b/chasm/context_mock.go index 9a6a0219d04..61d231edfe3 100644 --- a/chasm/context_mock.go +++ b/chasm/context_mock.go @@ -183,19 +183,17 @@ func (c *MockContext) withValue(key any, value any) Context { type MockMutableContext struct { MockContext - mu sync.Mutex - Tasks []MockTask - LinksByRequest map[Component]map[string][]*commonpb.Link - UserMetadataByComponent map[Component]*sdkpb.UserMetadata - TimeSkippingConfig *commonpb.TimeSkippingConfig - TimeSkippingConfigInited bool + mu sync.Mutex + Tasks []MockTask + LinksByRequest map[Component]map[string][]*commonpb.Link + UserMetadataByComponent map[Component]*sdkpb.UserMetadata + TimeSkippingConfig *commonpb.TimeSkippingConfig } func (c *MockMutableContext) SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) { c.mu.Lock() defer c.mu.Unlock() c.TimeSkippingConfig = config - c.TimeSkippingConfigInited = true } func (c *MockMutableContext) AddTask(component Component, attributes TaskAttributes, payload any) { diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index b0043ae51fe..2e288702fd2 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -55,11 +55,6 @@ var ( var _ chasm.VisibilitySearchAttributesProvider = (*Activity)(nil) var _ callback.CompletionSource = (*Activity)(nil) - -// The activity opts into time skipping: it gates idleness via TimeSkippable and chooses its own skip -// target via the optional TimeSkippingTargetFinder (it must, because its pending timers include timeout -// tasks the framework's default scan must not target). These compile-time assertions keep the optional -// interface from silently desyncing if a method signature drifts. var _ chasm.TimeSkippable = (*Activity)(nil) type ActivityStore interface { @@ -130,15 +125,11 @@ func (a *Activity) LifecycleState(_ chasm.Context) chasm.LifecycleState { } } -// HasInflightWork implements chasm.TimeSkippable. It reports whether advancing virtual time would -// skip past real work, gating the framework's per-transaction time-skipping decision. -// -// Only the activity attempt is considered: it is in flight unless the activity is SCHEDULED and -// still waiting for its next dispatch — out of a start delay or retry backoff with the next-attempt -// time strictly in the (virtual) future. Once that time arrives (dispatch pushed to matching, -// waiting for a poller), the attempt is running (STARTED) or being cancelled (CANCEL_REQUESTED), or -// the activity has closed, this returns true. This mirrors the workflow time-skipping rule for -// activities, and means a closed activity is never time-skipped. +// HasInflightWork implements chasm.TimeSkippable. +// It returns true if +// - the activity is not in a scheduled state +// - the activity is scheduled but the next dispatch is not in the future +// Otherwise, it returns false. func (a *Activity) HasInflightWork(ctx chasm.Context) bool { if a.GetStatus() != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { return true diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 24d689e035f..8c4fc391f58 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -15,41 +15,70 @@ import ( // - TimeSkippable is the mandatory component-implemented interface that must be implemented by the root // component of an execution to make time skipping work. +// TimeSkippingController is the framework-provided surface (embedded in MutableContext) through which a +// root component opts into and adjusts time skipping for the whole execution. type TimeSkippingController interface { - // SetTimeSkippingConfig sets the execution's time-skipping config: the first call establishes it, // later calls update it in place (preserving the accumulated skipped duration). SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) } +// TimeSkippable is the mandatory component-implemented half of the time-skipping contract. The root +// component of an execution that opts into time skipping implements it so the framework can decide, once +// per transaction, whether the execution is idle enough to fast-forward virtual time. ROOT COMPONENT +// ONLY — the framework consults only the root component's implementation. type TimeSkippable interface { - - // HasInflightWork returns true when an execution has in-flight work (e.g. a standalone activity - // whose attempt has been dispatched to a worker or is currently running). Time skipping won't skip - // any time when it returns true so that in-flight work has enough time to complete. + // HasInflightWork returns true when the execution has in-flight work (e.g. a standalone activity + // whose attempt has been dispatched to a worker or is currently running). The framework skips no + // time while it returns true, so in-flight work runs at real speed. HasInflightWork(ctx Context) bool } -// TimeSkippingTransition is a time-skipping decision: the virtual time the transition is recorded at -// (CurrentTime), the virtual time to advance to (TargetTime), and whether reaching it should disable -// time skipping because the fast-forward budget was reached (DisabledAfterFastForward). The applied -// skip delta is TargetTime - CurrentTime. It is an internal struct shared between the CHASM framework -// and the mutable-state backend — NOT part of the component-facing contract. -// -// The framework builds it directly: it seeds TargetTime with the fast-forward target and folds in each -// valid pending timer via considerTarget (keeping the earliest), then finalizes CurrentTime and -// DisabledAfterFastForward. The workflow (event-based) path uses the same struct. +// TimeSkippingTransition is a time-skipping decision and is shared between the CHASM framework and +// the history substrate. type TimeSkippingTransition struct { CurrentTime time.Time TargetTime time.Time DisabledAfterFastForward bool } -// IsValid reports whether the transition represents something to record and apply. +// NewTimeSkippingTransition creates a new time-skipping transition with the given current time. +// A valid transition always has a non-zero CurrentTime; the fold methods below no-op without it. +func NewTimeSkippingTransition(currentTime time.Time) *TimeSkippingTransition { + return &TimeSkippingTransition{CurrentTime: currentTime} +} + +// IsValid reports whether the transition is worth applying: a real skip target, or a bare disable +// signal. Nil-safe. func (t *TimeSkippingTransition) IsValid() bool { return t != nil && (!t.TargetTime.IsZero() || t.DisabledAfterFastForward) } -func IsActiveFastForward(ff *persistencespb.FastForwardInfo) bool { - return ff != nil && !ff.HasReached && ff.TargetTime != nil && !ff.TargetTime.AsTime().IsZero() +// CompareAndSetTargetTime folds a business wake candidate (virtual time) in, keeping the earliest one at +// or after CurrentTime. Zero or past candidates (and a nil/uninitialized transition) are ignored. It +// never touches DisabledAfterFastForward — only the fast-forward budget disables time skipping. +func (t *TimeSkippingTransition) CompareAndSetTargetTime(candidate time.Time) { + if t == nil || t.CurrentTime.IsZero() || candidate.IsZero() || candidate.Before(t.CurrentTime) { + return + } + if t.TargetTime.IsZero() || candidate.Before(t.TargetTime) { + t.TargetTime = candidate + } +} + +// GateByFastForward folds the fast-forward target (the skip budget) in and decides whether +// reaching it disables time skipping. It MUST be called AFTER all business candidates have been folded, +// because the budget is a ceiling. +func (t *TimeSkippingTransition) GateByFastForward(ff *persistencespb.FastForwardInfo) { + if ff.GetHasReached() || ff.GetTargetTime().AsTime().IsZero() { + return + } + if t == nil || t.CurrentTime.IsZero() { + return + } + ffTargetTime := ff.GetTargetTime().AsTime() + t.CompareAndSetTargetTime(ffTargetTime) + if !ffTargetTime.After(t.CurrentTime) { + t.DisabledAfterFastForward = true + } } diff --git a/chasm/tree.go b/chasm/tree.go index fb276791269..f921712c2cf 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2120,6 +2120,7 @@ func (n *Node) closeTransactionUpdateComponentTasks( } func (n *Node) closeTransactionHandleTimeSkipping() error { + // step1: time skipping only works on the root node // todo@time-skipping: it seems CloseTransaction is only a method reasonable for the root node // but currently the framework doesn't enforce this contract @@ -2127,7 +2128,7 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } if n.ArchetypeID() == WorkflowArchetypeID { - // todo: removed when workflows are migrated to use chasm + // todo: remove after workflows get migrated to CHASM return nil } tsi := n.backend.GetTimeSkippingInfo() @@ -2172,30 +2173,31 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { } tsRoot, ok := rootComponent.(TimeSkippable) if !ok { + // todo: add a metric for alert in real implementation n.logger.Error( "root component does not implement TimeSkippable when executions have turned on time skipping", tag.Error(fmt.Errorf("type: %s", reflect.TypeOf(rootComponent).Name()))) return nil } - // step2: while the root reports the execution idle, build the transition: seed TargetTime with the - // fast-forward target, fold in the component's chosen wake target, then finalize. + // step2: calculate the time-skipping transition. if tsRoot.HasInflightWork(immutableContext) { return nil } - - if err := n.calculateTimeSkippingTransition(immutableContext, transition); err != nil { + transition, err := n.calculateTimeSkippingTransition(immutableContext) + if err != nil { return err } + if !transition.IsValid() { + return nil + } - // step3: if a time skipping transition is needed, record it and regenerate timer tasks. - if transition.IsValid() { - if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { - return err - } - if err := n.regenerateTasksForTimeSkipping(immutableContext); err != nil { - return err - } + // step3: record and regenerate tasks. + if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { + return err + } + if err := n.regenerateTasksForTimeSkipping(immutableContext); err != nil { + return err } return nil } @@ -2203,8 +2205,8 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { // validateSerializedTask deserializes a serialized component task on this node and runs its validator // against current component state, returning true only if the task is still valid. It mirrors the // per-task validation in closeTransactionCleanupInvalidTasks and is used by the time-skipping paths -// (findNextTargetTime, regenerateTasksForTimeSkipping) so that an invalidated task — e.g. a timeout -// whose activity already executed — is neither chosen as a skip target nor re-stamped. The component +// (calculateTimeSkippingTransition, regenerateTasksForTimeSkipping) so that an invalidated task — e.g. a +// timeout whose activity already executed — is neither chosen as a skip target nor re-stamped. The component // value is hydrated first (prepareComponentValue is idempotent). func (n *Node) validateSerializedTask( ctx Context, @@ -2224,28 +2226,18 @@ func (n *Node) validateSerializedTask( } // calculateTimeSkippingTransition is the framework's default "where to skip to": it scans every scheduled -// (CategoryTimer) task across the tree — pure AND side-effect — and folds the earliest VALID one -// strictly after now into the transition. It is used only when the root component's FindNextTargetTime -// returns useDefault (see TimeSkippable). It mirrors regenerateTasksForTimeSkipping, which iterates the -// same tasks to re-stamp them. -// -// Each candidate is validated before being considered, so an invalidated task is never chosen as a skip -// target. Validation does NOT exclude timeout timers, though: a timeout is valid while the execution is -// idle. The default scan is therefore only safe for components whose pending timers are all genuine wake -// points — see the caveat on TimeSkippable.FindNextTargetTime. +// (CategoryTimer) task across the tree — pure AND side-effect — folds the earliest VALID one strictly +// after now in as the business wake target, then folds in the fast-forward budget. It returns nil when +// time skipping is disabled. func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransition, error) { tsi := n.backend.GetTimeSkippingInfo() - tsc := tsi.GetConfig() - if tsc == nil || !tsc.GetEnabled() { + if !tsi.GetConfig().GetEnabled() { return nil, nil } - transition := &TimeSkippingTransition{CurrentTime: n.Now(nil)} - ff := tsi.GetFastForwardInfo() - if ff != nil && !ff.GetHasReached() { - transition.considerTarget(ff.GetTargetTime().AsTime()) - } - now := n.Now(nil) + transition := NewTimeSkippingTransition(now) + + // Business wake targets: the earliest valid future-scheduled (CategoryTimer) task across the tree. for _, node := range n.andAllChildren() { componentAttr := node.serializedNode.GetMetadata().GetComponentAttributes() if componentAttr == nil { @@ -2261,20 +2253,23 @@ func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransi } scheduledTime := task.GetScheduledTime().AsTime() if !scheduledTime.After(now) { + // A past/stale task that the component failed to invalidate. + // Ignore it rather than let it pin the target. continue } valid, err := node.validateSerializedTask(ctx, task) if err != nil { - return err + return nil, err } if !valid { continue } - transition.considerTarget(scheduledTime) + transition.CompareAndSetTargetTime(scheduledTime) } } } - return nil + transition.GateByFastForward(tsi.GetFastForwardInfo()) + return transition, nil } // regenerateTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition @@ -2282,20 +2277,8 @@ func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransi // // "Regenerate all" by default: every CategoryTimer task (pure and side-effect) across the tree is // reset to physicalTaskStatusNone and re-generated. BOTH pure and side-effect tasks can be -// future-scheduled (CategoryTimer) tasks that need re-stamping, for example: -// - a side-effect activity retry timer task — the ActivityDispatchTask that fires to dispatch the -// next attempt after a retry backoff (or initial start delay); and -// - a pure start-workflow task — a pure timer that fires to schedule the workflow task after a -// start delay/backoff. -// -// Immediate Transfer/Outbound/Visibility tasks are left untouched — they fire at wall-clock now and -// are unaffected by accumulated skip. This default is deliberately broad so the process stays correct -// even as the set of skippable/in-flight tasks evolves; narrowing to only the tasks that actually need -// re-stamping is a future optimization. -// -// Physical tasks generated for the same logical tasks earlier this transaction (pre-skip, with the old -// accumulated skip) become stale; they fire early and no-op when their executor re-checks state. This -// matches the workflow RegenerateTimerTasksForTimeSkipping model (re-stamp, tolerate stale duplicates). +// future-scheduled (CategoryTimer) tasks that need re-stamping.Immediate Transfer/Outbound/Visibility +// tasks are left untouched. // // Invalidated tasks are excluded: each task is validated (validateSerializedTask) before being reset or // re-stamped, so a task whose validator now rejects it is left alone rather than re-emitted. @@ -2311,9 +2294,6 @@ func (n *Node) regenerateTasksForTimeSkipping(ctx Context) error { continue } - // Side-effect timer tasks: reset and re-generate each valid one. Unlike the normal generation - // loop in closeTransactionUpdateComponentTasks, there is no early break on physicalTaskStatusCreated - // — we re-stamp every valid CategoryTimer task regardless of prior status. for _, sideEffectTask := range componentAttr.GetSideEffectTasks() { if taskCategory(sideEffectTask) != tasks.CategoryTimer { continue @@ -2329,9 +2309,6 @@ func (n *Node) regenerateTasksForTimeSkipping(ctx Context) error { node.closeTransactionGeneratePhysicalSideEffectTask(sideEffectTask, nodePath, archetypeID) } - // Pure tasks are always future-scheduled (CategoryTimer). Reset every valid one, and track the - // earliest valid task across the tree; closeTransactionGeneratePhysicalPureTask emits the single - // earliest one. for _, pureTask := range componentAttr.GetPureTasks() { valid, err := node.validateSerializedTask(ctx, pureTask) if err != nil { From 615d975422947578d80b7ce6c270f32e749cc7ef Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Wed, 24 Jun 2026 17:32:22 -0700 Subject: [PATCH 07/13] rm GetTimeSkippingInfo --- chasm/node_backend_mock.go | 8 -------- chasm/timeskipping_test.go | 5 +++-- chasm/tree.go | 22 +++------------------- service/history/workflow/timeskipping.go | 7 ------- 4 files changed, 6 insertions(+), 36 deletions(-) diff --git a/chasm/node_backend_mock.go b/chasm/node_backend_mock.go index 5d4e1587936..7c87e944f2e 100644 --- a/chasm/node_backend_mock.go +++ b/chasm/node_backend_mock.go @@ -41,7 +41,6 @@ type MockNodeBackend struct { HandleGetNamespaceEntry func() *namespace.Namespace HandleEndpointRegistry func() EndpointRegistry HandleSetTimeSkippingConfig func(config *commonpb.TimeSkippingConfig) - HandleGetTimeSkippingInfo func() *persistencespb.TimeSkippingInfo HandleRecordTimeSkippingTransition func(ctx context.Context, transition TimeSkippingTransition, archetype ArchetypeID) error // Recorded calls (protected by mu). @@ -121,13 +120,6 @@ func (m *MockNodeBackend) SetTimeSkippingConfig(config *commonpb.TimeSkippingCon } } -func (m *MockNodeBackend) GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo { - if m.HandleGetTimeSkippingInfo != nil { - return m.HandleGetTimeSkippingInfo() - } - return nil -} - func (m *MockNodeBackend) RecordTimeSkippingTransition(ctx context.Context, transition TimeSkippingTransition, archetype ArchetypeID) error { if m.HandleRecordTimeSkippingTransition != nil { return m.HandleRecordTimeSkippingTransition(ctx, transition, archetype) diff --git a/chasm/timeskipping_test.go b/chasm/timeskipping_test.go index 03f031009fb..4f82d5fbb0d 100644 --- a/chasm/timeskipping_test.go +++ b/chasm/timeskipping_test.go @@ -106,8 +106,9 @@ func (s *nodeSuite) TestCloseTransaction_DoesNotRegenerateAlreadyMaterializedTim root, err := s.newTestTree(s.timeSkippingTaskNodes()) s.NoError(err) - // Time skipping is not enabled (the mock backend's GetTimeSkippingInfo returns nil) and nothing was - // mutated, so the normal generation loop must skip the already-Created timer tasks. + // Time skipping is not enabled (the mock backend's GetExecutionInfo returns nil, so TimeSkippingInfo + // is nil) and nothing was mutated, so the normal generation loop must skip the already-Created timer + // tasks. _, err = root.CloseTransaction() s.NoError(err) s.Equal(0, s.nodeBackend.NumTasksAdded()) diff --git a/chasm/tree.go b/chasm/tree.go index f921712c2cf..7a2d4a062ed 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -234,26 +234,10 @@ type ( requestID string, ) (nexusrpc.CompleteOperationOptions, error) EndpointRegistry() EndpointRegistry - - // Time-skipping backend surface. The component-facing config control (SetTimeSkippingConfig) is - // exposed to executions through MutableContext, which delegates the mutator here; the framework - // itself reads the full TimeSkippingInfo (config AND fast-forward info) via GetTimeSkippingInfo. SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) - GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo - // RecordTimeSkippingTransition records a single time-skipping transition for the execution: it - // advances the virtual clock to transition.TargetTime (and disables time skipping when the - // fast-forward target was reached). The caller (the framework's closeTransactionHandleTimeSkipping, - // or the workflow event path) is responsible for computing the final transition — including the - // min of the component candidate and the fast-forward target. archetype selects how the skip is - // recorded: the workflow archetype records a history event, others apply it directly to the TSI. RecordTimeSkippingTransition(ctx context.Context, transition TimeSkippingTransition, archetype ArchetypeID) error } - // nowProvider is an optional capability of a NodeBackend: a backend that owns its own clock - // (mutable state, whose clock reflects the execution's time-skipping offset) implements it so - // that Node.Now() reads a single, time-skipping-aware clock shared with the rest of the - // execution. Backends that do not implement it (e.g. bare test mocks) cause Node.Now() to fall - // back to the tree's own timeSource. nowProvider interface { Now() time.Time } @@ -2131,7 +2115,7 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { // todo: remove after workflows get migrated to CHASM return nil } - tsi := n.backend.GetTimeSkippingInfo() + tsi := n.backend.GetExecutionInfo().GetTimeSkippingInfo() if !tsi.GetConfig().GetEnabled() { return nil } @@ -2192,7 +2176,7 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } - // step3: record and regenerate tasks. + // step3: record and regenerate if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { return err } @@ -2230,7 +2214,7 @@ func (n *Node) validateSerializedTask( // after now in as the business wake target, then folds in the fast-forward budget. It returns nil when // time skipping is disabled. func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransition, error) { - tsi := n.backend.GetTimeSkippingInfo() + tsi := n.backend.GetExecutionInfo().GetTimeSkippingInfo() if !tsi.GetConfig().GetEnabled() { return nil, nil } diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index 9ef80394fc8..ec71fa16a4b 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -557,13 +557,6 @@ func (ms *MutableStateImpl) SetTimeSkippingConfig(config *commonpb.TimeSkippingC } } -// GetTimeSkippingInfo returns the execution's full TimeSkippingInfo (config + fast-forward info). It -// is the CHASM framework's window into time-skipping state: the framework reads the config and the -// fast-forward target/reached flag through here (NodeBackend), not through a component-facing getter. -func (ms *MutableStateImpl) GetTimeSkippingInfo() *persistencespb.TimeSkippingInfo { - return ms.GetExecutionInfo().GetTimeSkippingInfo() -} - // RecordTimeSkippingTransition records a single time-skipping transition for the execution. It is the // archetype-agnostic apply sink shared by both time-skipping front-ends: the CHASM framework's // closeTransactionHandleTimeSkipping (which builds the transition for non-workflow CHASM executions) From c92b6168343596453586c08333afed50f19543c5 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Thu, 25 Jun 2026 11:48:14 -0700 Subject: [PATCH 08/13] add comment of not including callbacks --- chasm/lib/activity/activity.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index 2e288702fd2..14b08cd87a7 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -130,6 +130,8 @@ func (a *Activity) LifecycleState(_ chasm.Context) chasm.LifecycleState { // - the activity is not in a scheduled state // - the activity is scheduled but the next dispatch is not in the future // Otherwise, it returns false. +// +// A terminal activity that is still delivering completion callbacks (including while backing off between retries) is treated as having in-flight work: the delivery is pending and resolves on the target's readiness, not this execution's virtual clock. func (a *Activity) HasInflightWork(ctx chasm.Context) bool { if a.GetStatus() != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { return true From 22f82e9f67753ca9c90406e46e9ba3d90f245aea Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Mon, 29 Jun 2026 22:29:31 -0700 Subject: [PATCH 09/13] renamed interface --- chasm/context.go | 4 ++-- chasm/lib/activity/activity.go | 22 ++++++++--------- chasm/timeskippping.go | 44 ++++++++++++++++++++-------------- chasm/tree.go | 6 ++--- 4 files changed, 41 insertions(+), 35 deletions(-) diff --git a/chasm/context.go b/chasm/context.go index 756766258cf..90f31f4d69a 100644 --- a/chasm/context.go +++ b/chasm/context.go @@ -97,9 +97,9 @@ type MutableContext interface { // SetUserMetadata replaces the user metadata attached to the given component. SetUserMetadata(Component, *sdkpb.UserMetadata) error - // TimeSkippingController is the framework-provided surface for managing this execution's + // TimeSkippingConfigurator is the framework-provided surface for managing this execution's // time-skipping configuration. It takes no Component argument as the configuration is execution-scoped. - TimeSkippingController + TimeSkippingConfigurator // Get a Ref for the component // This ref to the component state at the end of the transition diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index 14b08cd87a7..727db7d7f75 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -55,7 +55,7 @@ var ( var _ chasm.VisibilitySearchAttributesProvider = (*Activity)(nil) var _ callback.CompletionSource = (*Activity)(nil) -var _ chasm.TimeSkippable = (*Activity)(nil) +var _ chasm.TimeSkippingRuntime = (*Activity)(nil) type ActivityStore interface { // RecordCompleted applies the provided function to record activity completion @@ -125,22 +125,20 @@ func (a *Activity) LifecycleState(_ chasm.Context) chasm.LifecycleState { } } -// HasInflightWork implements chasm.TimeSkippable. -// It returns true if -// - the activity is not in a scheduled state -// - the activity is scheduled but the next dispatch is not in the future -// Otherwise, it returns false. -// -// A terminal activity that is still delivering completion callbacks (including while backing off between retries) is treated as having in-flight work: the delivery is pending and resolves on the target's readiness, not this execution's virtual clock. -func (a *Activity) HasInflightWork(ctx chasm.Context) bool { +// IsExecutionSkippable implements chasm.TimeSkippingRuntime. The activity is skippable only when it is +// SCHEDULED (any other status — running, or a terminal activity still delivering completion callbacks, +// including while backing off between retries — is real work whose delivery resolves on the target's +// readiness, not this execution's virtual clock) and its next attempt dispatch is in the (virtual) +// future (a dispatch at or before now means an attempt is being dispatched or running, i.e. in flight). +func (a *Activity) IsExecutionSkippable(ctx chasm.Context) bool { if a.GetStatus() != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { - return true + return false } nextDispatch := a.attemptScheduleTime(a.LastAttempt.Get(ctx)) if nextDispatch == nil || !ctx.Now(a).Before(nextDispatch.AsTime()) { - return true + return false } - return false + return true } func (a *Activity) ContextMetadata(_ chasm.Context) map[string]string { diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 8c4fc391f58..693d14f535f 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -7,31 +7,39 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" ) -// Time skipping is a two-sided contract between the CHASM framework and a root component: +// Time skipping is a two-sided contract between the CHASM framework and a root component, split along +// the configuration / runtime axis: // -// - TimeSkippingController is PROVIDED BY the framework and CALLED BY the -// component. It is embedded in MutableContext and allows the component to opt in / adjust the config via -// SetTimeSkippingConfig. -// - TimeSkippable is the mandatory component-implemented interface that must be implemented by the root -// component of an execution to make time skipping work. +// - TimeSkippingConfigurator is PROVIDED BY the framework and CALLED BY the component. It is embedded +// in MutableContext and lets the component opt in / adjust the config via SetTimeSkippingConfig. +// - TimeSkippingRuntime is the mandatory component-implemented half: the root component implements it +// so the framework can decide, at runtime, whether the execution may be skipped. -// TimeSkippingController is the framework-provided surface (embedded in MutableContext) through which a -// root component opts into and adjusts time skipping for the whole execution. -type TimeSkippingController interface { +// TimeSkippingConfigurator is the framework-provided, configuration-time surface (embedded in +// MutableContext) through which a root component opts into and adjusts time skipping for the whole +// execution. +type TimeSkippingConfigurator interface { // SetTimeSkippingConfig sets the execution's time-skipping config: the first call establishes it, // later calls update it in place (preserving the accumulated skipped duration). SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) } -// TimeSkippable is the mandatory component-implemented half of the time-skipping contract. The root -// component of an execution that opts into time skipping implements it so the framework can decide, once -// per transaction, whether the execution is idle enough to fast-forward virtual time. ROOT COMPONENT -// ONLY — the framework consults only the root component's implementation. -type TimeSkippable interface { - // HasInflightWork returns true when the execution has in-flight work (e.g. a standalone activity - // whose attempt has been dispatched to a worker or is currently running). The framework skips no - // time while it returns true, so in-flight work runs at real speed. - HasInflightWork(ctx Context) bool +// TimeSkippingRuntime is the mandatory component-implemented, runtime half of the time-skipping +// contract. The root component of an execution that opts into time skipping implements it so the +// framework can decide, once per transaction, whether the execution is idle enough to fast-forward +// virtual time. ROOT COMPONENT ONLY — the framework consults only the root component's implementation. +type TimeSkippingRuntime interface { + // IsExecutionSkippable reports whether the execution may have its virtual clock fast-forwarded right + // now. It returns true only when the execution is idle. The framework skips no time while it returns + // false, so in-flight work runs at real speed. + // + // A recommended implementation composes: + // 1. status check (optional): only states that can be idle are skip candidates; any active or + // terminal-but-still-working status returns false. + // 2. in-flight work check (mandatory): false whenever real work is dispatched or running (e.g. a + // scheduled attempt whose dispatch time is at or before now). + // 3. other execution-specific preconditions, as needed. + IsExecutionSkippable(ctx Context) bool } // TimeSkippingTransition is a time-skipping decision and is shared between the CHASM framework and diff --git a/chasm/tree.go b/chasm/tree.go index 7a2d4a062ed..ea9215628d9 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2155,17 +2155,17 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { if err != nil { return err } - tsRoot, ok := rootComponent.(TimeSkippable) + tsRoot, ok := rootComponent.(TimeSkippingRuntime) if !ok { // todo: add a metric for alert in real implementation n.logger.Error( - "root component does not implement TimeSkippable when executions have turned on time skipping", + "root component does not implement TimeSkippingRuntime when executions have turned on time skipping", tag.Error(fmt.Errorf("type: %s", reflect.TypeOf(rootComponent).Name()))) return nil } // step2: calculate the time-skipping transition. - if tsRoot.HasInflightWork(immutableContext) { + if !tsRoot.IsExecutionSkippable(immutableContext) { return nil } transition, err := n.calculateTimeSkippingTransition(immutableContext) From 9a5ce0a37de6eea1f047a31da0d01015b3f3296c Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Mon, 29 Jun 2026 22:59:26 -0700 Subject: [PATCH 10/13] add FindNextTargetTime default and override logic --- chasm/lib/activity/activity.go | 4 +-- chasm/timeskipping_test.go | 2 +- chasm/timeskippping.go | 36 +++++++++++++++------- chasm/tree.go | 55 +++++++++++++--------------------- 4 files changed, 49 insertions(+), 48 deletions(-) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index 727db7d7f75..d2cad8a6c8e 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -55,7 +55,7 @@ var ( var _ chasm.VisibilitySearchAttributesProvider = (*Activity)(nil) var _ callback.CompletionSource = (*Activity)(nil) -var _ chasm.TimeSkippingRuntime = (*Activity)(nil) +var _ chasm.TimeSkippingRuntimeGate = (*Activity)(nil) type ActivityStore interface { // RecordCompleted applies the provided function to record activity completion @@ -125,7 +125,7 @@ func (a *Activity) LifecycleState(_ chasm.Context) chasm.LifecycleState { } } -// IsExecutionSkippable implements chasm.TimeSkippingRuntime. The activity is skippable only when it is +// IsExecutionSkippable implements chasm.TimeSkippingRuntimeGate. The activity is skippable only when it is // SCHEDULED (any other status — running, or a terminal activity still delivering completion callbacks, // including while backing off between retries — is real work whose delivery resolves on the target's // readiness, not this execution's virtual clock) and its next attempt dispatch is in the (virtual) diff --git a/chasm/timeskipping_test.go b/chasm/timeskipping_test.go index 4f82d5fbb0d..e25ca3f820a 100644 --- a/chasm/timeskipping_test.go +++ b/chasm/timeskipping_test.go @@ -87,7 +87,7 @@ func (s *nodeSuite) TestRegenerateTasksForTimeSkipping_RestampsEachTimerExactlyO Validate(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(true, nil).AnyTimes() - err = root.regenerateTasksForTimeSkipping(NewContext(context.Background(), root)) + err = root.regenerateChasmTasksForTimeSkipping(NewContext(context.Background(), root)) s.NoError(err) // 2 side-effect CategoryTimer tasks (root + SubComponent1) + 1 pure timer (earliest) = 3, all diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 693d14f535f..5e5afc1521b 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -7,13 +7,18 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" ) -// Time skipping is a two-sided contract between the CHASM framework and a root component, split along -// the configuration / runtime axis: +// Time skipping is a contract between the CHASM framework and a root component, split along the +// configuration / runtime axis: // -// - TimeSkippingConfigurator is PROVIDED BY the framework and CALLED BY the component. It is embedded -// in MutableContext and lets the component opt in / adjust the config via SetTimeSkippingConfig. -// - TimeSkippingRuntime is the mandatory component-implemented half: the root component implements it -// so the framework can decide, at runtime, whether the execution may be skipped. +// - TimeSkippingConfigurator (PROVIDED BY the framework, CALLED BY the component) is the +// configuration-time surface. It is embedded in MutableContext and lets the component opt in / +// adjust the config via SetTimeSkippingConfig. +// - TimeSkippingRuntimeGate (component-implemented, MANDATORY) is the runtime surface the framework +// consults each transaction to decide whether the execution is idle enough to skip. +// - TimeSkippingRuntimeTargetProvider (component-implemented, OPTIONAL) lets a component nominate the +// next skip target itself, in place of the framework's default timer-task tree scan. +// +// The two runtime interfaces are ROOT COMPONENT ONLY — the framework consults only the root component. // TimeSkippingConfigurator is the framework-provided, configuration-time surface (embedded in // MutableContext) through which a root component opts into and adjusts time skipping for the whole @@ -24,11 +29,9 @@ type TimeSkippingConfigurator interface { SetTimeSkippingConfig(config *commonpb.TimeSkippingConfig) } -// TimeSkippingRuntime is the mandatory component-implemented, runtime half of the time-skipping -// contract. The root component of an execution that opts into time skipping implements it so the -// framework can decide, once per transaction, whether the execution is idle enough to fast-forward -// virtual time. ROOT COMPONENT ONLY — the framework consults only the root component's implementation. -type TimeSkippingRuntime interface { +// TimeSkippingRuntimeGate is the mandatory, runtime half of the time-skipping contract: the framework +// asks it, once per transaction, whether the execution may be skipped at all. +type TimeSkippingRuntimeGate interface { // IsExecutionSkippable reports whether the execution may have its virtual clock fast-forwarded right // now. It returns true only when the execution is idle. The framework skips no time while it returns // false, so in-flight work runs at real speed. @@ -42,6 +45,17 @@ type TimeSkippingRuntime interface { IsExecutionSkippable(ctx Context) bool } +// TimeSkippingRuntimeTargetProvider is the optional, by default the framework scans the tree's +// all pure and side-effect tasks that are future-scheduled and takes the earliest one as the skip target. +// If the execution needs a customized strategy, the root component can implement this interface +// and it will override the default strategy. +type TimeSkippingRuntimeTargetProvider interface { + // FindNextTargetTime provides a customized strategy to find the next skip target. + // The framework will provided a initilalized transition, and the implementation should just call + // transition.CompareAndSetTargetTime to set the target time, and it is allowed to leave the transition unchanged. + FindNextTargetTime(ctx Context, transition *TimeSkippingTransition) +} + // TimeSkippingTransition is a time-skipping decision and is shared between the CHASM framework and // the history substrate. type TimeSkippingTransition struct { diff --git a/chasm/tree.go b/chasm/tree.go index ea9215628d9..ba307623a68 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2155,32 +2155,40 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { if err != nil { return err } - tsRoot, ok := rootComponent.(TimeSkippingRuntime) + tsRoot, ok := rootComponent.(TimeSkippingRuntimeGate) if !ok { // todo: add a metric for alert in real implementation n.logger.Error( - "root component does not implement TimeSkippingRuntime when executions have turned on time skipping", + "root component does not implement TimeSkippingRuntimeGate when executions have turned on time skipping", tag.Error(fmt.Errorf("type: %s", reflect.TypeOf(rootComponent).Name()))) return nil } - // step2: calculate the time-skipping transition. + // step2: bail unless the execution is idle. if !tsRoot.IsExecutionSkippable(immutableContext) { return nil } - transition, err := n.calculateTimeSkippingTransition(immutableContext) - if err != nil { - return err + + // step3: find the next skip target. A root component MAY implement TimeSkippingRuntimeTargetProvider + // to nominate the target itself; otherwise the framework scans the tree's timer tasks. Either way the + // framework owns the transition: it constructs it here, then applies the fast-forward budget and the + // validity gate below. + transition := NewTimeSkippingTransition(n.Now(nil)) + if provider, ok := rootComponent.(TimeSkippingRuntimeTargetProvider); ok { + provider.FindNextTargetTime(immutableContext, transition) + } else { + n.defaultFindNextTargetTime(transition) } + transition.GateByFastForward(tsi.GetFastForwardInfo()) if !transition.IsValid() { return nil } - // step3: record and regenerate + // step4: record and regenerate if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { return err } - if err := n.regenerateTasksForTimeSkipping(immutableContext); err != nil { + if err := n.regenerateChasmTasksForTimeSkipping(immutableContext); err != nil { return err } return nil @@ -2209,19 +2217,8 @@ func (n *Node) validateSerializedTask( }, taskInstance) } -// calculateTimeSkippingTransition is the framework's default "where to skip to": it scans every scheduled -// (CategoryTimer) task across the tree — pure AND side-effect — folds the earliest VALID one strictly -// after now in as the business wake target, then folds in the fast-forward budget. It returns nil when -// time skipping is disabled. -func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransition, error) { - tsi := n.backend.GetExecutionInfo().GetTimeSkippingInfo() - if !tsi.GetConfig().GetEnabled() { - return nil, nil - } - now := n.Now(nil) - transition := NewTimeSkippingTransition(now) - - // Business wake targets: the earliest valid future-scheduled (CategoryTimer) task across the tree. +func (n *Node) defaultFindNextTargetTime(transition *TimeSkippingTransition) error { + now := transition.CurrentTime for _, node := range n.andAllChildren() { componentAttr := node.serializedNode.GetMetadata().GetComponentAttributes() if componentAttr == nil { @@ -2237,26 +2234,16 @@ func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransi } scheduledTime := task.GetScheduledTime().AsTime() if !scheduledTime.After(now) { - // A past/stale task that the component failed to invalidate. - // Ignore it rather than let it pin the target. - continue - } - valid, err := node.validateSerializedTask(ctx, task) - if err != nil { - return nil, err - } - if !valid { continue } transition.CompareAndSetTargetTime(scheduledTime) } } } - transition.GateByFastForward(tsi.GetFastForwardInfo()) - return transition, nil + return nil } -// regenerateTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition +// regenerateChasmTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition // so their wall-clock visibility reflects the new accumulated skip. // // "Regenerate all" by default: every CategoryTimer task (pure and side-effect) across the tree is @@ -2266,7 +2253,7 @@ func (n *Node) calculateTimeSkippingTransition(ctx Context) (*TimeSkippingTransi // // Invalidated tasks are excluded: each task is validated (validateSerializedTask) before being reset or // re-stamped, so a task whose validator now rejects it is left alone rather than re-emitted. -func (n *Node) regenerateTasksForTimeSkipping(ctx Context) error { +func (n *Node) regenerateChasmTasksForTimeSkipping(ctx Context) error { archetypeID := n.ArchetypeID() var firstPureTask *persistencespb.ChasmComponentAttributes_Task From 317441c15e7204b399698ffd41a7b68a7c269c84 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Mon, 29 Jun 2026 23:13:10 -0700 Subject: [PATCH 11/13] marked a bad design of this solution' --- chasm/tree.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chasm/tree.go b/chasm/tree.go index ba307623a68..762e41a97d7 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2185,6 +2185,9 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { } // step4: record and regenerate + // todo: a bad design of current implementation is that RecordTimeSkippingTransition called regenerateChasmFastForwardWakeTask + // for chasm-based executions, and this is a special treatment. And we also need this special treatment for passive cluster. + // The reason is current chasm MutableContext.AddTask requires a component. if err := n.backend.RecordTimeSkippingTransition(context.Background(), *transition, n.ArchetypeID()); err != nil { return err } From 96749809c98b1066fc5782ece46e9dd7b2083d72 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Mon, 29 Jun 2026 23:26:18 -0700 Subject: [PATCH 12/13] add activity ts feature flag --- chasm/lib/activity/config.go | 8 ++- chasm/lib/activity/frontend.go | 6 +++ chasm/timeskippping.go | 29 ++++++---- chasm/tree.go | 68 +++--------------------- service/history/workflow/timeskipping.go | 15 +++--- tests/activity_standalone_test.go | 4 +- 6 files changed, 49 insertions(+), 81 deletions(-) diff --git a/chasm/lib/activity/config.go b/chasm/lib/activity/config.go index ce01b5a69be..7b7d9fc5e30 100644 --- a/chasm/lib/activity/config.go +++ b/chasm/lib/activity/config.go @@ -39,6 +39,12 @@ var ( false, `Allows attaching completion callbacks to standalone activity executions.`, ) + + TimeSkippingEnabled = dynamicconfig.NewNamespaceBoolSetting( + "activity.timeSkippingEnabled", + false, + `Allows setting time_skipping_config on standalone activity executions. This is a standalone-activity-specific feature flag, separate from the workflow frontend.TimeSkippingEnabled flag.`, + ) ) type Config struct { @@ -69,7 +75,7 @@ func ConfigProvider(dc *dynamicconfig.Collection) *Config { LongPollTimeout: LongPollTimeout.Get(dc), MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc), StartDelayEnabled: StartDelayEnabled.Get(dc), - TimeSkippingEnabled: dynamicconfig.TimeSkippingEnabled.Get(dc), + TimeSkippingEnabled: TimeSkippingEnabled.Get(dc), MaxCallbacksPerExecution: callback.MaxPerExecution.Get(dc), VisibilityMaxPageSize: dynamicconfig.FrontendVisibilityMaxPageSize.Get(dc), } diff --git a/chasm/lib/activity/frontend.go b/chasm/lib/activity/frontend.go index 749e4b08033..c11f7ea395f 100644 --- a/chasm/lib/activity/frontend.go +++ b/chasm/lib/activity/frontend.go @@ -92,6 +92,12 @@ func (h *frontendHandler) StartActivityExecution(ctx context.Context, req *workf if !h.config.Enabled(req.GetNamespace()) { return nil, ErrStandaloneActivityDisabled } + if req.GetTimeSkippingConfig() != nil && !h.config.TimeSkippingEnabled(req.GetNamespace()) { + return nil, serviceerror.NewUnimplementedf( + "The Time-Skipping feature is not enabled for namespace %s", + req.GetNamespace(), + ) + } namespaceID, err := h.namespaceRegistry.GetNamespaceID(namespace.Name(req.GetNamespace())) if err != nil { diff --git a/chasm/timeskippping.go b/chasm/timeskippping.go index 5e5afc1521b..0ef9a5badbb 100644 --- a/chasm/timeskippping.go +++ b/chasm/timeskippping.go @@ -52,7 +52,7 @@ type TimeSkippingRuntimeGate interface { type TimeSkippingRuntimeTargetProvider interface { // FindNextTargetTime provides a customized strategy to find the next skip target. // The framework will provided a initilalized transition, and the implementation should just call - // transition.CompareAndSetTargetTime to set the target time, and it is allowed to leave the transition unchanged. + // transition.TrackEarliestFutureTime to set the target time, and it is allowed to leave the transition unchanged. FindNextTargetTime(ctx Context, transition *TimeSkippingTransition) } @@ -71,15 +71,16 @@ func NewTimeSkippingTransition(currentTime time.Time) *TimeSkippingTransition { } // IsValid reports whether the transition is worth applying: a real skip target, or a bare disable -// signal. Nil-safe. +// signal. Nil-safe. A transition without a current time is never valid — every meaningful field is +// derived relative to the current time, so without it there is nothing to apply. func (t *TimeSkippingTransition) IsValid() bool { - return t != nil && (!t.TargetTime.IsZero() || t.DisabledAfterFastForward) + return t != nil && !t.CurrentTime.IsZero() && (!t.TargetTime.IsZero() || t.DisabledAfterFastForward) } -// CompareAndSetTargetTime folds a business wake candidate (virtual time) in, keeping the earliest one at +// TrackEarliestFutureTime folds a business wake candidate (virtual time) in, keeping the earliest one at // or after CurrentTime. Zero or past candidates (and a nil/uninitialized transition) are ignored. It // never touches DisabledAfterFastForward — only the fast-forward budget disables time skipping. -func (t *TimeSkippingTransition) CompareAndSetTargetTime(candidate time.Time) { +func (t *TimeSkippingTransition) TrackEarliestFutureTime(candidate time.Time) { if t == nil || t.CurrentTime.IsZero() || candidate.IsZero() || candidate.Before(t.CurrentTime) { return } @@ -92,15 +93,23 @@ func (t *TimeSkippingTransition) CompareAndSetTargetTime(candidate time.Time) { // reaching it disables time skipping. It MUST be called AFTER all business candidates have been folded, // because the budget is a ceiling. func (t *TimeSkippingTransition) GateByFastForward(ff *persistencespb.FastForwardInfo) { - if ff.GetHasReached() || ff.GetTargetTime().AsTime().IsZero() { + if t == nil || t.CurrentTime.IsZero() { return } - if t == nil || t.CurrentTime.IsZero() { + if ff == nil || ff.GetHasReached() || ff.GetTargetTime() == nil || + ff.GetTargetTime().AsTime().IsZero() { return } ffTargetTime := ff.GetTargetTime().AsTime() - t.CompareAndSetTargetTime(ffTargetTime) - if !ffTargetTime.After(t.CurrentTime) { - t.DisabledAfterFastForward = true + // If a real candidate is scheduled strictly before the fast-forward target, we skip to + // that and the fast-forward budget is not yet exhausted — leave time skipping enabled. + if !t.TargetTime.IsZero() && t.TargetTime.Before(ffTargetTime) { + return } + // Otherwise the fast-forward target is the earliest target: skip to it (clamped to the + // present by TrackEarliestFutureTime) and disable time skipping — the budget is reached. + // This is what lets the budget cap a chain of runs: a run with no earlier candidate + // consumes the remaining budget by skipping to the fast-forward and disabling. + t.TrackEarliestFutureTime(ffTargetTime) + t.DisabledAfterFastForward = true } diff --git a/chasm/tree.go b/chasm/tree.go index 762e41a97d7..919053ee978 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -1730,6 +1730,8 @@ func (n *Node) CloseTransaction() (NodesMutation, error) { return NodesMutation{}, err } + // time skippign should be called at the end of the transaction, + // so that all invalid tasks are deleted. if err := n.closeTransactionHandleTimeSkipping(); err != nil { return NodesMutation{}, err } @@ -2174,11 +2176,11 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { // framework owns the transition: it constructs it here, then applies the fast-forward budget and the // validity gate below. transition := NewTimeSkippingTransition(n.Now(nil)) - if provider, ok := rootComponent.(TimeSkippingRuntimeTargetProvider); ok { - provider.FindNextTargetTime(immutableContext, transition) - } else { - n.defaultFindNextTargetTime(transition) - } + // if provider, ok := rootComponent.(TimeSkippingRuntimeTargetProvider); ok { + // provider.FindNextTargetTime(immutableContext, transition) + // } else { + n.defaultFindNextTargetTime(transition) + // } transition.GateByFastForward(tsi.GetFastForwardInfo()) if !transition.IsValid() { return nil @@ -2197,29 +2199,6 @@ func (n *Node) closeTransactionHandleTimeSkipping() error { return nil } -// validateSerializedTask deserializes a serialized component task on this node and runs its validator -// against current component state, returning true only if the task is still valid. It mirrors the -// per-task validation in closeTransactionCleanupInvalidTasks and is used by the time-skipping paths -// (calculateTimeSkippingTransition, regenerateTasksForTimeSkipping) so that an invalidated task — e.g. a -// timeout whose activity already executed — is neither chosen as a skip target nor re-stamped. The component -// value is hydrated first (prepareComponentValue is idempotent). -func (n *Node) validateSerializedTask( - ctx Context, - task *persistencespb.ChasmComponentAttributes_Task, -) (bool, error) { - if err := n.prepareComponentValue(ctx); err != nil { - return false, err - } - taskInstance, err := n.deserializeComponentTask(task) - if err != nil { - return false, err - } - return n.validateTask(ctx, TaskAttributes{ - ScheduledTime: task.GetScheduledTime().AsTime(), - Destination: task.GetDestination(), - }, taskInstance) -} - func (n *Node) defaultFindNextTargetTime(transition *TimeSkippingTransition) error { now := transition.CurrentTime for _, node := range n.andAllChildren() { @@ -2239,23 +2218,13 @@ func (n *Node) defaultFindNextTargetTime(transition *TimeSkippingTransition) err if !scheduledTime.After(now) { continue } - transition.CompareAndSetTargetTime(scheduledTime) + transition.TrackEarliestFutureTime(scheduledTime) } } } return nil } -// regenerateChasmTasksForTimeSkipping re-emits physical timer tasks after a time-skipping transition -// so their wall-clock visibility reflects the new accumulated skip. -// -// "Regenerate all" by default: every CategoryTimer task (pure and side-effect) across the tree is -// reset to physicalTaskStatusNone and re-generated. BOTH pure and side-effect tasks can be -// future-scheduled (CategoryTimer) tasks that need re-stamping.Immediate Transfer/Outbound/Visibility -// tasks are left untouched. -// -// Invalidated tasks are excluded: each task is validated (validateSerializedTask) before being reset or -// re-stamped, so a task whose validator now rejects it is left alone rather than re-emitted. func (n *Node) regenerateChasmTasksForTimeSkipping(ctx Context) error { archetypeID := n.ArchetypeID() @@ -2272,25 +2241,11 @@ func (n *Node) regenerateChasmTasksForTimeSkipping(ctx Context) error { if taskCategory(sideEffectTask) != tasks.CategoryTimer { continue } - valid, err := node.validateSerializedTask(ctx, sideEffectTask) - if err != nil { - return err - } - if !valid { - continue - } sideEffectTask.PhysicalTaskStatus = physicalTaskStatusNone node.closeTransactionGeneratePhysicalSideEffectTask(sideEffectTask, nodePath, archetypeID) } for _, pureTask := range componentAttr.GetPureTasks() { - valid, err := node.validateSerializedTask(ctx, pureTask) - if err != nil { - return err - } - if !valid { - continue - } pureTask.PhysicalTaskStatus = physicalTaskStatusNone if firstPureTask == nil || comparePureTasks(pureTask, firstPureTask) < 0 { firstPureTask = pureTask @@ -2918,13 +2873,6 @@ func (n *Node) applyUpdates( return nil } -// RefreshTasks resets the physical task status of every task in the tree so they are re-generated on -// the next CloseTransaction. It needs no time-skipping-specific logic: regeneration re-emits each task -// through MutableState.AddTasks, which applies ToRealTime to scheduled (CategoryTimer) tasks — i.e. it -// subtracts the execution's current accumulatedSkippedDuration (already in mutable state). So a refresh -// naturally re-stamps timer tasks at the correct wall-clock fire time for time skipping, just as the -// scoped closeTransactionRegenerateTimerTasks does after a skip; both rely on the same AddTasks -> -// ToRealTime conversion. func (n *Node) RefreshTasks() error { for _, node := range n.andAllChildren() { // Only reset task status here, the actual task generation will be done when diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index ec71fa16a4b..6e44884187e 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -302,7 +302,7 @@ func (ms *MutableStateImpl) isWorkflowSkippable() bool { } // A pending activity blocks time skipping unless it has failed and is still // waiting out its retry backoff (next attempt strictly in the future) — that one - // is a skip target, not in-flight work (see calculateTimeSkippingTransition). The + // is a skip target, not in-flight work (see findNextSkipTarget). The // strict future check is what keeps a just-scheduled or already-due activity (next // attempt <= now) blocking. for _, ai := range ms.GetPendingActivityInfos() { @@ -486,10 +486,9 @@ func (ms *MutableStateImpl) closeTransactionRegenTimerTasksForWorkflowTimeSkippi // applyTimeSkippingTransition mutates the execution's TimeSkippingInfo for a single skip transition: // it advances AccumulatedSkippedDuration by (TargetTime - CurrentTime), toggles Config.Enabled, and -// marks the fast-forward target reached. It is the archetype-agnostic core shared by the workflow -// event path (ApplyWorkflowExecutionTimeSkippingTransitionedEvent) and the CHASM close-transaction -// path (RecordTimeSkippingTransition); the workflow path records the transition as a history event while -// CHASM (which has no history) applies it directly, but the TSI mutation is identical. +// marks the fast-forward target reached. This is the CHASM (non-workflow) apply path, invoked by +// RecordTimeSkippingTransition; the workflow path applies the equivalent mutation from its history +// event in ApplyWorkflowExecutionTimeSkippingTransitionedEvent. // // A zero TargetTime means "no skip" (only valid together with DisabledAfterFastForward, e.g. a bound // hit exactly). transition.CurrentTime must be in the virtual frame (ms.Now() / the event's virtual @@ -558,9 +557,9 @@ func (ms *MutableStateImpl) SetTimeSkippingConfig(config *commonpb.TimeSkippingC } // RecordTimeSkippingTransition records a single time-skipping transition for the execution. It is the -// archetype-agnostic apply sink shared by both time-skipping front-ends: the CHASM framework's -// closeTransactionHandleTimeSkipping (which builds the transition for non-workflow CHASM executions) -// and the workflow event path (closeTransactionHandleTimeSkipping -> shouldExecuteTimeSkipping). The +// archetype-aware apply sink: the CHASM framework's closeTransactionHandleTimeSkipping calls it for +// non-workflow executions after building the transition, and the fast-forward wake timer executor +// calls it (for either archetype) to disable time skipping once the fast-forward target is hit. The // caller is responsible for computing the final transition — including the min of the component // candidate and the fast-forward target; this method only records it. // diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index b2c0a164b99..a8dfcac772d 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -146,8 +146,8 @@ func (s *standaloneActivityTestSuite) TestTimeSkipping_StartDelayAndRetryBackoff env := s.newTestEnv() t := s.T() - // Standalone activities only opt into time skipping when the namespace flag is on. - env.GetTestCluster().OverrideDynamicConfig(t, dynamicconfig.TimeSkippingEnabled, []dynamicconfig.ConstrainedValue{ + // Standalone activities only opt into time skipping when the SAA-specific namespace flag is on. + env.GetTestCluster().OverrideDynamicConfig(t, activity.TimeSkippingEnabled, []dynamicconfig.ConstrainedValue{ {Constraints: dynamicconfig.Constraints{Namespace: env.Namespace().String()}, Value: true}, }) From 6e5972ae5a3636bb0136a487586abe94ff863009 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Tue, 30 Jun 2026 23:34:22 -0700 Subject: [PATCH 13/13] fix bug of workflow ts impacting chasm --- service/history/timer_queue_active_task_executor.go | 2 +- service/history/workflow/timeskipping.go | 8 ++++++++ tests/activity_standalone_test.go | 8 ++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/service/history/timer_queue_active_task_executor.go b/service/history/timer_queue_active_task_executor.go index 82c44f964db..b96355e756c 100644 --- a/service/history/timer_queue_active_task_executor.go +++ b/service/history/timer_queue_active_task_executor.go @@ -940,7 +940,7 @@ func (t *timerQueueActiveTaskExecutor) executeTimeSkippingTimerTask( // (and other archetypes) apply the disable-only transition directly to the TimeSkippingInfo. if err = mutableState.RecordTimeSkippingTransition( ctx, - chasm.TimeSkippingTransition{DisabledAfterFastForward: true}, + chasm.TimeSkippingTransition{CurrentTime: mutableState.Now(), DisabledAfterFastForward: true}, mutableState.ChasmTree().ArchetypeID(), ); err != nil { return err diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index 6e44884187e..c0d1e7e158d 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -394,6 +394,14 @@ func (ms *MutableStateImpl) closeTransactionHandleWorkflowTimeSkipping( ctx context.Context, transactionPolicy historyi.TransactionPolicy, ) (needRegenTasks bool) { + // This is the workflow (history-event) time-skipping path. CHASM executions (e.g. standalone + // activities) run their own decision in the CHASM tree's closeTransactionHandleTimeSkipping; + // running this path for them would scan only workflow-level targets (timers, pending activities, + // run timeout) — none of which exist for a CHASM component — and its GateByFastForward would then + // skip straight to the fast-forward and disable time skipping before the CHASM path ever runs. + if !ms.IsWorkflow() { + return false + } switch transactionPolicy { case historyi.TransactionPolicyActive: // 1. gate: only a running, time-skipping-enabled, idle workflow may skip time diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index a8dfcac772d..97693f100f5 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -173,6 +173,14 @@ func (s *standaloneActivityTestSuite) TestTimeSkipping_StartDelayAndRetryBackoff BackoffCoefficient: 1.0, MaximumAttempts: 2, }, + // The 1h FastForward budget never binds here (both skip targets — the 20m start delay and + // the 20m retry backoff — are well under 1h, so time skipping advances to them, not to the + // budget). It is kept deliberately as a regression guard: it is precisely what surfaces the + // bug where the workflow (history-event) time-skipping path runs for a CHASM execution and, + // finding no workflow-level target, skips straight to the fast-forward and disables time + // skipping before the CHASM path runs (fixed by the !ms.IsWorkflow() gate in + // closeTransactionHandleWorkflowTimeSkipping). Without a FastForward that path has nothing to + // skip to and the regression would not reproduce. TimeSkippingConfig: &commonpb.TimeSkippingConfig{ Enabled: true, FastForward: durationpb.New(time.Hour),