diff --git a/CHANGELOG.md b/CHANGELOG.md index cac94858..93efc2f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Forced coroutine unloading no longer leaks durable actions or custom status from application defers: durable operations panic with `ErrTaskBlocked` before changing state. Normal-return defers remain awaitable across replay, and orchestration loggers suppress forced-unload output. +- Long durable timers no longer schedule another chunk when a trailing timer event is processed after the orchestration has finalized. - DTS authentication now caches access tokens and immutable gRPC metadata, honors credential `RefreshOn` guidance, and coalesces concurrent refreshes and failures. Previously credentials such as `AzureCLICredential` were invoked for every RPC, serializing high-throughput workers behind external token acquisition. - Disabled large-payload handling no longer builds transform closures, maps, and goroutines for ordinary payloads. Reserved payload-reference prefixes are still rejected when no store and resolver are configured. - Worker completion and abandon RPCs no longer retry `NotFound` responses ten times. A completion whose work item is already unavailable is dropped for normal DTS redelivery without a second, guaranteed-failing abandon RPC. diff --git a/README.md b/README.md index c0207767..c2654c01 100644 --- a/README.md +++ b/README.md @@ -307,6 +307,63 @@ Obey these rules when you write a retry handler: Full sample: [samples/retries](./samples/retries). +### Deferred work and cooperative cleanup + +A root or child coroutine's defers can schedule and await durable work during +normal return, error return, or an application panic. A deferred `Await` can +span turns: replay resumes the cleanup before that coroutine completes. + +Forced unloading is different. At the end of a waiting turn, or when the +orchestration ends with waiting children, Go unwinds their stacks. Durable +operations in those defers panic with `task.ErrTaskBlocked` before changing +durable state. Never recover this signal. The rest of that deferred function +is skipped, but other defers still run. Termination does not run business cleanup. + +`ctx.Logger()` suppresses replay and forced-unload output without changing +`ctx.IsReplaying`. Other local defers, such as metrics or `span.End`, can run +every turn. Keep local-resource cleanup separate from durable cleanup. + +To finish child cleanup before completion, run the child on the uncanceled +root, cancel only its body operations, and join it. A coroutine canceled before +it starts is skipped, so do not rely on a `Done` deferred inside `child.Go`. + +```go +func CleanupOrchestrator(ctx *task.OrchestrationContext) (any, error) { + work, cancel := ctx.WithCancel() + group := ctx.NewWaitGroup() + var childErr error + group.Add(1) + ctx.Go(func(cleanup *task.OrchestrationContext) { + defer group.Done() + defer func() { + childErr = errors.Join(childErr, cleanup.CallActivity("Cleanup").Await(nil)) + }() + childErr = work.WaitForSingleEvent("work", -1).Await(nil) + if childErr == task.ErrTaskCanceled { + childErr = nil // The body cancellation is expected; cleanup errors are not. + } + }) + if err := ctx.WaitForSingleEvent("stop", -1).Await(nil); err != nil { + return nil, err + } + cancel() + group.Wait(ctx) + return nil, childErr +} +``` + +The `Cleanup` activity must act only on resources this workflow owns and must +be idempotent, since activity delivery is at least once. For multiple children, +keep separate error slots and combine them after joining. + +Unlike `Await`, `Select` and `WaitGroup.Wait` panic on cancellation; wrap those +body operations with a handler that recovers only the exact +`task.ErrTaskCanceled` value and re-panics everything else. + +Use a deferred function when scheduling itself should be deferred. +`defer ctx.CallActivity("Cleanup").Await(nil)` schedules the activity immediately +and defers only the await. + ### Orchestration management Use a `TaskRegistry` to register your orchestrator, activity, and entity functions. Then use the client from `durabletaskscheduler.NewClient` to control the orchestrations. diff --git a/task/context.go b/task/context.go index c472e498..bbb108e4 100644 --- a/task/context.go +++ b/task/context.go @@ -36,7 +36,8 @@ func (ctx *OrchestrationContext) Context() context.Context { }) } -// Logger returns a slog logger that suppresses output while replaying history. +// Logger returns a slog logger that suppresses output while replaying history +// and while waiting coroutines are forcibly unloaded. func (ctx *OrchestrationContext) Logger() *slog.Logger { engine := ctx.engineContext() logger := engine.logger @@ -46,7 +47,7 @@ func (ctx *OrchestrationContext) Logger() *slog.Logger { handler := &replaySafeHandler{ handler: logger.Handler(), replaying: func() bool { - return engine.IsReplaying + return engine.IsReplaying || engine.unwinding }, } return slog.New(handler).With( diff --git a/task/eventchannel.go b/task/eventchannel.go index 726052e4..93e599fe 100644 --- a/task/eventchannel.go +++ b/task/eventchannel.go @@ -28,7 +28,7 @@ func NewEventChannel[T any](ctx *OrchestrationContext, name string) *EventChanne if ctx == nil { panic("event channel requires an orchestration context") } - engine := ctx.engineContext() + engine := ctx.effectContext() key := strings.ToUpper(name) if existing, ok := engine.eventChannels[key]; ok { channel, ok := existing.(*EventChannel[T]) @@ -57,7 +57,7 @@ func (c *EventChannel[T]) Receive(ctx *OrchestrationContext) T { // ReceiveErr waits for and consumes the next event value, returning payload // decoding and cancellation errors to the orchestrator. func (c *EventChannel[T]) ReceiveErr(ctx *OrchestrationContext) (T, error) { - if ctx.engineContext() != c.ctx { + if ctx.effectContext() != c.ctx { panic("event channel used with a different orchestration context") } var value T diff --git a/task/orchestrator.go b/task/orchestrator.go index 2727fa2b..8078dc4c 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -33,6 +33,11 @@ type replayEvent struct { } // OrchestrationContext is the parameter type for orchestrator functions. +// +// Deferred functions can issue and await durable work when their coroutine +// returns normally. When the runtime forcibly unloads a waiting coroutine, +// durable operations instead panic with ErrTaskBlocked before changing state. +// Applications must not recover that control-flow signal. type OrchestrationContext struct { ID api.InstanceID Name string @@ -75,6 +80,7 @@ type OrchestrationContext struct { converter api.DataConverter entitiesSupported bool scheduler *coroutineScheduler + unwinding bool root *OrchestrationContext scope *cancellationScope derived []*OrchestrationContext @@ -123,6 +129,7 @@ type ContinueAsNewOption func(*OrchestrationContext) // runtime to carry forward any unprocessed external events to the new instance. func WithKeepUnprocessedEvents() ContinueAsNewOption { return func(ctx *OrchestrationContext) { + ctx.effectContext() ctx.saveBufferedExternalEvents = true } } @@ -130,6 +137,7 @@ func WithKeepUnprocessedEvents() ContinueAsNewOption { // WithContinueAsNewVersion migrates the next execution to a new orchestration version. func WithContinueAsNewVersion(version string) ContinueAsNewOption { return func(ctx *OrchestrationContext) { + ctx.effectContext() if version != "" && strings.TrimSpace(version) == "" { ctx.continuedAsNewVersion = nil return @@ -289,6 +297,15 @@ func (ctx *OrchestrationContext) engineContext() *OrchestrationContext { return ctx } +// effectContext stops forced-unwind defers before they can mutate durable state. +func (ctx *OrchestrationContext) effectContext() *OrchestrationContext { + engine := ctx.engineContext() + if engine.unwinding { + panic(ErrTaskBlocked) + } + return engine +} + func (ctx *OrchestrationContext) syncDerivedContexts() { active := ctx.derived[:0] for _, derived := range ctx.derived { @@ -310,8 +327,10 @@ func (ctx *OrchestrationContext) syncDerivedContexts() { // WithCancel creates a child orchestration context whose tasks, nested scopes, // and coroutines are canceled together at the next scheduler step. // A child of an already-canceled scope is canceled immediately. +// Await cleanup from an uncanceled wrapper, cancel only its body operations, +// and join the wrapper before returning. func (ctx *OrchestrationContext) WithCancel() (*OrchestrationContext, func()) { - engine := ctx.engineContext() + engine := ctx.effectContext() if engine.scheduler == nil { panic("cancellation scope created outside orchestrator execution") } @@ -583,12 +602,12 @@ func (ctx *OrchestrationContext) processEvent(e *protos.HistoryEvent) error { // SetCustomStatus stores a raw, pre-serialized custom status string. // Use SetCustomStatusValue to apply the configured data converter. func (octx *OrchestrationContext) SetCustomStatus(cs string) { - octx.engineContext().customStatus = cs + octx.effectContext().customStatus = cs } // SetCustomStatusValue serializes and stores a typed custom status value. func (octx *OrchestrationContext) SetCustomStatusValue(value any) error { - engine := octx.engineContext() + engine := octx.effectContext() payload, err := api.SerializeData(engine.converter, value) if err != nil { return fmt.Errorf("failed to serialize custom status: %w", err) @@ -599,14 +618,14 @@ func (octx *OrchestrationContext) SetCustomStatusValue(value any) error { // SetRawCustomStatus stores a pre-serialized custom status value. func (octx *OrchestrationContext) SetRawCustomStatus(payload string) { - octx.engineContext().customStatus = payload + octx.effectContext().customStatus = payload } var guidNamespace = uuid.MustParse("9e952958-5e33-4daf-827f-2fa12937b875") // NewGuid returns a deterministic UUID that is stable across orchestration replay. func (ctx *OrchestrationContext) NewGuid() string { - engine := ctx.engineContext() + engine := ctx.effectContext() timestamp := engine.CurrentTimeUtc.UTC().Format("2006-01-02T15:04:05.0000000Z") name := fmt.Sprintf("%s_%s_%d", engine.ID, timestamp, engine.newGuidCounter) engine.newGuidCounter++ @@ -623,7 +642,7 @@ func (octx *OrchestrationContext) GetInput(v any) error { // parameter can be either the name of an activity as a string or can be a pointer to the function // that implements the activity, in which case the name is obtained via reflection. func (ctx *OrchestrationContext) CallActivity(activity any, opts ...CallActivityOption) Task { - engine := ctx.engineContext() + engine := ctx.effectContext() options := new(callActivityOptions) for _, configure := range opts { if err := configure(options, engine.converter); err != nil { @@ -659,6 +678,7 @@ func (ctx *OrchestrationContext) newFailedTask(engine *OrchestrationContext, err // Go starts a coroutine that is cooperatively scheduled with the orchestration. // Only one orchestration coroutine runs at a time, in monotonically increasing ID order. // A callback whose scope is canceled before it starts is not invoked. +// Join children explicitly when their cleanup must finish before the root returns. func (ctx *OrchestrationContext) Go(fn func(ctx *OrchestrationContext)) { if fn == nil { panic("orchestration coroutine function must be non-nil") @@ -712,7 +732,7 @@ func (ctx *OrchestrationContext) internalScheduleActivity( } func (ctx *OrchestrationContext) CallSubOrchestrator(orchestrator any, opts ...SubOrchestratorOption) Task { - engine := ctx.engineContext() + engine := ctx.effectContext() if engine.criticalSectionID != "" { return ctx.newFailedTask(engine, fmt.Errorf("sub-orchestrations cannot be started while holding entity locks")) } @@ -902,7 +922,7 @@ func computeNextDelay(currentTimeUtc time.Time, policy RetryPolicy, attempt int, // CreateTimer schedules a durable timer that expires after the specified delay. func (ctx *OrchestrationContext) CreateTimer(delay time.Duration) Task { - engine := ctx.engineContext() + engine := ctx.effectContext() if ctx.scope.isCanceled() { return newTaskInScope(engine, ctx.scope) } @@ -926,7 +946,7 @@ func (ctx *OrchestrationContext) createTimerInternal( var scheduleNextChunk func() scheduleNextChunk = func() { - if logicalTimer.isCompleted { + if logicalTimer.isCompleted || (ctx.scheduler != nil && ctx.scheduler.isStopping()) { return } @@ -988,7 +1008,7 @@ func (ctx *OrchestrationContext) createTimerAction( // // Note that event names are case-insensitive. func (ctx *OrchestrationContext) WaitForSingleEvent(eventName string, timeout time.Duration) Task { - engine := ctx.engineContext() + engine := ctx.effectContext() task := newTaskInScope(engine, ctx.scope) if ctx.scope.isCanceled() { return task @@ -1023,7 +1043,7 @@ func (ctx *OrchestrationContext) WaitForSingleEvent(eventName string, timeout ti // CallEntity sends an operation request to an entity and waits for its response. func (ctx *OrchestrationContext) CallEntity(entityID api.EntityID, operationName string, opts ...callEntityOption) Task { - engine := ctx.engineContext() + engine := ctx.effectContext() if engine.isTerminated || ctx.scope.isCanceled() { task := newTaskInScope(engine, ctx.scope) task.cancel() @@ -1086,7 +1106,7 @@ func (ctx *OrchestrationContext) CallEntity(entityID api.EntityID, operationName // SignalEntity sends a fire-and-forget entity operation. func (ctx *OrchestrationContext) SignalEntity(entityID api.EntityID, operationName string, opts ...signalEntityOption) error { - engine := ctx.engineContext() + engine := ctx.effectContext() if engine.isTerminated || ctx.scope.isCanceled() { return ErrTaskCanceled } @@ -1128,7 +1148,7 @@ func (ctx *OrchestrationContext) SignalEntity(entityID api.EntityID, operationNa // If cancellation follows a committed request, critical-section restrictions // remain in effect until the eventual grant is received and automatically released. func (ctx *OrchestrationContext) LockEntities(entityIDs ...api.EntityID) (func(), error) { - engine := ctx.engineContext() + engine := ctx.effectContext() if engine.isTerminated || ctx.scope.isCanceled() { return nil, ErrTaskCanceled } @@ -1211,8 +1231,9 @@ func (ctx *OrchestrationContext) IsInCriticalSection() bool { return ctx.engineContext().criticalSectionID != "" } +// ContinueAsNew requests a new execution when the root coroutine finishes. func (ctx *OrchestrationContext) ContinueAsNew(newInput any, options ...ContinueAsNewOption) { - engine := ctx.engineContext() + engine := ctx.effectContext() engine.continuedAsNew = true engine.continuedAsNewInput = newInput for _, option := range options { @@ -1223,7 +1244,7 @@ func (ctx *OrchestrationContext) ContinueAsNew(newInput any, options ...Continue // SendEvent sends an event to another orchestration instance as part of the // current durable orchestration transaction. func (ctx *OrchestrationContext) SendEvent(instanceID api.InstanceID, eventName string, payload any) error { - engine := ctx.engineContext() + engine := ctx.effectContext() raw, err := marshalData(engine.converter, payload) if err != nil { return fmt.Errorf("failed to marshal event payload: %w", err) @@ -1426,6 +1447,7 @@ func (ctx *OrchestrationContext) peekBufferedEvent(key string) (*bufferedEvent, } func (ctx *OrchestrationContext) takeBufferedEvent(key string) (*bufferedEvent, bool) { + ctx.effectContext() eventList, ok := ctx.bufferedExternalEvents[key] if !ok || eventList.Len() == 0 { return nil, false @@ -1788,6 +1810,7 @@ func (ctx *OrchestrationContext) clearCriticalSection() { } func (ctx *OrchestrationContext) getNextSequenceNumber() int32 { + ctx.effectContext() current := ctx.sequenceNumber ctx.sequenceNumber++ return current diff --git a/task/scheduler.go b/task/scheduler.go index 4018dc70..29113840 100644 --- a/task/scheduler.go +++ b/task/scheduler.go @@ -182,6 +182,10 @@ func (s *coroutineScheduler) shutdown() { return } s.stopping = true + engine := s.ctx.engineContext() + // Stopping persists while trailing history is processed; unwinding must not. + engine.unwinding = true + defer func() { engine.unwinding = false }() for _, c := range s.all { c.exit() } diff --git a/task/select.go b/task/select.go index 06f9939f..118d7e9e 100644 --- a/task/select.go +++ b/task/select.go @@ -94,10 +94,10 @@ func (ctx *OrchestrationContext) Select(cases ...SelectCase) { } func (ctx *OrchestrationContext) selectCase(cases []SelectCase) SelectCase { + engine := ctx.effectContext() if len(cases) == 0 { panic("Select requires at least one case") } - engine := ctx.engineContext() scheduler := engine.scheduler if scheduler == nil { panic("Select called outside orchestrator execution") diff --git a/task/select_test.go b/task/select_test.go index aabc3257..14bcfc54 100644 --- a/task/select_test.go +++ b/task/select_test.go @@ -389,9 +389,10 @@ func executeOrchestrationTurn( instanceID api.InstanceID, oldEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent, + options ...TaskExecutorOption, ) *protos.OrchestratorResponse { t.Helper() - result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + result, err := NewTaskExecutor(registry, options...).ExecuteOrchestrator( context.Background(), instanceID, oldEvents, diff --git a/task/task.go b/task/task.go index 498fc6aa..57b3b0db 100644 --- a/task/task.go +++ b/task/task.go @@ -13,6 +13,8 @@ import ( // ErrTaskBlocked is not an error, but rather a control flow signal indicating that an orchestrator // function has executed as far as it can and that it now needs to unload, dispatch any scheduled tasks, // and commit its current execution progress to durable storage. +// Durable operations also panic with this signal when called by deferred code +// during forced coroutine unloading, before creating or modifying durable work. var ErrTaskBlocked = errors.New("the current task is blocked") // ErrTaskCanceled is used to indicate that a task was canceled. Tasks can be canceled, for example, @@ -45,6 +47,7 @@ type completableTask struct { } func newTaskInScope(ctx *OrchestrationContext, scope *cancellationScope) *completableTask { + ctx.effectContext() task := &completableTask{ orchestrationCtx: ctx, waiters: make(map[*coroutine]struct{}), @@ -62,11 +65,13 @@ func newTaskInScope(ctx *OrchestrationContext, scope *cancellationScope) *comple // // Await will return ErrTaskCanceled if the task was canceled - e.g. due to a timeout. // -// Await may panic with ErrTaskBlocked as the panic value if called on a task that has not yet completed. +// Await may panic with ErrTaskBlocked as the panic value if called on a task that has not yet completed, +// or if called while the runtime forcibly unloads the current coroutine. // This is normal control flow behavior for orchestrator functions and doesn't actually indicate a failure // of any kind. However, orchestrator functions must never attempt to recover from such panics to ensure that // the orchestration execution can procede normally. func (t *completableTask) Await(v any) error { + t.orchestrationCtx.effectContext() for { if t.isCompleted { if t.localErr != nil { @@ -116,7 +121,6 @@ func (t *completableTask) Await(v any) error { break } } - // TODO: Need a rule about using "defer" in orchestrations because planned panics will invoke them unexpectedly panic(ErrTaskBlocked) } diff --git a/task/teardown_test.go b/task/teardown_test.go new file mode 100644 index 00000000..ba46782e --- /dev/null +++ b/task/teardown_test.go @@ -0,0 +1,585 @@ +package task + +import ( + "bytes" + "errors" + "log/slog" + "maps" + "slices" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +var teardownClock = time.Date(2026, 9, 28, 0, 0, 0, 0, time.UTC) + +func teardownStarted() []*protos.HistoryEvent { + started := helpers.NewOrchestratorStartedEvent() + started.Timestamp = timestamppb.New(teardownClock) + execution := helpers.NewExecutionStartedEvent("teardown", "teardown-instance", nil, nil, nil, nil) + execution.Timestamp = started.Timestamp + execution.GetExecutionStarted().OrchestrationInstance.ExecutionId = wrapperspb.String("teardown-execution") + return []*protos.HistoryEvent{started, execution} +} + +func teardownRegistry(t *testing.T, fn Orchestrator) *TaskRegistry { + t.Helper() + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("teardown", fn)) + return registry +} + +func TestTeardownTerminalPaths(t *testing.T) { + for _, outcome := range []string{"completed", "continued", "terminated", "error", "panic", "child-panic"} { + t.Run(outcome, func(t *testing.T) { + deferred := 0 + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + ctx.SetCustomStatus("live") + ctx.Go(func(child *OrchestrationContext) { + defer child.SetCustomStatus("teardown") + defer func() { + deferred++ + _ = child.CallActivity("cleanup").Await(nil) + }() + _ = child.WaitForSingleEvent("child-signal", -1).Await(nil) + }) + if outcome == "child-panic" { + ctx.Go(func(child *OrchestrationContext) { + _ = child.WaitForSingleEvent("finish", -1).Await(nil) + panic("child failure") + }) + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + } + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + switch outcome { + case "continued": + ctx.ContinueAsNew("next", WithKeepUnprocessedEvents()) + case "error": + return nil, errors.New("root failure") + case "panic": + panic("root failure") + } + return "done", nil + }) + trigger := helpers.NewEventRaisedEvent("finish", nil) + expected := api.RUNTIME_STATUS_COMPLETED + switch outcome { + case "continued": + expected = api.RUNTIME_STATUS_CONTINUED_AS_NEW + case "terminated": + trigger = helpers.NewExecutionTerminatedEvent(nil, false) + expected = api.RUNTIME_STATUS_TERMINATED + case "error", "panic", "child-panic": + expected = api.RUNTIME_STATUS_FAILED + } + response := executeOrchestrationTurn(t, registry, "teardown-instance", nil, append(teardownStarted(), trigger)) + require.Equal(t, 1, deferred) + require.Len(t, response.Actions, 1, "teardown must not append actions: %v", response.Actions) + require.Equal(t, expected, completionAction(t, response).OrchestrationStatus) + require.Equal(t, "live", response.GetCustomStatus().GetValue()) + }) + } +} + +func TestTeardownUnloadPreservesStatusAndTurnPolicy(t *testing.T) { + for _, mode := range []string{"ordinary", "partial", "suspended", "history-continue", "history-failure"} { + t.Run(mode, func(t *testing.T) { + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + ctx.SetCustomStatus("waiting") + defer ctx.SetCustomStatus("finished") + return nil, ctx.WaitForSingleEvent("finish", -1).Await(nil) + }) + events := teardownStarted() + var options OrchestrationOptions + switch mode { + case "partial": + options.MaxEventsPerTurn = 1 + events = append(events, helpers.NewEventRaisedEvent("extra", nil)) + case "suspended": + events = append(events, helpers.NewSuspendOrchestrationEvent("pause")) + case "history-continue", "history-failure": + options.MaxHistoryEvents = 3 + events = append(events, events[0], events[0]) + if mode == "history-continue" { + options.OnHistoryLimitExceeded = func(HistoryLimitInfo) (any, error) { return "next", nil } + } + } + response := executeOrchestrationTurn(t, registry, "teardown-instance", nil, events, WithOrchestrationOptions(options)) + require.Equal(t, "waiting", response.GetCustomStatus().GetValue()) + switch mode { + case "history-continue": + require.Equal(t, api.RUNTIME_STATUS_CONTINUED_AS_NEW, completionAction(t, response).OrchestrationStatus) + case "history-failure": + require.Equal(t, api.RUNTIME_STATUS_FAILED, completionAction(t, response).OrchestrationStatus) + default: + require.Empty(t, response.Actions) + } + if mode == "partial" { + require.NotNil(t, response.NumEventsProcessed) + require.EqualValues(t, 1, response.NumEventsProcessed.Value) + } + }) + } +} + +type teardownHandles struct { + pending, complete Task + channel *EventChannel[int] + group, emptyGroup WaitGroup + value, handlers int +} + +type teardownConverter struct { + serializations, deserializations int +} + +func (c *teardownConverter) Serialize(value any) (string, error) { + c.serializations++ + return api.DefaultDataConverter().Serialize(value) +} + +func (c *teardownConverter) Deserialize(value string, target any) error { + c.deserializations++ + return api.DefaultDataConverter().Deserialize(value, target) +} + +type teardownState struct { + sequence int32 + guids uint64 + actions, tasks []int32 + entityTasks, channels []string + waits, eventWaiters map[string]int + buffered map[string][]string + continued, keep bool + input any + version, status string +} + +func teardownSnapshot(ctx *OrchestrationContext) teardownState { + state := teardownState{ + sequence: ctx.sequenceNumber, guids: ctx.newGuidCounter, + waits: make(map[string]int), eventWaiters: make(map[string]int), buffered: make(map[string][]string), + continued: ctx.continuedAsNew, keep: ctx.saveBufferedExternalEvents, input: ctx.continuedAsNewInput, + version: ctx.continuedAsNewVersion.GetValue(), status: ctx.customStatus, + } + state.actions = slices.Sorted(maps.Keys(ctx.pendingActions)) + state.tasks = slices.Sorted(maps.Keys(ctx.pendingTasks)) + state.entityTasks = slices.Sorted(maps.Keys(ctx.pendingEntityTasks)) + state.channels = slices.Sorted(maps.Keys(ctx.eventChannels)) + for name, list := range ctx.pendingExternalEventTasks { + state.waits[name] = list.Len() + } + for name, waiters := range ctx.eventWaiters { + state.eventWaiters[name] = len(waiters) + } + for name, list := range ctx.bufferedExternalEvents { + for item := list.Front(); item != nil; item = item.Next() { + state.buffered[name] = append(state.buffered[name], item.Value.(*bufferedEvent).event.GetEventRaised().GetInput().GetValue()) + } + } + return state +} + +func TestTeardownAPIsDoNotMutateState(t *testing.T) { + entity := api.NewEntityID("counter", "teardown") + cases := []struct { + name string + call func(*OrchestrationContext, *teardownHandles) + }{ + {"activity", func(ctx *OrchestrationContext, _ *teardownHandles) { + ctx.CallActivity("cleanup", WithActivityInput("payload")) + }}, + {"activity-retry", func(ctx *OrchestrationContext, _ *teardownHandles) { + ctx.CallActivity("cleanup", WithActivityRetryPolicy(&RetryPolicy{MaxAttempts: 2, InitialRetryInterval: time.Second})) + }}, + {"sub-orchestration", func(ctx *OrchestrationContext, _ *teardownHandles) { + ctx.CallSubOrchestrator("cleanup", WithSubOrchestratorInput("payload")) + }}, + {"timer", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.CreateTimer(time.Hour) }}, + {"event-wait", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.WaitForSingleEvent("new-wait", -1) }}, + {"event-timeout", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.WaitForSingleEvent("new-wait", time.Hour) }}, + {"event-ready", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.WaitForSingleEvent("buffered", 0) }}, + {"channel-new", func(ctx *OrchestrationContext, _ *teardownHandles) { NewEventChannel[int](ctx, "new-channel") }}, + {"channel-cached", func(ctx *OrchestrationContext, _ *teardownHandles) { NewEventChannel[int](ctx, "buffered") }}, + {"channel-try", func(_ *OrchestrationContext, h *teardownHandles) { h.channel.TryReceive() }}, + {"channel-receive", func(ctx *OrchestrationContext, h *teardownHandles) { _, _ = h.channel.ReceiveErr(ctx) }}, + {"task-pending", func(_ *OrchestrationContext, h *teardownHandles) { _ = h.pending.Await(nil) }}, + {"task-completed", func(_ *OrchestrationContext, h *teardownHandles) { _ = h.complete.Await(&h.value) }}, + {"select", func(ctx *OrchestrationContext, h *teardownHandles) { + ctx.Select(OnTask(h.complete, func(Task) { h.handlers++ })) + }}, + {"when-any", func(ctx *OrchestrationContext, h *teardownHandles) { ctx.WhenAny(h.complete) }}, + {"when-all", func(ctx *OrchestrationContext, h *teardownHandles) { _ = ctx.WhenAll(h.complete) }}, + {"group-wait", func(ctx *OrchestrationContext, h *teardownHandles) { h.group.Wait(ctx) }}, + {"group-ready", func(ctx *OrchestrationContext, h *teardownHandles) { h.emptyGroup.Wait(ctx) }}, + {"entity-call", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.CallEntity(entity, "cleanup") }}, + {"entity-signal", func(ctx *OrchestrationContext, _ *teardownHandles) { _ = ctx.SignalEntity(entity, "cleanup") }}, + {"entity-lock", func(ctx *OrchestrationContext, _ *teardownHandles) { _, _ = ctx.LockEntities(entity) }}, + {"continue", func(ctx *OrchestrationContext, _ *teardownHandles) { + ctx.ContinueAsNew("next", WithKeepUnprocessedEvents()) + }}, + {"keep-option", func(ctx *OrchestrationContext, _ *teardownHandles) { WithKeepUnprocessedEvents()(ctx) }}, + {"version-option", func(ctx *OrchestrationContext, _ *teardownHandles) { WithContinueAsNewVersion("next")(ctx) }}, + {"send-event", func(ctx *OrchestrationContext, _ *teardownHandles) { _ = ctx.SendEvent("target", "cleanup", "payload") }}, + {"status", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.SetCustomStatus("finished") }}, + {"raw-status", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.SetRawCustomStatus("finished") }}, + {"typed-status", func(ctx *OrchestrationContext, _ *teardownHandles) { _ = ctx.SetCustomStatusValue("finished") }}, + {"guid", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.NewGuid() }}, + {"scope", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.WithCancel() }}, + {"task-factory", func(ctx *OrchestrationContext, _ *teardownHandles) { newTaskInScope(ctx, ctx.scope) }}, + {"sequence", func(ctx *OrchestrationContext, _ *teardownHandles) { ctx.getNextSequenceNumber() }}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + var before, after teardownState + var recovered any + returned, observed := false, false + converter := new(teardownConverter) + handles := new(teardownHandles) + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + ctx.SetCustomStatus("live") + handles.channel = NewEventChannel[int](ctx, "buffered") + handles.pending = ctx.WaitForSingleEvent("pending", -1) + complete := newTaskInScope(ctx, ctx.scope) + complete.complete([]byte("42")) + handles.complete = complete + handles.group, handles.emptyGroup = ctx.NewWaitGroup(), ctx.NewWaitGroup() + handles.group.Add(1) + defer func() { + before = teardownSnapshot(ctx) + defer func() { + recovered = recover() + after = teardownSnapshot(ctx) + observed = true + if recovered != nil { + panic(recovered) + } + }() + test.call(ctx, handles) + returned = true + }() + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }) + events := append(teardownStarted(), helpers.NewEventRaisedEvent("buffered", wrapperspb.String("7"))) + response := executeOrchestrationTurn(t, registry, "teardown-instance", nil, events, WithDataConverter(converter)) + require.True(t, observed) + require.False(t, returned) + require.Equal(t, ErrTaskBlocked, recovered) + require.Equal(t, before, after) + require.Zero(t, handles.value) + require.Zero(t, handles.handlers) + require.Zero(t, converter.serializations) + require.Zero(t, converter.deserializations) + require.Empty(t, response.Actions) + require.Equal(t, "live", response.GetCustomStatus().GetValue()) + }) + } +} + +func TestTeardownAllowsLocalCoordination(t *testing.T) { + finished := false + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + group := ctx.NewWaitGroup() + group.Add(1) + defer func() { + group.Done() + group.Add(1) + group.Done() + finished = ctx.WhenAll() == nil + }() + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }) + response := executeOrchestrationTurn(t, registry, "teardown-instance", nil, teardownStarted()) + require.True(t, finished, "local coordination must not interrupt the rest of a defer") + require.Empty(t, response.Actions) +} + +func TestTeardownCannotConsumeContinueAsNewCarryover(t *testing.T) { + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[int](ctx, "carry") + ctx.Go(func(child *OrchestrationContext) { + defer func() { channel.TryReceive() }() + _ = child.WaitForSingleEvent("child-signal", -1).Await(nil) + }) + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + ctx.ContinueAsNew("next", WithKeepUnprocessedEvents()) + return nil, nil + }) + events := append(teardownStarted(), + helpers.NewEventRaisedEvent("carry", wrapperspb.String("7")), + helpers.NewEventRaisedEvent("finish", nil)) + response := executeOrchestrationTurn(t, registry, "teardown-instance", nil, events) + require.Len(t, response.Actions, 1) + completed := completionAction(t, response) + require.Equal(t, api.RUNTIME_STATUS_CONTINUED_AS_NEW, completed.OrchestrationStatus) + require.Len(t, completed.CarryoverEvents, 1) + require.Equal(t, "7", completed.CarryoverEvents[0].GetEventRaised().GetInput().GetValue()) +} + +func TestTeardownInsideDeferredAwaitPreservesNormalReplay(t *testing.T) { + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + defer func() { _ = ctx.CallActivity("cleanup-A").Await(nil) }() + defer func() { _ = ctx.CallActivity("cleanup-B").Await(nil) }() + return "done", nil + }) + start := teardownStarted() + first := executeOrchestrationTurn(t, registry, "teardown-instance", nil, start) + require.Len(t, first.Actions, 1) + require.Equal(t, "cleanup-B", first.Actions[0].GetScheduleTask().GetName()) + + old := append(slices.Clone(start), helpers.NewTaskScheduledEvent(0, "cleanup-B", nil, nil, nil)) + terminated := executeOrchestrationTurn(t, registry, "teardown-instance", old, []*protos.HistoryEvent{helpers.NewExecutionTerminatedEvent(nil, false)}) + require.Len(t, terminated.Actions, 1, "remaining defer A must not schedule on termination") + require.Equal(t, api.RUNTIME_STATUS_TERMINATED, completionAction(t, terminated).OrchestrationStatus) + + completedB := helpers.NewTaskCompletedEvent(0, nil) + second := executeOrchestrationTurn(t, registry, "teardown-instance", old, []*protos.HistoryEvent{completedB}) + require.Len(t, second.Actions, 1) + require.Equal(t, "cleanup-A", second.Actions[0].GetScheduleTask().GetName()) + old = append(old, completedB, helpers.NewTaskScheduledEvent(1, "cleanup-A", nil, nil, nil)) + third := executeOrchestrationTurn(t, registry, "teardown-instance", old, []*protos.HistoryEvent{helpers.NewTaskCompletedEvent(1, nil)}) + require.Len(t, third.Actions, 1) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completionAction(t, third).OrchestrationStatus) +} + +func TestTeardownNormalDefersRemainAwaitable(t *testing.T) { + for _, mode := range []string{"return", "error", "panic", "joined-child"} { + t.Run(mode, func(t *testing.T) { + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + if mode == "joined-child" { + group := ctx.NewWaitGroup() + group.Add(1) + ctx.Go(func(child *OrchestrationContext) { + defer group.Done() + defer func() { _ = child.CallActivity("cleanup").Await(nil) }() + }) + group.Wait(ctx) + return "done", nil + } + defer func() { _ = ctx.CallActivity("cleanup").Await(nil) }() + if mode == "error" { + return nil, errors.New("root failure") + } + if mode == "panic" { + panic("root failure") + } + return "done", nil + }) + start := teardownStarted() + first := executeOrchestrationTurn(t, registry, "teardown-instance", nil, start) + require.Len(t, first.Actions, 1) + require.Equal(t, "cleanup", first.Actions[0].GetScheduleTask().GetName()) + old := append(slices.Clone(start), helpers.NewTaskScheduledEvent(0, "cleanup", nil, nil, nil)) + second := executeOrchestrationTurn(t, registry, "teardown-instance", old, []*protos.HistoryEvent{helpers.NewTaskCompletedEvent(0, nil)}) + require.Len(t, second.Actions, 1) + expected := api.RUNTIME_STATUS_COMPLETED + if mode == "error" || mode == "panic" { + expected = api.RUNTIME_STATUS_FAILED + } + require.Equal(t, expected, completionAction(t, second).OrchestrationStatus) + }) + } +} + +func TestTeardownStopsLateTimerChunks(t *testing.T) { + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + ctx.CreateTimer(10 * 24 * time.Hour) + return "done", ctx.WaitForSingleEvent("finish", -1).Await(nil) + }) + first := executeOrchestrationTurn(t, registry, "teardown-instance", nil, teardownStarted()) + require.Len(t, first.Actions, 1) + fireAt := first.Actions[0].GetCreateTimer().FireAt + old := append(teardownStarted(), helpers.NewTimerCreatedEvent(0, fireAt)) + fired := helpers.NewTimerFiredEvent(0, fireAt, nil) + + active := executeOrchestrationTurn(t, registry, "teardown-instance", old, []*protos.HistoryEvent{fired}) + require.Len(t, active.Actions, 1) + require.NotNil(t, active.Actions[0].GetCreateTimer()) + + terminal := executeOrchestrationTurn(t, registry, "teardown-instance", old, []*protos.HistoryEvent{ + helpers.NewEventRaisedEvent("finish", nil), fired, + }) + require.Len(t, terminal.Actions, 1) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completionAction(t, terminal).OrchestrationStatus) +} + +func TestTeardownLoggerSuppressesUnwindNotLiveOrHostLogs(t *testing.T) { + var output bytes.Buffer + var root *OrchestrationContext + var replaying bool + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + root = ctx + child, cancel := ctx.WithCancel() + cached := child.Logger().With("scope", "child") + cancel() + ctx.Logger().Info("live-message") + defer func() { + replaying = ctx.IsReplaying + ctx.Logger().Info("unwind-message") + child.Logger().Info("unwind-child") + cached.Info("unwind-cached") + }() + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }) + executeOrchestrationTurn(t, registry, "teardown-instance", nil, append(teardownStarted(), helpers.NewEventRaisedEvent("other", nil)), + WithLogger(slog.New(slog.NewTextHandler(&output, nil))), + WithMetricsHooks(MetricsHooks{History: func(HistoryMetric) { root.Logger().Info("host-message") }})) + require.False(t, replaying, "forced unload must not change IsReplaying") + require.Contains(t, output.String(), "live-message") + require.Contains(t, output.String(), "host-message") + require.NotContains(t, output.String(), "unwind-") +} + +func TestTeardownGuardUsesRootOfCanceledContexts(t *testing.T) { + localDefers := 0 + var captured *OrchestrationContext + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + captured = ctx + child, cancel := ctx.WithCancel() + cancel() + defer func() { localDefers++ }() + defer func() { + child.SetCustomStatus("finished") + localDefers += 100 + }() + ctx.SetCustomStatus("live") + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }) + response := executeOrchestrationTurn(t, registry, "teardown-instance", nil, append(teardownStarted(), helpers.NewEventRaisedEvent("other", nil))) + require.Equal(t, 1, localDefers) + require.Empty(t, captured.derived, "canceled child should have been pruned") + require.Equal(t, "live", response.GetCustomStatus().GetValue()) +} + +func TestTeardownStillAllowsLaterEngineTermination(t *testing.T) { + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + ctx.Go(func(child *OrchestrationContext) { + defer child.SetCustomStatus("teardown") + _ = child.WaitForSingleEvent("never", -1).Await(nil) + }) + return "done", ctx.WaitForSingleEvent("finish", -1).Await(nil) + }) + response := executeOrchestrationTurn(t, registry, "teardown-instance", nil, append(teardownStarted(), + helpers.NewEventRaisedEvent("finish", nil), + helpers.NewExecutionTerminatedEvent(nil, false))) + require.Len(t, response.Actions, 1) + require.Equal(t, api.RUNTIME_STATUS_TERMINATED, completionAction(t, response).OrchestrationStatus) + require.Empty(t, response.GetCustomStatus().GetValue()) +} + +func teardownCancelableBody(body func() error) (err error) { + defer func() { + value := recover() + if value == ErrTaskCanceled { //nolint:errorlint // Do not swallow wrapped or joined application errors. + err = ErrTaskCanceled + } else if value != nil { + panic(value) + } + }() + return body() +} + +func TestTeardownExplicitCleanupJoinsBeforeCompletion(t *testing.T) { + for _, mode := range []string{"await", "select", "cancel-before-start", "cleanup-fails"} { + t.Run(mode, func(t *testing.T) { + registry := teardownRegistry(t, func(root *OrchestrationContext) (any, error) { + work, cancel := root.WithCancel() + group := root.NewWaitGroup() + var childErr error + group.Add(1) + root.Go(func(cleanup *OrchestrationContext) { + defer group.Done() + defer func() { + childErr = errors.Join(childErr, cleanup.CallActivity("cleanup").Await(nil)) + }() + childErr = teardownCancelableBody(func() error { + if mode == "select" { + work.Select(OnEvent(NewEventChannel[string](work, "work"), nil)) + return nil + } + return work.WaitForSingleEvent("work", -1).Await(nil) + }) + if childErr == ErrTaskCanceled { //nolint:errorlint // Only the body's bare cancellation signal is expected. + childErr = nil + } + }) + if mode != "cancel-before-start" { + if err := root.WaitForSingleEvent("stop", -1).Await(nil); err != nil { + return nil, err + } + } + cancel() + group.Wait(root) + return "joined", childErr + }) + history := teardownStarted() + if mode != "cancel-before-start" { + history = append(history, helpers.NewEventRaisedEvent("stop", nil)) + } + first := executeOrchestrationTurn(t, registry, "teardown-instance", nil, history) + require.Len(t, first.Actions, 1) + require.Equal(t, "cleanup", first.Actions[0].GetScheduleTask().GetName()) + history = append(history, helpers.NewTaskScheduledEvent(0, "cleanup", nil, nil, nil)) + cleanupResult := helpers.NewTaskCompletedEvent(0, nil) + expected := api.RUNTIME_STATUS_COMPLETED + if mode == "cleanup-fails" { + cleanupResult = helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ + ErrorType: "CleanupFailure", ErrorMessage: "cleanup failed", + }) + expected = api.RUNTIME_STATUS_FAILED + } + second := executeOrchestrationTurn(t, registry, "teardown-instance", history, []*protos.HistoryEvent{cleanupResult}) + require.Len(t, second.Actions, 1) + completed := completionAction(t, second) + require.Equal(t, expected, completed.OrchestrationStatus) + if mode == "cleanup-fails" { + require.Contains(t, completed.GetFailureDetails().GetErrorMessage(), "cleanup failed") + } + }) + } +} + +func TestTeardownFinalizationReleasesLocksExactlyOnce(t *testing.T) { + entities := []api.EntityID{api.NewEntityID("counter", "a"), api.NewEntityID("counter", "b")} + registry := teardownRegistry(t, func(ctx *OrchestrationContext) (any, error) { + ctx.Go(func(child *OrchestrationContext) { + unlock, err := child.LockEntities(entities...) + if err != nil { + panic(err) + } + defer unlock() + _ = child.WaitForSingleEvent("never", -1).Await(nil) + }) + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + return nil, errors.New("root failure") + }) + first := executeOrchestrationTurn(t, registry, "teardown-instance", nil, teardownStarted()) + require.Len(t, first.Actions, 1) + request := first.Actions[0].GetSendEntityMessage().GetEntityLockRequested() + require.NotNil(t, request) + old := append(teardownStarted(), lockRequestHistory(first.Actions[0])) + response := executeOrchestrationTurn(t, registry, "teardown-instance", old, []*protos.HistoryEvent{ + lockGrantedHistory(request.CriticalSectionId), + helpers.NewEventRaisedEvent("finish", nil), + }) + require.Len(t, response.Actions, 3) + for index, entity := range entities { + unlocked := response.Actions[index].GetSendEntityMessage().GetEntityUnlockSent() + require.NotNil(t, unlocked) + require.Equal(t, request.CriticalSectionId, unlocked.CriticalSectionId) + require.Equal(t, entity.String(), unlocked.GetTargetInstanceId().GetValue()) + } + require.Equal(t, api.RUNTIME_STATUS_FAILED, response.Actions[2].GetCompleteOrchestration().GetOrchestrationStatus()) +} diff --git a/task/waitgroup.go b/task/waitgroup.go index 6eb39b44..12ec558f 100644 --- a/task/waitgroup.go +++ b/task/waitgroup.go @@ -2,6 +2,9 @@ package task // WaitGroup coordinates orchestration coroutines. It behaves like sync.WaitGroup, // but Wait yields to the deterministic orchestration scheduler. +// Do not pair Add with a Done deferred only inside a coroutine that can be skipped +// before starting because its scope was canceled. Use an uncanceled wrapper and +// a separate cancellation scope for its body operations instead. type WaitGroup interface { Add(delta int) Done() @@ -41,7 +44,7 @@ func (wg *orchestrationWaitGroup) Done() { } func (wg *orchestrationWaitGroup) Wait(ctx *OrchestrationContext) { - if ctx.engineContext().scheduler != wg.scheduler { + if ctx.effectContext().scheduler != wg.scheduler { panic("task.WaitGroup belongs to a different orchestration") } for wg.count > 0 { diff --git a/tests/durabletaskscheduler/dts_teardown_test.go b/tests/durabletaskscheduler/dts_teardown_test.go new file mode 100644 index 00000000..cb6ec72e --- /dev/null +++ b/tests/durabletaskscheduler/dts_teardown_test.go @@ -0,0 +1,210 @@ +package durabletaskscheduler_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +func TestDTSTeardown(t *testing.T) { + options := emulatorOptions(t) + for _, test := range []struct { + name string + status api.OrchestrationStatus + }{ + {"status", api.RUNTIME_STATUS_COMPLETED}, + {"completed", api.RUNTIME_STATUS_COMPLETED}, + {"continued", api.RUNTIME_STATUS_COMPLETED}, + {"terminated", api.RUNTIME_STATUS_TERMINATED}, + {"root-failure", api.RUNTIME_STATUS_FAILED}, + {"root-defer", api.RUNTIME_STATUS_COMPLETED}, + {"join", api.RUNTIME_STATUS_COMPLETED}, + } { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() // Release the interceptor before instance-cleanup callbacks run. + id := uniqueInstanceID("go-teardown-" + test.name) + name, cleanup := string(id), string(id)+"-cleanup" + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN(cleanup, func(task.ActivityContext) (any, error) { + return "cleaned", nil + }), "SETUP: register cleanup") + require.NoError(t, registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + if test.name == "continued" { + var generation int + if err := ctx.GetInput(&generation); err != nil { + return nil, err + } + if generation == 1 { + return "next-generation", nil + } + } + switch test.name { + case "status": + ctx.SetCustomStatus("waiting") + defer ctx.SetCustomStatus("finished") + case "root-defer": + defer func() { _ = ctx.CallActivity(cleanup).Await(nil) }() + case "join": + work, cancel := ctx.WithCancel() + group := ctx.NewWaitGroup() + var childErr error + group.Add(1) + ctx.Go(func(child *task.OrchestrationContext) { + defer group.Done() + defer func() { childErr = errors.Join(childErr, child.CallActivity(cleanup).Await(nil)) }() + childErr = work.WaitForSingleEvent("work", -1).Await(nil) + if childErr == task.ErrTaskCanceled { //nolint:errorlint // Only body cancellation is expected, not cleanup failure. + childErr = nil + } + }) + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + cancel() + group.Wait(ctx) + return "joined", childErr + default: + ctx.Go(func(child *task.OrchestrationContext) { + defer func() { _ = child.CallActivity(cleanup).Await(nil) }() + _ = child.WaitForSingleEvent("signal", -1).Await(nil) + }) + } + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + if test.name == "root-failure" { + return nil, errors.New("intentional root failure") + } + if test.name == "continued" { + ctx.ContinueAsNew(1) + } + return "done", nil + }), "SETUP: register orchestrator") + + type completion struct { + response *protos.OrchestratorResponse + err error + } + completions := make(chan completion) + workerOptions := *options + workerOptions.UnaryInterceptors = []grpc.UnaryClientInterceptor{ + func(callCtx context.Context, method string, request, reply any, connection *grpc.ClientConn, invoke grpc.UnaryInvoker, callOptions ...grpc.CallOption) error { + err := invoke(callCtx, method, request, reply, connection, callOptions...) + if response, ok := request.(*protos.OrchestratorResponse); ok && + method == protos.TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName && response.InstanceId == string(id) { + copy := proto.CloneOf(response) + copy.CompletionToken = "" + select { + case completions <- completion{copy, err}: + case <-ctx.Done(): + } + } + return err + }, + } + client, _ := startEmulatorWithOptions(t, &workerOptions, registry, + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithWorkerRPCTimeout(30*time.Second), + durabletaskclient.WithWorkerTransientRetryPolicy(1, time.Millisecond, time.Millisecond)) + t.Cleanup(func() { cleanupTeardownInstance(t, client, id) }) + nextCompletion := func() *protos.CompleteOrchestrationAction { + t.Helper() + select { + case result := <-completions: + var terminal *protos.CompleteOrchestrationAction + for index, action := range result.response.Actions { + if completed := action.GetCompleteOrchestration(); completed != nil { + require.Equal(t, len(result.response.Actions)-1, index, "RED:ILLEGAL-ACTION-SEQUENCE") + terminal = completed + } + } + if test.name == "status" && terminal == nil { + require.Equal(t, "waiting", result.response.GetCustomStatus().GetValue(), "RED:DURABLE-STATE-LEAK") + } + require.NoError(t, result.err, "SERVICE: completion RPC; outcome unknown on failure") + return terminal + case <-ctx.Done(): + t.Fatal("HOST-TIMEOUT: waiting for completion", ctx.Err()) + return nil + } + } + + _, err := client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id), api.WithInput(0)) + require.NoError(t, err, "SETUP: schedule") + require.Nil(t, nextCompletion(), "RED:PREMATURE-COMPLETION") + if test.name == "status" { + require.NoError(t, client.RaiseEvent(ctx, id, "unrelated"), "SETUP: unrelated event") + require.Nil(t, nextCompletion(), "RED:PREMATURE-COMPLETION") + } + if test.name == "terminated" { + require.NoError(t, client.TerminateOrchestration(ctx, id, api.WithRecursiveTerminate(false)), "SETUP: terminate") + } else { + require.NoError(t, client.RaiseEvent(ctx, id, "finish"), "SETUP: finish event") + } + for { + if terminal := nextCompletion(); terminal != nil && + terminal.OrchestrationStatus != api.RUNTIME_STATUS_CONTINUED_AS_NEW { + require.Equal(t, test.status, terminal.OrchestrationStatus, "RED:TERMINAL-STATUS") + break + } + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err, "SERVICE: terminal metadata") + require.Equal(t, test.status, metadata.RuntimeStatus, "RED:TERMINAL-STATUS") + if test.name == "root-defer" || test.name == "join" { + history, err := client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: metadata.ExecutionID}) + require.NoError(t, err, "SERVICE: cleanup history") + activityID, completed, terminal := int32(-1), false, false + for _, event := range history.Events { + switch event.Type { + case api.HistoryEventTaskScheduled: + require.Equal(t, cleanup, event.TaskScheduled.Name, "RED:CLEANUP-ORDER") + require.EqualValues(t, -1, activityID, "RED:CLEANUP-ORDER: duplicate schedule") + require.False(t, terminal, "RED:CLEANUP-ORDER: schedule after completion") + activityID = event.EventID + case api.HistoryEventTaskCompleted: + require.NotEqualValues(t, -1, activityID, "RED:CLEANUP-ORDER: missing schedule") + require.Equal(t, activityID, event.TaskCompleted.TaskScheduledID, "RED:CLEANUP-ORDER") + require.False(t, terminal, "RED:CLEANUP-ORDER: result after completion") + completed = true + case api.HistoryEventExecutionCompleted: + require.True(t, completed, "RED:CLEANUP-ORDER: cleanup was not awaited") + terminal = true + } + } + require.True(t, terminal, "RED:CLEANUP-ORDER: missing terminal history") + } + }) + } +} + +func cleanupTeardownInstance(t *testing.T, client *durabletaskscheduler.Client, id api.InstanceID) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + metadata, err := client.FetchOrchestrationMetadata(ctx, id) + if errors.Is(err, api.ErrInstanceNotFound) { + return + } + require.NoError(t, err, "CLEANUP: fetch %s", id) + if !metadata.IsComplete() { + require.NoError(t, client.TerminateOrchestration(ctx, id, api.WithRecursiveTerminate(false)), "CLEANUP: terminate %s", id) + _, err = client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err, "CLEANUP: termination %s", id) + } + _, err = client.PurgeInstances(ctx, api.PurgeInstancesRequest{InstanceIDs: []api.InstanceID{id}, Recursive: false}) + require.NoError(t, err, "CLEANUP: batch purge %s", id) + _, err = client.FetchOrchestrationMetadata(ctx, id) + require.ErrorIs(t, err, api.ErrInstanceNotFound, "CLEANUP: verify absent %s", id) +}