From 89b9fe88e789e2ef9c89abea4dc6e7255ebbaeff Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Tue, 5 May 2026 13:41:00 -0700 Subject: [PATCH 01/52] Implement standalone callbacks --- chasm/lib/activity/activity.go | 4 +- chasm/lib/callback/component.go | 155 ++- chasm/lib/callback/component_properties.go | 165 +++ chasm/lib/callback/config.go | 70 +- chasm/lib/callback/frontend.go | 335 +++++ chasm/lib/callback/frontend_validation.go | 261 ++++ chasm/lib/callback/fx.go | 32 +- .../callbackpb/v1/message.go-helpers.pb.go | 1 + .../callback/gen/callbackpb/v1/message.pb.go | 118 +- .../v1/request_response.go-helpers.pb.go | 376 +++++ .../gen/callbackpb/v1/request_response.pb.go | 617 +++++++++ .../callback/gen/callbackpb/v1/service.pb.go | 90 ++ .../gen/callbackpb/v1/service_client.pb.go | 275 ++++ .../gen/callbackpb/v1/service_grpc.pb.go | 258 ++++ .../gen/callbackpb/v1/tasks.go-helpers.pb.go | 37 + .../callback/gen/callbackpb/v1/tasks.pb.go | 49 +- chasm/lib/callback/handler.go | 352 +++++ chasm/lib/callback/invocable_internal.go | 13 +- chasm/lib/callback/invocable_outbound.go | 11 +- chasm/lib/callback/library.go | 63 +- chasm/lib/callback/proto/v1/message.proto | 18 + .../callback/proto/v1/request_response.proto | 62 + chasm/lib/callback/proto/v1/service.proto | 36 + chasm/lib/callback/proto/v1/tasks.proto | 3 + chasm/lib/callback/statemachine.go | 81 +- chasm/lib/callback/statemachine_test.go | 54 +- chasm/lib/callback/tasks.go | 27 + chasm/lib/callback/tasks_test.go | 4 +- chasm/lib/callback/validator.go | 8 + chasm/lib/workflow/workflow.go | 2 +- client/frontend/client_gen.go | 70 + client/frontend/metric_client_gen.go | 98 ++ client/frontend/retryable_client_gen.go | 105 ++ .../logtags/workflow_service_server_gen.go | 40 + .../v1/service_grpc.pb.mock.go | 140 ++ go.mod | 2 +- go.sum | 4 +- service/frontend/configs/quotas.go | 9 +- service/frontend/configs/quotas_test.go | 2 + service/frontend/fx.go | 3 + service/frontend/service.go | 4 + service/frontend/workflow_handler.go | 11 +- service/frontend/workflow_handler_test.go | 1 + service/history/fx.go | 2 + temporal/fx.go | 2 - tests/standalone_callbacks_test.go | 1228 +++++++++++++++++ 46 files changed, 5184 insertions(+), 114 deletions(-) create mode 100644 chasm/lib/callback/component_properties.go create mode 100644 chasm/lib/callback/frontend.go create mode 100644 chasm/lib/callback/frontend_validation.go create mode 100644 chasm/lib/callback/gen/callbackpb/v1/request_response.go-helpers.pb.go create mode 100644 chasm/lib/callback/gen/callbackpb/v1/request_response.pb.go create mode 100644 chasm/lib/callback/gen/callbackpb/v1/service.pb.go create mode 100644 chasm/lib/callback/gen/callbackpb/v1/service_client.pb.go create mode 100644 chasm/lib/callback/gen/callbackpb/v1/service_grpc.pb.go create mode 100644 chasm/lib/callback/handler.go create mode 100644 chasm/lib/callback/proto/v1/request_response.proto create mode 100644 chasm/lib/callback/proto/v1/service.proto create mode 100644 tests/standalone_callbacks_test.go diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index df45a9ac490..a8d08c282fc 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -335,9 +335,9 @@ func (a *Activity) addCompletionCallbacks( return serviceerror.NewInvalidArgumentf("unsupported callback variant: %T", variant) } - // requestID (unique per API call) + idx (position within the request) ensures unique,idempotent callback IDs. + // requestID (unique per API call) + idx (position within the request) ensures unique, idempotent callback IDs. id := fmt.Sprintf("%s-%d", requestID, idx) - callbackObj := callback.NewCallback(requestID, registrationTime, &callbackspb.CallbackState{}, chasmCB) + callbackObj := callback.NewEmbeddedCallback(ctx, requestID, registrationTime, chasmCB) a.Callbacks[id] = chasm.NewComponentField(ctx, callbackObj) } return nil diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index c017a7d2f30..02a04e8f18e 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -4,13 +4,17 @@ import ( "fmt" "time" + callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" + failurepb "go.temporal.io/api/failure/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/nexus/nexusrpc" queueserrors "go.temporal.io/server/service/history/queues/errors" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -18,8 +22,27 @@ type CompletionSource interface { GetNexusCompletion(ctx chasm.Context, requestID string) (nexusrpc.CompleteOperationOptions, error) } -var _ chasm.Component = (*Callback)(nil) -var _ chasm.StateMachine[callbackspb.CallbackStatus] = (*Callback)(nil) +// CompletionSourceFn allows a function value to be used as a CompletionSource instance. +type CompletionSourceFn func(chasm.Context, string) (nexusrpc.CompleteOperationOptions, error) + +func (csFunc CompletionSourceFn) GetNexusCompletion(ctx chasm.Context, requestID string) (nexusrpc.CompleteOperationOptions, error) { + return csFunc(ctx, requestID) +} + +var ( + _ chasm.Component = (*Callback)(nil) + _ chasm.StateMachine[callbackspb.CallbackStatus] = (*Callback)(nil) + + // Capabilities only supported/used for standalone callbacks. + _ chasm.RootComponent = (*Callback)(nil) + _ chasm.VisibilityMemoProvider = (*Callback)(nil) + _ chasm.VisibilitySearchAttributesProvider = (*Callback)(nil) +) + +var executionStatusSearchAttribute = chasm.NewSearchAttributeKeyword( + "ExecutionStatus", + chasm.SearchAttributeFieldLowCardinalityKeyword01, +) // Callback represents a callback component in CHASM. type Callback struct { @@ -27,15 +50,26 @@ type Callback struct { // Persisted internal state *callbackspb.CallbackState + // Failure from an external termination (timeout or terminate), stored separately because + // of its potential size, and to not overload CallbackState::LastAttemptFailure. + TerminalFailure chasm.Field[*failurepb.Failure] + + // For most callbacks, the completion result is obtained from the parent component. + // e.g. the Workflow result to be delivered. However, for "standalone" callbacks, there + // is no parent and the user-supplied SuppliedCompletion will be used instead. + ParentCompletionSource chasm.ParentPtr[CompletionSource] + SuppliedCompletion chasm.Field[*callbackpb.CallbackExecutionCompletion] - // Interface to retrieve Nexus operation completion data - CompletionSource chasm.ParentPtr[CompletionSource] + // Visibility sub-component for search attributes and memo indexing. + Visibility chasm.Field[*chasm.Visibility] } -func NewCallback( +// NewEmbeddedCallback returns a Callback component, which will deliver the completion from +// its parent CHASM component. The parent must implement CompletionSource. +func NewEmbeddedCallback( + ctx chasm.MutableContext, requestID string, registrationTime *timestamppb.Timestamp, - state *callbackspb.CallbackState, cb *callbackspb.Callback, ) *Callback { return &Callback{ @@ -45,14 +79,49 @@ func NewCallback( Callback: cb, Status: callbackspb.CALLBACK_STATUS_STANDBY, }, + TerminalFailure: chasm.NewDataField[*failurepb.Failure](ctx, nil), } } +type newStandaloneCallbackOpts struct { + RequestID string + RegistrationTime *timestamppb.Timestamp + Callback *callbackspb.Callback + + CallbackID string + CompletionScheduleToCloseTimeout *durationpb.Duration + Completion *callbackpb.CallbackExecutionCompletion + SearchAttributes map[string]*commonpb.Payload +} + +// newStandaloneCallback returns a new Callback component which will deliver the supplied +// completion result. +func newStandaloneCallback( + ctx chasm.MutableContext, + opts newStandaloneCallbackOpts, +) *Callback { + cb := NewEmbeddedCallback(ctx, opts.RequestID, opts.RegistrationTime, opts.Callback) + + // Add standalone-specific fields. + cb.CallbackId = opts.CallbackID + cb.CompletionScheduleToCloseTimeout = opts.CompletionScheduleToCloseTimeout + cb.SuppliedCompletion = chasm.NewDataField(ctx, opts.Completion) + + visibility := chasm.NewVisibilityWithData(ctx, opts.SearchAttributes, nil) + cb.Visibility = chasm.NewComponentField(ctx, visibility) + + return cb +} + func (c *Callback) LifecycleState(_ chasm.Context) chasm.LifecycleState { switch c.Status { case callbackspb.CALLBACK_STATUS_SUCCEEDED: return chasm.LifecycleStateCompleted - case callbackspb.CALLBACK_STATUS_FAILED: + case callbackspb.CALLBACK_STATUS_FAILED, + callbackspb.CALLBACK_STATUS_TERMINATED: + // TODO: Use chasm.LifecycleStateTerminated when it's available (currently commented out + // in chasm/component.go:70). For now, LifecycleStateFailed is functionally correct + // as IsClosed() returns true for all states >= LifecycleStateCompleted. return chasm.LifecycleStateFailed default: return chasm.LifecycleStateRunning @@ -67,6 +136,62 @@ func (c *Callback) SetStateMachineState(status callbackspb.CallbackStatus) { c.Status = status } +func (c *Callback) ContextMetadata(_ chasm.Context) map[string]string { + return map[string]string{ + "RequestID": c.RequestId, + // Only set for standalone callbacks. + "CallbackID": c.CallbackId, + } +} + +// SearchAttributes implements chasm.VisibilitySearchAttributesProvider. +func (c *Callback) SearchAttributes(ctx chasm.Context) []chasm.SearchAttributeKeyValue { + apiStatus := callbackStatusToAPIExecutionStatus(c.Status) + return []chasm.SearchAttributeKeyValue{ + executionStatusSearchAttribute.Value(apiStatus.String()), + } +} + +// Memo implements chasm.VisibilityMemoProvider. Returns the CallbackExecutionListInfo +// as the memo for visibility queries. +func (c *Callback) Memo(ctx chasm.Context) proto.Message { + return &callbackpb.CallbackExecutionListInfo{ + CallbackId: c.CallbackId, + Status: callbackStatusToAPIExecutionStatus(c.Status), + CreateTime: c.RegistrationTime, + CloseTime: c.CloseTime, + } +} + +// Terminate forcefully terminates the callback execution. +// +// If already terminated with the same request ID, this is a no-op. +// If already terminated with a different request ID, returns FailedPrecondition. +func (c *Callback) Terminate( + ctx chasm.MutableContext, + req chasm.TerminateComponentRequest, +) (chasm.TerminateComponentResponse, error) { + if c.LifecycleState(ctx).IsClosed() { + if c.TerminateRequestId == "" { + // Completed organically (succeeded/failed/timed out), not via Terminate. + err := serviceerror.NewFailedPreconditionf("callback execution already in terminal state %v", c.Status) + return chasm.TerminateComponentResponse{}, err + } + if c.TerminateRequestId != req.RequestID { + err := serviceerror.NewFailedPreconditionf("already terminated with request ID %s", c.TerminateRequestId) + return chasm.TerminateComponentResponse{}, err + } + return chasm.TerminateComponentResponse{}, nil + } + if err := TransitionTerminated.Apply(c, ctx, EventTerminated{Reason: req.Reason}); err != nil { + return chasm.TerminateComponentResponse{}, fmt.Errorf("failed to terminate callback: %w", err) + } + + c.TerminateRequestId = req.RequestID + // c.TerminalFailure is set in the transition handler. + return chasm.TerminateComponentResponse{}, nil +} + func (c *Callback) recordAttempt(ts time.Time) { c.Attempt++ c.LastAttemptCompleteTime = timestamppb.New(ts) @@ -77,9 +202,9 @@ func (c *Callback) loadInvocationArgs( ctx chasm.Context, _ chasm.NoValue, ) (invocable, error) { - target := c.CompletionSource.Get(ctx) - - completion, err := target.GetNexusCompletion(ctx, c.RequestId) + // Get the completion result to be delivered. + completionSource := c.CompletionSource(ctx) + completion, err := completionSource.GetNexusCompletion(ctx, c.RequestId) if err != nil { return nil, err } @@ -117,6 +242,16 @@ func (c *Callback) saveResult( ctx chasm.MutableContext, input saveResultInput, ) (chasm.NoValue, error) { + // If the callback was terminated while the invocation was in-flight, + // the result is no longer relevant. We'll just drop it silently. + // + // This shouldn't happen outside of tests, since the Nexus machinary + // would prevent an invalid transition anyways. (e.g. terminating + // an already terminated Callback.) + if c.LifecycleState(ctx).IsClosed() { + return nil, nil + } + switch r := input.result.(type) { case invocationResultOK: err := TransitionSucceeded.Apply(c, ctx, EventSucceeded{Time: ctx.Now(c)}) diff --git a/chasm/lib/callback/component_properties.go b/chasm/lib/callback/component_properties.go new file mode 100644 index 00000000000..497e15f876f --- /dev/null +++ b/chasm/lib/callback/component_properties.go @@ -0,0 +1,165 @@ +package callback + +import ( + "fmt" + + "github.com/nexus-rpc/sdk-go/nexus" + callbackpb "go.temporal.io/api/callback/v1" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" +) + +func callbackCompletionToNexusCompleteOperationOpts( + cb *Callback, + completion *callbackpb.CallbackExecutionCompletion) (nexusrpc.CompleteOperationOptions, error) { + + nexusCompletion := nexusrpc.CompleteOperationOptions{ + StartTime: cb.GetRegistrationTime().AsTime(), + CloseTime: cb.CloseTime.AsTime(), + } + + switch completion.Result.(type) { + case *callbackpb.CallbackExecutionCompletion_Success: + nexusCompletion.Result = completion.GetSuccess() + return nexusCompletion, nil + + case *callbackpb.CallbackExecutionCompletion_Failure: + f, err := commonnexus.TemporalFailureToNexusFailure(completion.GetFailure()) + if err != nil { + wrappedErr := fmt.Errorf("failed to convert failure: %w", err) + return nexusrpc.CompleteOperationOptions{}, wrappedErr + } + opErr := &nexus.OperationError{ + State: nexus.OperationStateFailed, + Message: "operation failed", + Cause: &nexus.FailureError{Failure: f}, + } + if err := nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr); err != nil { + wrappedErr := fmt.Errorf("failed to mark wrapper error: %w", err) + return nexusrpc.CompleteOperationOptions{}, wrappedErr + } + nexusCompletion.Error = opErr + return nexusCompletion, nil + + default: + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInvalidArgument("no completion result provided") + } +} + +// CompletionSource returns the CompletionSource from the callback, which depends on whether it +// is embedded or is running in standalone mode. +func (c *Callback) CompletionSource(ctx chasm.Context) CompletionSource { + // Embedded callbacks use their parent component as a CompletionSource. + source, ok := c.ParentCompletionSource.TryGet(ctx) + if ok { + return source + } + + // For standalone completions, get the user-supplied value and convert it + // into the Nexus API type. + suppliedCompletion, ok := c.SuppliedCompletion.TryGet(ctx) + if !ok { + return CompletionSourceFn(func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInternal("no completion available") + }) + } + + convertOutcomeProtoFn := func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { + return callbackCompletionToNexusCompleteOperationOpts(c, suppliedCompletion) + } + return CompletionSourceFn(convertOutcomeProtoFn) +} + +// callbackStatusToAPIExecutionStatus maps internal CallbackStatus to public API CallbackExecutionStatus. +func callbackStatusToAPIExecutionStatus(status callbackspb.CallbackStatus) enumspb.CallbackExecutionStatus { + switch status { + case callbackspb.CALLBACK_STATUS_STANDBY, + callbackspb.CALLBACK_STATUS_SCHEDULED, + callbackspb.CALLBACK_STATUS_BACKING_OFF: + return enumspb.CALLBACK_EXECUTION_STATUS_RUNNING + case callbackspb.CALLBACK_STATUS_FAILED: + return enumspb.CALLBACK_EXECUTION_STATUS_FAILED + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + return enumspb.CALLBACK_EXECUTION_STATUS_SUCCEEDED + case callbackspb.CALLBACK_STATUS_TERMINATED: + return enumspb.CALLBACK_EXECUTION_STATUS_TERMINATED + default: + return enumspb.CALLBACK_EXECUTION_STATUS_UNSPECIFIED + } +} + +// callbackStatusToAPIState maps internal CallbackStatus to public API CallbackState. +func callbackStatusToAPIState(status callbackspb.CallbackStatus) enumspb.CallbackState { + switch status { + case callbackspb.CALLBACK_STATUS_STANDBY: + return enumspb.CALLBACK_STATE_STANDBY + case callbackspb.CALLBACK_STATUS_SCHEDULED: + return enumspb.CALLBACK_STATE_SCHEDULED + case callbackspb.CALLBACK_STATUS_BACKING_OFF: + return enumspb.CALLBACK_STATE_BACKING_OFF + case callbackspb.CALLBACK_STATUS_FAILED: + return enumspb.CALLBACK_STATE_FAILED + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + return enumspb.CALLBACK_STATE_SUCCEEDED + case callbackspb.CALLBACK_STATUS_TERMINATED: + return enumspb.CALLBACK_STATE_TERMINATED + default: + return enumspb.CALLBACK_STATE_UNSPECIFIED + } +} + +// Describe returns the CallbackExecutionInfo for the describe RPC. Only applies to standalone callbacks. +func (c *Callback) Describe(ctx chasm.Context) (*callbackpb.CallbackExecutionInfo, error) { + apiCb, err := c.ToAPICallback() + if err != nil { + return nil, err + } + + exInfo := ctx.ExecutionInfo() + info := &callbackpb.CallbackExecutionInfo{ + CallbackId: c.CallbackId, + RunId: ctx.ExecutionKey().RunID, + Callback: apiCb, + Status: callbackStatusToAPIExecutionStatus(c.Status), + State: callbackStatusToAPIState(c.Status), + Attempt: c.Attempt, + CreateTime: c.RegistrationTime, + LastAttemptCompleteTime: c.LastAttemptCompleteTime, + LastAttemptFailure: c.LastAttemptFailure, + NextAttemptScheduleTime: c.NextAttemptScheduleTime, + CloseTime: c.CloseTime, + ScheduleToCloseTimeout: c.CompletionScheduleToCloseTimeout, + StateTransitionCount: exInfo.StateTransitionCount, + } + return info, nil +} + +// Outcome returns the callback execution outcome if the execution is in a terminal state. (Otherwise, nil.) +// +// IMPORTANT: This is specific to the callback delivery, and not the actual completion. The outcome will be +// a success even if it was to deliver a failed completion result. +func (c *Callback) Outcome(ctx chasm.Context) *callbackpb.CallbackExecutionOutcome { + switch c.Status { + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + val := &callbackpb.CallbackExecutionOutcome_Success{} + return &callbackpb.CallbackExecutionOutcome{ + Value: val, + } + + case callbackspb.CALLBACK_STATUS_FAILED, + callbackspb.CALLBACK_STATUS_TERMINATED: + val := &callbackpb.CallbackExecutionOutcome_Failure{ + Failure: c.TerminalFailure.Get(ctx), + } + return &callbackpb.CallbackExecutionOutcome{ + Value: val, + } + + default: + return nil + } +} diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 844add8d671..63a1ffa58ee 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -14,6 +14,25 @@ import ( "google.golang.org/grpc/status" ) +var EnableStandaloneExecutions = dynamicconfig.NewNamespaceBoolSetting( + "callback.enableStandaloneExecutions", + false, + `Toggles standalone callback execution functionality on the server.`, +) + +var LongPollBuffer = dynamicconfig.NewNamespaceDurationSetting( + "callback.longPollBuffer", + time.Second, + `A buffer used to adjust the callback execution long-poll timeouts. +The long-poll response is sent before the caller's deadline by this amount of time.`, +) + +var LongPollTimeout = dynamicconfig.NewNamespaceDurationSetting( + "callback.longPollTimeout", + 20*time.Second, + `Timeout for callback execution long-poll requests.`, +) + var MaxPerExecution = dynamicconfig.NewNamespaceIntSetting( "callback.maxPerExecution", 2000, @@ -39,13 +58,37 @@ var RetryPolicyMaximumInterval = dynamicconfig.NewGlobalDurationSetting( ) type Config struct { - RequestTimeout dynamicconfig.DurationPropertyFnWithDestinationFilter - RetryPolicy func() backoff.RetryPolicy + // callback.* settings. + EnableStandaloneExecutions dynamicconfig.BoolPropertyFnWithNamespaceFilter + LongPollBuffer dynamicconfig.DurationPropertyFnWithNamespaceFilter + LongPollTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter + RequestTimeout dynamicconfig.DurationPropertyFnWithDestinationFilter + RetryPolicy func() backoff.RetryPolicy + + // Settings defined elsewhere. + CHASMEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter + CHASMCallbacksEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter + + // Validation related config. + BlobSizeLimitError dynamicconfig.IntPropertyFnWithNamespaceFilter + BlobSizeLimitWarn dynamicconfig.IntPropertyFnWithNamespaceFilter + MaxIDLength dynamicconfig.IntPropertyFn // Used to check CallbackID, RequestID, etc. + + // NOTE: The configuration setting defining the allowlist of supported callback + // addresses is defined in components/callbacks/config.go, via AllowedAddresses. + // + // Similarly, MaxPerExecution is missing. It is used by `Validator` and is loaded there. + // Once HSM callbacks (components/callbacks) are removed, the callbackValidatorProvider in + // frontend/fx.go can be moved into this package. And at that time, we can simply have the + // callback.Validator inject callback.Config. (And have a single location for all config.) } -func configProvider(dc *dynamicconfig.Collection) *Config { +func ConfigProvider(dc *dynamicconfig.Collection) *Config { return &Config{ - RequestTimeout: RequestTimeout.Get(dc), + EnableStandaloneExecutions: EnableStandaloneExecutions.Get(dc), + LongPollBuffer: LongPollBuffer.Get(dc), + LongPollTimeout: LongPollTimeout.Get(dc), + RequestTimeout: RequestTimeout.Get(dc), RetryPolicy: func() backoff.RetryPolicy { return backoff.NewExponentialRetryPolicy( RetryPolicyInitialInterval.Get(dc)(), @@ -55,21 +98,16 @@ func configProvider(dc *dynamicconfig.Collection) *Config { backoff.NoInterval, ) }, + + CHASMEnabled: dynamicconfig.EnableChasm.Get(dc), + CHASMCallbacksEnabled: dynamicconfig.EnableCHASMCallbacks.Get(dc), + + MaxIDLength: dynamicconfig.MaxIDLengthLimit.Get(dc), + BlobSizeLimitError: dynamicconfig.BlobSizeLimitError.Get(dc), + BlobSizeLimitWarn: dynamicconfig.BlobSizeLimitWarn.Get(dc), } } -var AllowedAddresses = dynamicconfig.NewNamespaceTypedSettingWithConverter( - "chasm.callback.allowedAddresses", - allowedAddressConverter, - AddressMatchRules{}, - `The per-namespace list of addresses that are allowed for callbacks and whether secure connections (https) are required. -URL: "temporal://system" is always allowed for worker callbacks. The default is no address rules. -URLs are checked against each in order when starting a workflow with attached callbacks and only need to match one to pass validation. -This configuration is required for external endpoint targets; any invalid entries are ignored. Each entry is a map with possible values: - - "Pattern":string (required) the host:port pattern to which this config applies. - Wildcards, '*', are supported and can match any number of characters (e.g. '*' matches everything, 'prefix.*.domain' matches 'prefix.a.domain' as well as 'prefix.a.b.domain'). - - "AllowInsecure":bool (optional, default=false) indicates whether https is required`) - type AddressMatchRules struct { Rules []AddressMatchRule } diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go new file mode 100644 index 00000000000..b2d79d6942f --- /dev/null +++ b/chasm/lib/callback/frontend.go @@ -0,0 +1,335 @@ +package callback + +import ( + "context" + + callbackpb "go.temporal.io/api/callback/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/searchattribute" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ErrStandaloneCallbacksDisabled = serviceerror.NewUnimplemented("standalone callback executions are not enabled") + +// FrontendHandler defines the frontend interface for standalone callback execution RPCs, +// in which the Frontend microservice receives requests from the Temporal SDK and proxies +// them to the implementation of the CHASM component running in the History service. +type FrontendHandler interface { + StartCallbackExecution(context.Context, *workflowservice.StartCallbackExecutionRequest) (*workflowservice.StartCallbackExecutionResponse, error) + DescribeCallbackExecution(context.Context, *workflowservice.DescribeCallbackExecutionRequest) (*workflowservice.DescribeCallbackExecutionResponse, error) + PollCallbackExecution(context.Context, *workflowservice.PollCallbackExecutionRequest) (*workflowservice.PollCallbackExecutionResponse, error) + ListCallbackExecutions(context.Context, *workflowservice.ListCallbackExecutionsRequest) (*workflowservice.ListCallbackExecutionsResponse, error) + CountCallbackExecutions(context.Context, *workflowservice.CountCallbackExecutionsRequest) (*workflowservice.CountCallbackExecutionsResponse, error) + TerminateCallbackExecution(context.Context, *workflowservice.TerminateCallbackExecutionRequest) (*workflowservice.TerminateCallbackExecutionResponse, error) + DeleteCallbackExecution(context.Context, *workflowservice.DeleteCallbackExecutionRequest) (*workflowservice.DeleteCallbackExecutionResponse, error) +} + +type frontendHandler struct { + logger log.Logger + namespaceRegistry namespace.Registry + + client callbackspb.CallbackServiceClient + config *Config + reqValidator *frontendRequestValidator +} + +func NewFrontendHandler( + logger log.Logger, + namespaceRegistry namespace.Registry, + client callbackspb.CallbackServiceClient, + config *Config, + callbackValidator Validator, + saMapperProvider searchattribute.MapperProvider, + saValidator *searchattribute.Validator, +) FrontendHandler { + return &frontendHandler{ + logger: logger, + namespaceRegistry: namespaceRegistry, + client: client, + config: config, + reqValidator: &frontendRequestValidator{ + config: config, + cbValidator: callbackValidator, + logger: logger, + saMapperProvider: saMapperProvider, + saValidator: saValidator, + }, + } +} + +type Namespacer interface{ GetNamespace() string } + +// Looks up the namespace ID from the user-supplied namespace name in the request proto. +func (h *frontendHandler) getTargetNamespace(requestProto Namespacer) (namespace.ID, error) { + targetNamespaceName := namespace.Name(requestProto.GetNamespace()) + namespaceID, err := h.namespaceRegistry.GetNamespaceID(targetNamespaceName) + if err != nil { + return "", err + } + return namespaceID, nil +} + +// Checks if standalone callback executions are supported in the target namespace. +func (h *frontendHandler) checkFeatureEnabled(requestProto Namespacer) error { + // Confirm CHASM is enabled. + targetNamespaceName := requestProto.GetNamespace() + if !h.config.CHASMEnabled(targetNamespaceName) || !h.config.CHASMCallbacksEnabled(targetNamespaceName) { + return ErrStandaloneCallbacksDisabled + } + if !h.config.EnableStandaloneExecutions(targetNamespaceName) { + return ErrStandaloneCallbacksDisabled + } + return nil +} + +// StartCallbackExecution creates a new standalone callback execution that will deliver the +// provided Nexus completion payload to the target callback URL with retries. +func (h *frontendHandler) StartCallbackExecution( + ctx context.Context, + request *workflowservice.StartCallbackExecutionRequest, +) (*workflowservice.StartCallbackExecutionResponse, error) { + // Validate + if err := h.checkFeatureEnabled(request); err != nil { + return nil, err + } + if err := h.reqValidator.ValidateStartCallbackExecution(request); err != nil { + return nil, err + } + + // Execute + namespaceID, err := h.getTargetNamespace(request) + if err != nil { + return nil, err + } + resp, err := h.client.StartCallbackExecution(ctx, &callbackspb.StartCallbackExecutionRequest{ + NamespaceId: namespaceID.String(), + FrontendRequest: request, + }) + if err != nil { + return nil, err + } + return resp.GetFrontendResponse(), nil +} + +// DescribeCallbackExecution returns detailed information about a callback execution +// including its current state, delivery attempt history, and timing information. +// Optionally takes a long-poll token and waits for any change. +func (h *frontendHandler) DescribeCallbackExecution( + ctx context.Context, + request *workflowservice.DescribeCallbackExecutionRequest, +) (*workflowservice.DescribeCallbackExecutionResponse, error) { + // Validate + if err := h.checkFeatureEnabled(request); err != nil { + return nil, err + } + if err := h.reqValidator.ValidateDescribeCallbackExecution(request); err != nil { + return nil, err + } + + // Execute + namespaceID, err := h.getTargetNamespace(request) + if err != nil { + return nil, err + } + resp, err := h.client.DescribeCallbackExecution(ctx, &callbackspb.DescribeCallbackExecutionRequest{ + NamespaceId: namespaceID.String(), + FrontendRequest: request, + }) + if err != nil { + return nil, err + } + return resp.GetFrontendResponse(), nil +} + +// PollCallbackExecution blocks until the callback execution completes and returns its outcome. +func (h *frontendHandler) PollCallbackExecution( + ctx context.Context, + request *workflowservice.PollCallbackExecutionRequest, +) (*workflowservice.PollCallbackExecutionResponse, error) { + // Validate + if err := h.checkFeatureEnabled(request); err != nil { + return nil, err + } + if err := h.reqValidator.ValidatePollCallbackExecution(request); err != nil { + return nil, err + } + + // Execute + namespaceID, err := h.getTargetNamespace(request) + if err != nil { + return nil, err + } + resp, err := h.client.PollCallbackExecution(ctx, &callbackspb.PollCallbackExecutionRequest{ + NamespaceId: namespaceID.String(), + FrontendRequest: request, + }) + if err != nil { + return nil, err + } + return resp.GetFrontendResponse(), nil +} + +// TerminateCallbackExecution forcefully stops a running callback execution. +// No-op if already in a terminal state. +func (h *frontendHandler) TerminateCallbackExecution( + ctx context.Context, + request *workflowservice.TerminateCallbackExecutionRequest, +) (*workflowservice.TerminateCallbackExecutionResponse, error) { + // Validate + if err := h.checkFeatureEnabled(request); err != nil { + return nil, err + } + if err := h.reqValidator.ValidateTerminateCallbackExecution(request); err != nil { + return nil, err + } + + // Execute + namespaceID, err := h.getTargetNamespace(request) + if err != nil { + return nil, err + } + resp, err := h.client.TerminateCallbackExecution(ctx, &callbackspb.TerminateCallbackExecutionRequest{ + NamespaceId: namespaceID.String(), + FrontendRequest: request, + }) + if err != nil { + return nil, err + } + return resp.GetFrontendResponse(), nil +} + +// DeleteCallbackExecution terminates the callback if still running and marks it for cleanup. +func (h *frontendHandler) DeleteCallbackExecution( + ctx context.Context, + request *workflowservice.DeleteCallbackExecutionRequest, +) (*workflowservice.DeleteCallbackExecutionResponse, error) { + // Validate + if err := h.checkFeatureEnabled(request); err != nil { + return nil, err + } + if err := h.reqValidator.ValidateDeleteCallbackExecution(request); err != nil { + return nil, err + } + + // Execute + namespaceID, err := h.getTargetNamespace(request) + if err != nil { + return nil, err + } + resp, err := h.client.DeleteCallbackExecution(ctx, &callbackspb.DeleteCallbackExecutionRequest{ + NamespaceId: namespaceID.String(), + FrontendRequest: request, + }) + if err != nil { + return nil, err + } + return resp.GetFrontendResponse(), nil +} + +// ListCallbackExecutions queries the visibility store for callback executions matching +// the provided filter. Supports the same query syntax as workflow list filters. +func (h *frontendHandler) ListCallbackExecutions( + ctx context.Context, + request *workflowservice.ListCallbackExecutionsRequest, +) (*workflowservice.ListCallbackExecutionsResponse, error) { + // Validate + if err := h.checkFeatureEnabled(request); err != nil { + return nil, err + } + if err := h.reqValidator.ValidateListCallbackExecutions(request); err != nil { + return nil, err + } + + // Lookup the namespace by its name, to confirm it actually exists. + namespaceName := namespace.Name(request.GetNamespace()) + if _, err := h.namespaceRegistry.GetNamespaceID(namespaceName); err != nil { + return nil, err + } + + resp, err := chasm.ListExecutions[*Callback, *callbackpb.CallbackExecutionListInfo]( + ctx, + &chasm.ListExecutionsRequest{ + NamespaceName: namespaceName.String(), + PageSize: int(request.GetPageSize()), + NextPageToken: request.GetNextPageToken(), + Query: request.GetQuery(), + }, + ) + if err != nil { + return nil, err + } + + // Build the response object. + executions := make([]*callbackpb.CallbackExecutionListInfo, 0, len(resp.Executions)) + for _, exec := range resp.Executions { + + statusStr, _ := chasm.SearchAttributeValue(exec.ChasmSearchAttributes, executionStatusSearchAttribute) + status, _ := enumspb.CallbackExecutionStatusFromString(statusStr) + + info := callbackpb.CallbackExecutionListInfo{ + CallbackId: exec.BusinessID, + RunId: exec.RunID, + Status: status, + CreateTime: timestamppb.New(exec.StartTime), + CloseTime: timestamppb.New(exec.CloseTime), + SearchAttributes: &commonpb.SearchAttributes{IndexedFields: exec.CustomSearchAttributes}, + StateTransitionCount: exec.StateTransitionCount, + } + executions = append(executions, &info) + } + return &workflowservice.ListCallbackExecutionsResponse{ + Executions: executions, + NextPageToken: resp.NextPageToken, + }, nil +} + +// CountCallbackExecutions returns the number of callback executions matching the query, +// with optional grouping by search attribute values. +func (h *frontendHandler) CountCallbackExecutions( + ctx context.Context, + request *workflowservice.CountCallbackExecutionsRequest, +) (*workflowservice.CountCallbackExecutionsResponse, error) { + // Validate + if err := h.checkFeatureEnabled(request); err != nil { + return nil, err + } + if err := h.reqValidator.ValidateCountCallbackExecutions(request); err != nil { + return nil, err + } + + // Lookup the namespace by its name, to confirm it actually exists. + namespaceName := namespace.Name(request.GetNamespace()) + if _, err := h.namespaceRegistry.GetNamespaceID(namespaceName); err != nil { + return nil, err + } + resp, err := chasm.CountExecutions[*Callback]( + ctx, + &chasm.CountExecutionsRequest{ + NamespaceName: namespaceName.String(), + Query: request.GetQuery(), + }, + ) + if err != nil { + return nil, err + } + + // Build the response object. + groups := make([]*workflowservice.CountCallbackExecutionsResponse_AggregationGroup, 0, len(resp.Groups)) + for _, g := range resp.Groups { + groups = append(groups, &workflowservice.CountCallbackExecutionsResponse_AggregationGroup{ + GroupValues: g.Values, + Count: g.Count, + }) + } + return &workflowservice.CountCallbackExecutionsResponse{ + Count: resp.Count, + Groups: groups, + }, nil +} diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go new file mode 100644 index 00000000000..69ad72df517 --- /dev/null +++ b/chasm/lib/callback/frontend_validation.go @@ -0,0 +1,261 @@ +package callback + +import ( + "fmt" + + "github.com/google/uuid" + callbackpb "go.temporal.io/api/callback/v1" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/searchattribute" + "google.golang.org/protobuf/proto" +) + +// Returns a serviceerror.InvalidArgument error for a missing required field. +func missingRequiredFieldError(fieldName string) error { + msg := fmt.Sprintf("%s is not set on request.", fieldName) + return serviceerror.NewInvalidArgument(msg) +} + +type RequestIDer interface { + GetRequestId() string +} + +func verifyRequestIDLength(reqProto RequestIDer, config *Config) error { + l := len(reqProto.GetRequestId()) + maxLen := config.MaxIDLength() + if l > maxLen { + return serviceerror.NewInvalidArgumentf("callback ID exceeds length limit. Length=%d Limit=%d", l, maxLen) + } + return nil +} + +type CallbackIDer interface { + GetCallbackId() string +} + +func verifyCallbackIDLength(reqProto CallbackIDer, config *Config) error { + l := len(reqProto.GetCallbackId()) + maxLen := config.MaxIDLength() + if l > maxLen { + return serviceerror.NewInvalidArgumentf("callback ID exceeds length limit. Length=%d Limit=%d", l, maxLen) + } + return nil +} + +// frontendRequestValidator bundles the configuration data for validating an incomming request. +// +// IMPORTANT: Validation methods MAY mutate the incomming request, in order to ensure they all have +// a valid RunID (if one was not specified already). +type frontendRequestValidator struct { + config *Config + cbValidator Validator + logger log.Logger + saMapperProvider searchattribute.MapperProvider + saValidator *searchattribute.Validator +} + +func (rv *frontendRequestValidator) ValidateStartCallbackExecution(req *workflowservice.StartCallbackExecutionRequest) error { + // Set RequestID if missing. + if req.GetRequestId() == "" { + req.RequestId = uuid.NewString() + } + + // Required fields. + requiredFields := map[string]string{ + "Namespace": req.GetNamespace(), + "Identity": req.GetIdentity(), + "RequestId": req.GetRequestId(), + "CallbackId": req.GetCallbackId(), + } + for k, v := range requiredFields { + if v == "" { + return missingRequiredFieldError(k) + } + } + + // Field lengths + if err := verifyRequestIDLength(req, rv.config); err != nil { + return err + } + if err := verifyCallbackIDLength(req, rv.config); err != nil { + return err + } + + // Validate the callback to be invoked and its parameters. + if err := rv.cbValidator.Validate(req.GetNamespace(), []*commonpb.Callback{req.Callback}); err != nil { + return err + } + + // ScheduleToCloseTimeout + if req.GetScheduleToCloseTimeout() == nil || req.GetScheduleToCloseTimeout().AsDuration() <= 0 { + return serviceerror.NewInvalidArgument("ScheduleToCloseTimeout must be set and positive.") + } + + // Validate the input data to deliver to the callback URL, currently only one kind is supported (Completion). + completion := req.GetCompletion() + if completion == nil { + return serviceerror.NewInvalidArgument("Completion is not set on request.") + } + if completion.GetSuccess() == nil && completion.GetFailure() == nil { + return serviceerror.NewInvalidArgument("Completion must have either success or failure set.") + } + if completion.GetSuccess() != nil && completion.GetFailure() != nil { + return serviceerror.NewInvalidArgument("Completion must have exactly one of success or failure set, not both.") + } + // Validate the size of the completion is reasonable. + if err := rv.validateCompletionSize(req, completion); err != nil { + return err + } + + // Search Attributes + if searchAttrib := req.GetSearchAttributes(); searchAttrib != nil { + if err := rv.validateSearchAttributes(req, searchAttrib); err != nil { + return err + } + } + + return nil +} + +func (rv *frontendRequestValidator) validateCompletionSize(req Namespacer, completion *callbackpb.CallbackExecutionCompletion) error { + namespace := req.GetNamespace() + + sizeWarnLimit := rv.config.BlobSizeLimitWarn(namespace) + sizeErrorLimit := rv.config.BlobSizeLimitError(namespace) + + blobSize := proto.Size(completion) + if blobSize > sizeWarnLimit { + rv.logger.Warn("Completion blob size exceeds the warning limit.", + tag.WorkflowNamespace(namespace), + tag.BlobSize(int64(blobSize))) + } + + if blobSize > sizeErrorLimit { + return common.ErrBlobSizeExceedsLimit + } + + return nil +} + +func (rv *frontendRequestValidator) validateSearchAttributes(req Namespacer, saToValidate *commonpb.SearchAttributes) error { + namespaceName := req.GetNamespace() + + // Unalias search attributes for validation. + if rv.saMapperProvider != nil && saToValidate != nil { + var err error + saToValidate, err = searchattribute.UnaliasFields(rv.saMapperProvider, saToValidate, namespaceName) + if err != nil { + return err + } + } + + if err := rv.saValidator.Validate(saToValidate, namespaceName); err != nil { + return err + } + + return rv.saValidator.ValidateSize(saToValidate, namespaceName) +} + +func (rv *frontendRequestValidator) ValidateDescribeCallbackExecution(req *workflowservice.DescribeCallbackExecutionRequest) error { + // Required fields. + requiredFields := map[string]string{ + "Namespace": req.GetNamespace(), + "CallbackId": req.GetCallbackId(), + } + for k, v := range requiredFields { + if v == "" { + return missingRequiredFieldError(k) + } + } + + // Field lengths + if err := verifyCallbackIDLength(req, rv.config); err != nil { + return err + } + + // A long-poll token requires the RunID be set. + if len(req.GetLongPollToken()) > 0 && req.GetRunId() == "" { + return serviceerror.NewInvalidArgument("RunID is required when LongPollToken is provided") + } + + return nil +} + +func (rv *frontendRequestValidator) ValidatePollCallbackExecution(req *workflowservice.PollCallbackExecutionRequest) error { + // Required fields. + requiredFields := map[string]string{ + "Namespace": req.GetNamespace(), + "CallbackId": req.GetCallbackId(), + } + for k, v := range requiredFields { + if v == "" { + return missingRequiredFieldError(k) + } + } + + // Field lengths + return verifyCallbackIDLength(req, rv.config) +} + +func (rv *frontendRequestValidator) ValidateTerminateCallbackExecution(req *workflowservice.TerminateCallbackExecutionRequest) error { + // Set RequestID if missing. + if req.GetRequestId() == "" { + req.RequestId = uuid.NewString() + } + + // Required fields. + requiredFields := map[string]string{ + "RequestId": req.GetRequestId(), + "Namespace": req.GetNamespace(), + "CallbackId": req.GetCallbackId(), + + // NOTE: We don't require the Identity or Reason fields to be set, + // and just set reasonable defaults. + } + for k, v := range requiredFields { + if v == "" { + return missingRequiredFieldError(k) + } + } + + // Field lengths + if err := verifyRequestIDLength(req, rv.config); err != nil { + return err + } + return verifyCallbackIDLength(req, rv.config) +} + +func (rv *frontendRequestValidator) ValidateDeleteCallbackExecution(req *workflowservice.DeleteCallbackExecutionRequest) error { + // Required fields. + requiredFields := map[string]string{ + "Namespace": req.GetNamespace(), + "CallbackId": req.GetCallbackId(), + } + for k, v := range requiredFields { + if v == "" { + return missingRequiredFieldError(k) + } + } + + // Field lengths + return verifyCallbackIDLength(req, rv.config) +} + +func (rv *frontendRequestValidator) ValidateListCallbackExecutions(req *workflowservice.ListCallbackExecutionsRequest) error { + if req.GetNamespace() == "" { + return missingRequiredFieldError("Namespace") + } + return nil +} + +func (rv *frontendRequestValidator) ValidateCountCallbackExecutions(req *workflowservice.CountCallbackExecutionsRequest) error { + if req.GetNamespace() == "" { + return missingRequiredFieldError("Namespace") + } + return nil +} diff --git a/chasm/lib/callback/fx.go b/chasm/lib/callback/fx.go index 1da518d8368..3d4401db7f3 100644 --- a/chasm/lib/callback/fx.go +++ b/chasm/lib/callback/fx.go @@ -5,6 +5,7 @@ import ( "net/http" "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/collection" @@ -15,13 +16,6 @@ import ( "go.uber.org/fx" ) -func register( - registry *chasm.Registry, - library *Library, -) error { - return registry.Register(library) -} - // httpCallerProviderProvider provides an HTTPCallerProvider for CHASM callbacks. func httpCallerProviderProvider( clusterMetadata cluster.Metadata, @@ -53,12 +47,30 @@ func httpCallerProviderProvider( return m.Get, nil } -var Module = fx.Module( +// FrontendModule just contains the CHASM components, but not their implementation. +var FrontendModule = fx.Module( + "callback-frontend", + fx.Provide(callbackspb.NewCallbackServiceLayeredClient), + fx.Provide(ConfigProvider), + fx.Provide(NewFrontendHandler), + + fx.Provide(newComponentOnlyLibrary), + fx.Invoke(func(registry *chasm.Registry, coLibrary *componentOnlyLibrary) error { + return registry.Register(coLibrary) + }), +) + +var HistoryModule = fx.Module( "chasm.lib.callback", - fx.Provide(configProvider), + fx.Provide(ConfigProvider), fx.Provide(httpCallerProviderProvider), fx.Provide(newInvocationTaskHandler), fx.Provide(newBackoffTaskHandler), + fx.Provide(newCallbackHandler), + fx.Provide(NewCompletionScheduleToCloseTimeoutTaskHandler), + fx.Provide(newLibrary), - fx.Invoke(register), + fx.Invoke(func(registry *chasm.Registry, library *library) error { + return registry.Register(library) + }), ) diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go index 4e8000266ae..c9900bb64b0 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go @@ -89,6 +89,7 @@ var ( "BackingOff": 3, "Failed": 4, "Succeeded": 5, + "Terminated": 6, } ) diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index d998ef3fc8f..6ae4eed6fd5 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -16,6 +16,7 @@ import ( v1 "go.temporal.io/api/failure/v1" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" timestamppb "google.golang.org/protobuf/types/known/timestamppb" ) @@ -42,6 +43,8 @@ const ( CALLBACK_STATUS_FAILED CallbackStatus = 4 // Callback has succeeded. CALLBACK_STATUS_SUCCEEDED CallbackStatus = 5 + // Callback was terminated by request. + CALLBACK_STATUS_TERMINATED CallbackStatus = 6 ) // Enum value maps for CallbackStatus. @@ -53,6 +56,7 @@ var ( 3: "CALLBACK_STATUS_BACKING_OFF", 4: "CALLBACK_STATUS_FAILED", 5: "CALLBACK_STATUS_SUCCEEDED", + 6: "CALLBACK_STATUS_TERMINATED", } CallbackStatus_value = map[string]int32{ "CALLBACK_STATUS_UNSPECIFIED": 0, @@ -61,6 +65,7 @@ var ( "CALLBACK_STATUS_BACKING_OFF": 3, "CALLBACK_STATUS_FAILED": 4, "CALLBACK_STATUS_SUCCEEDED": 5, + "CALLBACK_STATUS_TERMINATED": 6, } ) @@ -84,6 +89,8 @@ func (x CallbackStatus) String() string { return "Failed" case CALLBACK_STATUS_SUCCEEDED: return "Succeeded" + case CALLBACK_STATUS_TERMINATED: + return "Terminated" default: return strconv.Itoa(int(x)) } @@ -126,9 +133,19 @@ type CallbackState struct { // https://github.com/temporalio/temporal/pull/8473#discussion_r2427348436 NextAttemptScheduleTime *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=next_attempt_schedule_time,json=nextAttemptScheduleTime,proto3" json:"next_attempt_schedule_time,omitempty"` // Request ID that added the callback. - RequestId string `protobuf:"bytes,9,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + RequestId string `protobuf:"bytes,9,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + // Request ID that terminated the callback, if applicable. Used for idempotency. + TerminateRequestId string `protobuf:"bytes,10,opt,name=terminate_request_id,json=terminateRequestId,proto3" json:"terminate_request_id,omitempty"` + // The time when the callback reached a terminal state. + CloseTime *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=close_time,json=closeTime,proto3" json:"close_time,omitempty"` + // (standalone only) User-supplied business ID set when StartCallbackExecution() is + // called. Used to identify the callback for operations like Describe- or Terminate-. + CallbackId string `protobuf:"bytes,12,opt,name=callback_id,json=callbackId,proto3" json:"callback_id,omitempty"` + // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() + // is called to when the result gets delivered. + CompletionScheduleToCloseTimeout *durationpb.Duration `protobuf:"bytes,13,opt,name=completion_schedule_to_close_timeout,json=completionScheduleToCloseTimeout,proto3" json:"completion_schedule_to_close_timeout,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CallbackState) Reset() { @@ -217,6 +234,34 @@ func (x *CallbackState) GetRequestId() string { return "" } +func (x *CallbackState) GetTerminateRequestId() string { + if x != nil { + return x.TerminateRequestId + } + return "" +} + +func (x *CallbackState) GetCloseTime() *timestamppb.Timestamp { + if x != nil { + return x.CloseTime + } + return nil +} + +func (x *CallbackState) GetCallbackId() string { + if x != nil { + return x.CallbackId + } + return "" +} + +func (x *CallbackState) GetCompletionScheduleToCloseTimeout() *durationpb.Duration { + if x != nil { + return x.CompletionScheduleToCloseTimeout + } + return nil +} + type Callback struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Variant: @@ -336,7 +381,9 @@ type Callback_Nexus struct { // aip.dev/not-precedent: Not respecting aip here. --) Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // Header to attach to callback request. - Header map[string]string `protobuf:"bytes,2,rep,name=header,proto3" json:"header,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Header map[string]string `protobuf:"bytes,2,rep,name=header,proto3" json:"header,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Token identifying the target callback to resolve. + Token string `protobuf:"bytes,3,opt,name=token,proto3" json:"token,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -385,11 +432,18 @@ func (x *Callback_Nexus) GetHeader() map[string]string { return nil } +func (x *Callback_Nexus) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + var File_temporal_server_chasm_lib_callback_proto_v1_message_proto protoreflect.FileDescriptor const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = "" + "\n" + - "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\xd3\x04\n" + + "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\xcc\x06\n" + "\rCallbackState\x12R\n" + "\bcallback\x18\x01 \x01(\v26.temporal.server.chasm.lib.callbacks.proto.v1.CallbackR\bcallback\x12G\n" + "\x11registration_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x10registrationTime\x12T\n" + @@ -399,25 +453,34 @@ const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = " "\x14last_attempt_failure\x18\a \x01(\v2 .temporal.api.failure.v1.FailureR\x12lastAttemptFailure\x12W\n" + "\x1anext_attempt_schedule_time\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\x17nextAttemptScheduleTime\x12\x1d\n" + "\n" + - "request_id\x18\t \x01(\tR\trequestId\x1a\x10\n" + - "\x0eWorkflowClosed\"\xde\x02\n" + + "request_id\x18\t \x01(\tR\trequestId\x120\n" + + "\x14terminate_request_id\x18\n" + + " \x01(\tR\x12terminateRequestId\x129\n" + + "\n" + + "close_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcloseTime\x12\x1f\n" + + "\vcallback_id\x18\f \x01(\tR\n" + + "callbackId\x12i\n" + + "$completion_schedule_to_close_timeout\x18\r \x01(\v2\x19.google.protobuf.DurationR completionScheduleToCloseTimeout\x1a\x10\n" + + "\x0eWorkflowClosed\"\xf4\x02\n" + "\bCallback\x12T\n" + "\x05nexus\x18\x02 \x01(\v2<.temporal.server.chasm.lib.callbacks.proto.v1.Callback.NexusH\x00R\x05nexus\x122\n" + - "\x05links\x18d \x03(\v2\x1c.temporal.api.common.v1.LinkR\x05links\x1a\xb6\x01\n" + + "\x05links\x18d \x03(\v2\x1c.temporal.api.common.v1.LinkR\x05links\x1a\xcc\x01\n" + "\x05Nexus\x12\x10\n" + "\x03url\x18\x01 \x01(\tR\x03url\x12`\n" + - "\x06header\x18\x02 \x03(\v2H.temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntryR\x06header\x1a9\n" + + "\x06header\x18\x02 \x03(\v2H.temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntryR\x06header\x12\x14\n" + + "\x05token\x18\x03 \x01(\tR\x05token\x1a9\n" + "\vHeaderEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\t\n" + - "\avariantJ\x04\b\x01\x10\x02*\xc9\x01\n" + + "\avariantJ\x04\b\x01\x10\x02*\xe9\x01\n" + "\x0eCallbackStatus\x12\x1f\n" + "\x1bCALLBACK_STATUS_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17CALLBACK_STATUS_STANDBY\x10\x01\x12\x1d\n" + "\x19CALLBACK_STATUS_SCHEDULED\x10\x02\x12\x1f\n" + "\x1bCALLBACK_STATUS_BACKING_OFF\x10\x03\x12\x1a\n" + "\x16CALLBACK_STATUS_FAILED\x10\x04\x12\x1d\n" + - "\x19CALLBACK_STATUS_SUCCEEDED\x10\x05BGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" + "\x19CALLBACK_STATUS_SUCCEEDED\x10\x05\x12\x1e\n" + + "\x1aCALLBACK_STATUS_TERMINATED\x10\x06BGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" var ( file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDescOnce sync.Once @@ -442,23 +505,26 @@ var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_goTypes = []a nil, // 5: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp (*v1.Failure)(nil), // 7: temporal.api.failure.v1.Failure - (*v11.Link)(nil), // 8: temporal.api.common.v1.Link + (*durationpb.Duration)(nil), // 8: google.protobuf.Duration + (*v11.Link)(nil), // 9: temporal.api.common.v1.Link } var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_depIdxs = []int32{ - 2, // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.callback:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback - 6, // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.registration_time:type_name -> google.protobuf.Timestamp - 0, // 2: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.status:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus - 6, // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 7, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure - 6, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp - 4, // 6: temporal.server.chasm.lib.callbacks.proto.v1.Callback.nexus:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus - 8, // 7: temporal.server.chasm.lib.callbacks.proto.v1.Callback.links:type_name -> temporal.api.common.v1.Link - 5, // 8: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.header:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry - 9, // [9:9] is the sub-list for method output_type - 9, // [9:9] is the sub-list for method input_type - 9, // [9:9] is the sub-list for extension type_name - 9, // [9:9] is the sub-list for extension extendee - 0, // [0:9] is the sub-list for field type_name + 2, // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.callback:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback + 6, // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.registration_time:type_name -> google.protobuf.Timestamp + 0, // 2: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.status:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus + 6, // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 7, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure + 6, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp + 6, // 6: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.close_time:type_name -> google.protobuf.Timestamp + 8, // 7: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.completion_schedule_to_close_timeout:type_name -> google.protobuf.Duration + 4, // 8: temporal.server.chasm.lib.callbacks.proto.v1.Callback.nexus:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus + 9, // 9: temporal.server.chasm.lib.callbacks.proto.v1.Callback.links:type_name -> temporal.api.common.v1.Link + 5, // 10: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.header:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name } func init() { file_temporal_server_chasm_lib_callback_proto_v1_message_proto_init() } diff --git a/chasm/lib/callback/gen/callbackpb/v1/request_response.go-helpers.pb.go b/chasm/lib/callback/gen/callbackpb/v1/request_response.go-helpers.pb.go new file mode 100644 index 00000000000..aae605a5967 --- /dev/null +++ b/chasm/lib/callback/gen/callbackpb/v1/request_response.go-helpers.pb.go @@ -0,0 +1,376 @@ +// Code generated by protoc-gen-go-helpers. DO NOT EDIT. +package callbackspb + +import ( + "google.golang.org/protobuf/proto" +) + +// Marshal an object of type StartCallbackExecutionRequest to the protobuf v3 wire format +func (val *StartCallbackExecutionRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StartCallbackExecutionRequest from the protobuf v3 wire format +func (val *StartCallbackExecutionRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StartCallbackExecutionRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StartCallbackExecutionRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StartCallbackExecutionRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StartCallbackExecutionRequest + switch t := that.(type) { + case *StartCallbackExecutionRequest: + that1 = t + case StartCallbackExecutionRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type StartCallbackExecutionResponse to the protobuf v3 wire format +func (val *StartCallbackExecutionResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type StartCallbackExecutionResponse from the protobuf v3 wire format +func (val *StartCallbackExecutionResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *StartCallbackExecutionResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two StartCallbackExecutionResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *StartCallbackExecutionResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *StartCallbackExecutionResponse + switch t := that.(type) { + case *StartCallbackExecutionResponse: + that1 = t + case StartCallbackExecutionResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeCallbackExecutionRequest to the protobuf v3 wire format +func (val *DescribeCallbackExecutionRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeCallbackExecutionRequest from the protobuf v3 wire format +func (val *DescribeCallbackExecutionRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeCallbackExecutionRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeCallbackExecutionRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeCallbackExecutionRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeCallbackExecutionRequest + switch t := that.(type) { + case *DescribeCallbackExecutionRequest: + that1 = t + case DescribeCallbackExecutionRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DescribeCallbackExecutionResponse to the protobuf v3 wire format +func (val *DescribeCallbackExecutionResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DescribeCallbackExecutionResponse from the protobuf v3 wire format +func (val *DescribeCallbackExecutionResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DescribeCallbackExecutionResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DescribeCallbackExecutionResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DescribeCallbackExecutionResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DescribeCallbackExecutionResponse + switch t := that.(type) { + case *DescribeCallbackExecutionResponse: + that1 = t + case DescribeCallbackExecutionResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollCallbackExecutionRequest to the protobuf v3 wire format +func (val *PollCallbackExecutionRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollCallbackExecutionRequest from the protobuf v3 wire format +func (val *PollCallbackExecutionRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollCallbackExecutionRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollCallbackExecutionRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollCallbackExecutionRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollCallbackExecutionRequest + switch t := that.(type) { + case *PollCallbackExecutionRequest: + that1 = t + case PollCallbackExecutionRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type PollCallbackExecutionResponse to the protobuf v3 wire format +func (val *PollCallbackExecutionResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type PollCallbackExecutionResponse from the protobuf v3 wire format +func (val *PollCallbackExecutionResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *PollCallbackExecutionResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two PollCallbackExecutionResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *PollCallbackExecutionResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *PollCallbackExecutionResponse + switch t := that.(type) { + case *PollCallbackExecutionResponse: + that1 = t + case PollCallbackExecutionResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type TerminateCallbackExecutionRequest to the protobuf v3 wire format +func (val *TerminateCallbackExecutionRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type TerminateCallbackExecutionRequest from the protobuf v3 wire format +func (val *TerminateCallbackExecutionRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *TerminateCallbackExecutionRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two TerminateCallbackExecutionRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *TerminateCallbackExecutionRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *TerminateCallbackExecutionRequest + switch t := that.(type) { + case *TerminateCallbackExecutionRequest: + that1 = t + case TerminateCallbackExecutionRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type TerminateCallbackExecutionResponse to the protobuf v3 wire format +func (val *TerminateCallbackExecutionResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type TerminateCallbackExecutionResponse from the protobuf v3 wire format +func (val *TerminateCallbackExecutionResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *TerminateCallbackExecutionResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two TerminateCallbackExecutionResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *TerminateCallbackExecutionResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *TerminateCallbackExecutionResponse + switch t := that.(type) { + case *TerminateCallbackExecutionResponse: + that1 = t + case TerminateCallbackExecutionResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DeleteCallbackExecutionRequest to the protobuf v3 wire format +func (val *DeleteCallbackExecutionRequest) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DeleteCallbackExecutionRequest from the protobuf v3 wire format +func (val *DeleteCallbackExecutionRequest) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DeleteCallbackExecutionRequest) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DeleteCallbackExecutionRequest values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DeleteCallbackExecutionRequest) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DeleteCallbackExecutionRequest + switch t := that.(type) { + case *DeleteCallbackExecutionRequest: + that1 = t + case DeleteCallbackExecutionRequest: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} + +// Marshal an object of type DeleteCallbackExecutionResponse to the protobuf v3 wire format +func (val *DeleteCallbackExecutionResponse) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type DeleteCallbackExecutionResponse from the protobuf v3 wire format +func (val *DeleteCallbackExecutionResponse) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *DeleteCallbackExecutionResponse) Size() int { + return proto.Size(val) +} + +// Equal returns whether two DeleteCallbackExecutionResponse values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *DeleteCallbackExecutionResponse) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *DeleteCallbackExecutionResponse + switch t := that.(type) { + case *DeleteCallbackExecutionResponse: + that1 = t + case DeleteCallbackExecutionResponse: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/chasm/lib/callback/gen/callbackpb/v1/request_response.pb.go b/chasm/lib/callback/gen/callbackpb/v1/request_response.pb.go new file mode 100644 index 00000000000..3af5bf8dc16 --- /dev/null +++ b/chasm/lib/callback/gen/callbackpb/v1/request_response.pb.go @@ -0,0 +1,617 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/callback/proto/v1/request_response.proto + +package callbackspb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + v1 "go.temporal.io/api/workflowservice/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type StartCallbackExecutionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Internal namespace ID (UUID). + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *v1.StartCallbackExecutionRequest `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartCallbackExecutionRequest) Reset() { + *x = StartCallbackExecutionRequest{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartCallbackExecutionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartCallbackExecutionRequest) ProtoMessage() {} + +func (x *StartCallbackExecutionRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartCallbackExecutionRequest.ProtoReflect.Descriptor instead. +func (*StartCallbackExecutionRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{0} +} + +func (x *StartCallbackExecutionRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *StartCallbackExecutionRequest) GetFrontendRequest() *v1.StartCallbackExecutionRequest { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type StartCallbackExecutionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *v1.StartCallbackExecutionResponse `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartCallbackExecutionResponse) Reset() { + *x = StartCallbackExecutionResponse{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartCallbackExecutionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartCallbackExecutionResponse) ProtoMessage() {} + +func (x *StartCallbackExecutionResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StartCallbackExecutionResponse.ProtoReflect.Descriptor instead. +func (*StartCallbackExecutionResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{1} +} + +func (x *StartCallbackExecutionResponse) GetFrontendResponse() *v1.StartCallbackExecutionResponse { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type DescribeCallbackExecutionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Internal namespace ID (UUID). + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *v1.DescribeCallbackExecutionRequest `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeCallbackExecutionRequest) Reset() { + *x = DescribeCallbackExecutionRequest{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeCallbackExecutionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeCallbackExecutionRequest) ProtoMessage() {} + +func (x *DescribeCallbackExecutionRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeCallbackExecutionRequest.ProtoReflect.Descriptor instead. +func (*DescribeCallbackExecutionRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{2} +} + +func (x *DescribeCallbackExecutionRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *DescribeCallbackExecutionRequest) GetFrontendRequest() *v1.DescribeCallbackExecutionRequest { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type DescribeCallbackExecutionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *v1.DescribeCallbackExecutionResponse `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DescribeCallbackExecutionResponse) Reset() { + *x = DescribeCallbackExecutionResponse{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DescribeCallbackExecutionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescribeCallbackExecutionResponse) ProtoMessage() {} + +func (x *DescribeCallbackExecutionResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescribeCallbackExecutionResponse.ProtoReflect.Descriptor instead. +func (*DescribeCallbackExecutionResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{3} +} + +func (x *DescribeCallbackExecutionResponse) GetFrontendResponse() *v1.DescribeCallbackExecutionResponse { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type PollCallbackExecutionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Internal namespace ID (UUID). + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *v1.PollCallbackExecutionRequest `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollCallbackExecutionRequest) Reset() { + *x = PollCallbackExecutionRequest{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollCallbackExecutionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollCallbackExecutionRequest) ProtoMessage() {} + +func (x *PollCallbackExecutionRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollCallbackExecutionRequest.ProtoReflect.Descriptor instead. +func (*PollCallbackExecutionRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{4} +} + +func (x *PollCallbackExecutionRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *PollCallbackExecutionRequest) GetFrontendRequest() *v1.PollCallbackExecutionRequest { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type PollCallbackExecutionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *v1.PollCallbackExecutionResponse `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PollCallbackExecutionResponse) Reset() { + *x = PollCallbackExecutionResponse{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PollCallbackExecutionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PollCallbackExecutionResponse) ProtoMessage() {} + +func (x *PollCallbackExecutionResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PollCallbackExecutionResponse.ProtoReflect.Descriptor instead. +func (*PollCallbackExecutionResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{5} +} + +func (x *PollCallbackExecutionResponse) GetFrontendResponse() *v1.PollCallbackExecutionResponse { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type TerminateCallbackExecutionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Internal namespace ID (UUID). + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *v1.TerminateCallbackExecutionRequest `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateCallbackExecutionRequest) Reset() { + *x = TerminateCallbackExecutionRequest{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateCallbackExecutionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateCallbackExecutionRequest) ProtoMessage() {} + +func (x *TerminateCallbackExecutionRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateCallbackExecutionRequest.ProtoReflect.Descriptor instead. +func (*TerminateCallbackExecutionRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{6} +} + +func (x *TerminateCallbackExecutionRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *TerminateCallbackExecutionRequest) GetFrontendRequest() *v1.TerminateCallbackExecutionRequest { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type TerminateCallbackExecutionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *v1.TerminateCallbackExecutionResponse `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateCallbackExecutionResponse) Reset() { + *x = TerminateCallbackExecutionResponse{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateCallbackExecutionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateCallbackExecutionResponse) ProtoMessage() {} + +func (x *TerminateCallbackExecutionResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateCallbackExecutionResponse.ProtoReflect.Descriptor instead. +func (*TerminateCallbackExecutionResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{7} +} + +func (x *TerminateCallbackExecutionResponse) GetFrontendResponse() *v1.TerminateCallbackExecutionResponse { + if x != nil { + return x.FrontendResponse + } + return nil +} + +type DeleteCallbackExecutionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Internal namespace ID (UUID). + NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + FrontendRequest *v1.DeleteCallbackExecutionRequest `protobuf:"bytes,2,opt,name=frontend_request,json=frontendRequest,proto3" json:"frontend_request,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteCallbackExecutionRequest) Reset() { + *x = DeleteCallbackExecutionRequest{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteCallbackExecutionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteCallbackExecutionRequest) ProtoMessage() {} + +func (x *DeleteCallbackExecutionRequest) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteCallbackExecutionRequest.ProtoReflect.Descriptor instead. +func (*DeleteCallbackExecutionRequest) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{8} +} + +func (x *DeleteCallbackExecutionRequest) GetNamespaceId() string { + if x != nil { + return x.NamespaceId + } + return "" +} + +func (x *DeleteCallbackExecutionRequest) GetFrontendRequest() *v1.DeleteCallbackExecutionRequest { + if x != nil { + return x.FrontendRequest + } + return nil +} + +type DeleteCallbackExecutionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + FrontendResponse *v1.DeleteCallbackExecutionResponse `protobuf:"bytes,1,opt,name=frontend_response,json=frontendResponse,proto3" json:"frontend_response,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteCallbackExecutionResponse) Reset() { + *x = DeleteCallbackExecutionResponse{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteCallbackExecutionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteCallbackExecutionResponse) ProtoMessage() {} + +func (x *DeleteCallbackExecutionResponse) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteCallbackExecutionResponse.ProtoReflect.Descriptor instead. +func (*DeleteCallbackExecutionResponse) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP(), []int{9} +} + +func (x *DeleteCallbackExecutionResponse) GetFrontendResponse() *v1.DeleteCallbackExecutionResponse { + if x != nil { + return x.FrontendResponse + } + return nil +} + +var File_temporal_server_chasm_lib_callback_proto_v1_request_response_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDesc = "" + + "\n" + + "Btemporal/server/chasm/lib/callback/proto/v1/request_response.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a6temporal/api/workflowservice/v1/request_response.proto\"\xad\x01\n" + + "\x1dStartCallbackExecutionRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12i\n" + + "\x10frontend_request\x18\x02 \x01(\v2>.temporal.api.workflowservice.v1.StartCallbackExecutionRequestR\x0ffrontendRequest\"\x8e\x01\n" + + "\x1eStartCallbackExecutionResponse\x12l\n" + + "\x11frontend_response\x18\x01 \x01(\v2?.temporal.api.workflowservice.v1.StartCallbackExecutionResponseR\x10frontendResponse\"\xb3\x01\n" + + " DescribeCallbackExecutionRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12l\n" + + "\x10frontend_request\x18\x02 \x01(\v2A.temporal.api.workflowservice.v1.DescribeCallbackExecutionRequestR\x0ffrontendRequest\"\x94\x01\n" + + "!DescribeCallbackExecutionResponse\x12o\n" + + "\x11frontend_response\x18\x01 \x01(\v2B.temporal.api.workflowservice.v1.DescribeCallbackExecutionResponseR\x10frontendResponse\"\xab\x01\n" + + "\x1cPollCallbackExecutionRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12h\n" + + "\x10frontend_request\x18\x02 \x01(\v2=.temporal.api.workflowservice.v1.PollCallbackExecutionRequestR\x0ffrontendRequest\"\x8c\x01\n" + + "\x1dPollCallbackExecutionResponse\x12k\n" + + "\x11frontend_response\x18\x01 \x01(\v2>.temporal.api.workflowservice.v1.PollCallbackExecutionResponseR\x10frontendResponse\"\xb5\x01\n" + + "!TerminateCallbackExecutionRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12m\n" + + "\x10frontend_request\x18\x02 \x01(\v2B.temporal.api.workflowservice.v1.TerminateCallbackExecutionRequestR\x0ffrontendRequest\"\x96\x01\n" + + "\"TerminateCallbackExecutionResponse\x12p\n" + + "\x11frontend_response\x18\x01 \x01(\v2C.temporal.api.workflowservice.v1.TerminateCallbackExecutionResponseR\x10frontendResponse\"\xaf\x01\n" + + "\x1eDeleteCallbackExecutionRequest\x12!\n" + + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12j\n" + + "\x10frontend_request\x18\x02 \x01(\v2?.temporal.api.workflowservice.v1.DeleteCallbackExecutionRequestR\x0ffrontendRequest\"\x90\x01\n" + + "\x1fDeleteCallbackExecutionResponse\x12m\n" + + "\x11frontend_response\x18\x01 \x01(\v2@.temporal.api.workflowservice.v1.DeleteCallbackExecutionResponseR\x10frontendResponseBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" + +var ( + file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescOnce sync.Once + file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescData []byte +) + +func file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescGZIP() []byte { + file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescOnce.Do(func() { + file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDesc), len(file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDesc))) + }) + return file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDescData +} + +var file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_goTypes = []any{ + (*StartCallbackExecutionRequest)(nil), // 0: temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionRequest + (*StartCallbackExecutionResponse)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionResponse + (*DescribeCallbackExecutionRequest)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionRequest + (*DescribeCallbackExecutionResponse)(nil), // 3: temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionResponse + (*PollCallbackExecutionRequest)(nil), // 4: temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionRequest + (*PollCallbackExecutionResponse)(nil), // 5: temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionResponse + (*TerminateCallbackExecutionRequest)(nil), // 6: temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionRequest + (*TerminateCallbackExecutionResponse)(nil), // 7: temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionResponse + (*DeleteCallbackExecutionRequest)(nil), // 8: temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionRequest + (*DeleteCallbackExecutionResponse)(nil), // 9: temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionResponse + (*v1.StartCallbackExecutionRequest)(nil), // 10: temporal.api.workflowservice.v1.StartCallbackExecutionRequest + (*v1.StartCallbackExecutionResponse)(nil), // 11: temporal.api.workflowservice.v1.StartCallbackExecutionResponse + (*v1.DescribeCallbackExecutionRequest)(nil), // 12: temporal.api.workflowservice.v1.DescribeCallbackExecutionRequest + (*v1.DescribeCallbackExecutionResponse)(nil), // 13: temporal.api.workflowservice.v1.DescribeCallbackExecutionResponse + (*v1.PollCallbackExecutionRequest)(nil), // 14: temporal.api.workflowservice.v1.PollCallbackExecutionRequest + (*v1.PollCallbackExecutionResponse)(nil), // 15: temporal.api.workflowservice.v1.PollCallbackExecutionResponse + (*v1.TerminateCallbackExecutionRequest)(nil), // 16: temporal.api.workflowservice.v1.TerminateCallbackExecutionRequest + (*v1.TerminateCallbackExecutionResponse)(nil), // 17: temporal.api.workflowservice.v1.TerminateCallbackExecutionResponse + (*v1.DeleteCallbackExecutionRequest)(nil), // 18: temporal.api.workflowservice.v1.DeleteCallbackExecutionRequest + (*v1.DeleteCallbackExecutionResponse)(nil), // 19: temporal.api.workflowservice.v1.DeleteCallbackExecutionResponse +} +var file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_depIdxs = []int32{ + 10, // 0: temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.StartCallbackExecutionRequest + 11, // 1: temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionResponse.frontend_response:type_name -> temporal.api.workflowservice.v1.StartCallbackExecutionResponse + 12, // 2: temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.DescribeCallbackExecutionRequest + 13, // 3: temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionResponse.frontend_response:type_name -> temporal.api.workflowservice.v1.DescribeCallbackExecutionResponse + 14, // 4: temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.PollCallbackExecutionRequest + 15, // 5: temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionResponse.frontend_response:type_name -> temporal.api.workflowservice.v1.PollCallbackExecutionResponse + 16, // 6: temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.TerminateCallbackExecutionRequest + 17, // 7: temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionResponse.frontend_response:type_name -> temporal.api.workflowservice.v1.TerminateCallbackExecutionResponse + 18, // 8: temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.DeleteCallbackExecutionRequest + 19, // 9: temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionResponse.frontend_response:type_name -> temporal.api.workflowservice.v1.DeleteCallbackExecutionResponse + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name +} + +func init() { file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_init() } +func file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_init() { + if File_temporal_server_chasm_lib_callback_proto_v1_request_response_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDesc), len(file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_rawDesc)), + NumEnums: 0, + NumMessages: 10, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_goTypes, + DependencyIndexes: file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_depIdxs, + MessageInfos: file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_msgTypes, + }.Build() + File_temporal_server_chasm_lib_callback_proto_v1_request_response_proto = out.File + file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_goTypes = nil + file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_depIdxs = nil +} diff --git a/chasm/lib/callback/gen/callbackpb/v1/service.pb.go b/chasm/lib/callback/gen/callbackpb/v1/service.pb.go new file mode 100644 index 00000000000..2abe4351309 --- /dev/null +++ b/chasm/lib/callback/gen/callbackpb/v1/service.pb.go @@ -0,0 +1,90 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// plugins: +// protoc-gen-go +// protoc +// source: temporal/server/chasm/lib/callback/proto/v1/service.proto + +package callbackspb + +import ( + reflect "reflect" + unsafe "unsafe" + + _ "go.temporal.io/server/api/common/v1" + _ "go.temporal.io/server/api/routing/v1" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_temporal_server_chasm_lib_callback_proto_v1_service_proto protoreflect.FileDescriptor + +const file_temporal_server_chasm_lib_callback_proto_v1_service_proto_rawDesc = "" + + "\n" + + "9temporal/server/chasm/lib/callback/proto/v1/service.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1aBtemporal/server/chasm/lib/callback/proto/v1/request_response.proto\x1a0temporal/server/api/common/v1/api_category.proto\x1a.temporal/server/api/routing/v1/extension.proto2\x86\t\n" + + "\x0fCallbackService\x12\xdd\x01\n" + + "\x16StartCallbackExecution\x12K.temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionRequest\x1aL.temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xe6\x01\n" + + "\x19DescribeCallbackExecution\x12N.temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionRequest\x1aO.temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xda\x01\n" + + "\x15PollCallbackExecution\x12J.temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionRequest\x1aK.temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x02\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xe9\x01\n" + + "\x1aTerminateCallbackExecution\x12O.temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionRequest\x1aP.temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xe0\x01\n" + + "\x17DeleteCallbackExecution\x12L.temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionRequest\x1aM.temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_idBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" + +var file_temporal_server_chasm_lib_callback_proto_v1_service_proto_goTypes = []any{ + (*StartCallbackExecutionRequest)(nil), // 0: temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionRequest + (*DescribeCallbackExecutionRequest)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionRequest + (*PollCallbackExecutionRequest)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionRequest + (*TerminateCallbackExecutionRequest)(nil), // 3: temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionRequest + (*DeleteCallbackExecutionRequest)(nil), // 4: temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionRequest + (*StartCallbackExecutionResponse)(nil), // 5: temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionResponse + (*DescribeCallbackExecutionResponse)(nil), // 6: temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionResponse + (*PollCallbackExecutionResponse)(nil), // 7: temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionResponse + (*TerminateCallbackExecutionResponse)(nil), // 8: temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionResponse + (*DeleteCallbackExecutionResponse)(nil), // 9: temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionResponse +} +var file_temporal_server_chasm_lib_callback_proto_v1_service_proto_depIdxs = []int32{ + 0, // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.StartCallbackExecution:input_type -> temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionRequest + 1, // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.DescribeCallbackExecution:input_type -> temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionRequest + 2, // 2: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.PollCallbackExecution:input_type -> temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionRequest + 3, // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.TerminateCallbackExecution:input_type -> temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionRequest + 4, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.DeleteCallbackExecution:input_type -> temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionRequest + 5, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.StartCallbackExecution:output_type -> temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionResponse + 6, // 6: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.DescribeCallbackExecution:output_type -> temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionResponse + 7, // 7: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.PollCallbackExecution:output_type -> temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionResponse + 8, // 8: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.TerminateCallbackExecution:output_type -> temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionResponse + 9, // 9: temporal.server.chasm.lib.callbacks.proto.v1.CallbackService.DeleteCallbackExecution:output_type -> temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionResponse + 5, // [5:10] is the sub-list for method output_type + 0, // [0:5] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_temporal_server_chasm_lib_callback_proto_v1_service_proto_init() } +func file_temporal_server_chasm_lib_callback_proto_v1_service_proto_init() { + if File_temporal_server_chasm_lib_callback_proto_v1_service_proto != nil { + return + } + file_temporal_server_chasm_lib_callback_proto_v1_request_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_callback_proto_v1_service_proto_rawDesc), len(file_temporal_server_chasm_lib_callback_proto_v1_service_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_temporal_server_chasm_lib_callback_proto_v1_service_proto_goTypes, + DependencyIndexes: file_temporal_server_chasm_lib_callback_proto_v1_service_proto_depIdxs, + }.Build() + File_temporal_server_chasm_lib_callback_proto_v1_service_proto = out.File + file_temporal_server_chasm_lib_callback_proto_v1_service_proto_goTypes = nil + file_temporal_server_chasm_lib_callback_proto_v1_service_proto_depIdxs = nil +} diff --git a/chasm/lib/callback/gen/callbackpb/v1/service_client.pb.go b/chasm/lib/callback/gen/callbackpb/v1/service_client.pb.go new file mode 100644 index 00000000000..7cba8c99a73 --- /dev/null +++ b/chasm/lib/callback/gen/callbackpb/v1/service_client.pb.go @@ -0,0 +1,275 @@ +// Code generated by protoc-gen-go-chasm. DO NOT EDIT. +package callbackspb + +import ( + "context" + "time" + + "go.temporal.io/server/client/history" + "go.temporal.io/server/common" + "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/config" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/membership" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/primitives" + "google.golang.org/grpc" +) + +// CallbackServiceLayeredClient is a client for CallbackService. +type CallbackServiceLayeredClient struct { + metricsHandler metrics.Handler + numShards int32 + redirector history.Redirector[CallbackServiceClient] + retryPolicy backoff.RetryPolicy +} + +// NewCallbackServiceLayeredClient initializes a new CallbackServiceLayeredClient. +func NewCallbackServiceLayeredClient( + dc *dynamicconfig.Collection, + rpcFactory common.RPCFactory, + monitor membership.Monitor, + config *config.Persistence, + logger log.Logger, + metricsHandler metrics.Handler, +) (CallbackServiceClient, error) { + resolver, err := monitor.GetResolver(primitives.HistoryService) + if err != nil { + return nil, err + } + connections := history.NewConnectionPool(resolver, rpcFactory, NewCallbackServiceClient) + var redirector history.Redirector[CallbackServiceClient] + if dynamicconfig.HistoryClientOwnershipCachingEnabled.Get(dc)() { + redirector = history.NewCachingRedirector( + connections, + resolver, + logger, + dynamicconfig.HistoryClientOwnershipCachingStaleTTL.Get(dc), + ) + } else { + redirector = history.NewBasicRedirector(connections, resolver) + } + return &CallbackServiceLayeredClient{ + metricsHandler: metricsHandler, + redirector: redirector, + numShards: config.NumHistoryShards, + retryPolicy: common.CreateHistoryClientRetryPolicy(), + }, nil +} +func (c *CallbackServiceLayeredClient) callStartCallbackExecutionNoRetry( + ctx context.Context, + request *StartCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*StartCallbackExecutionResponse, error) { + var response *StartCallbackExecutionResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("CallbackService.StartCallbackExecution"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetCallbackId(), c.numShards) + op := func(ctx context.Context, client CallbackServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.StartCallbackExecution(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *CallbackServiceLayeredClient) StartCallbackExecution( + ctx context.Context, + request *StartCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*StartCallbackExecutionResponse, error) { + call := func(ctx context.Context) (*StartCallbackExecutionResponse, error) { + return c.callStartCallbackExecutionNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *CallbackServiceLayeredClient) callDescribeCallbackExecutionNoRetry( + ctx context.Context, + request *DescribeCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*DescribeCallbackExecutionResponse, error) { + var response *DescribeCallbackExecutionResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("CallbackService.DescribeCallbackExecution"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetCallbackId(), c.numShards) + op := func(ctx context.Context, client CallbackServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.DescribeCallbackExecution(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *CallbackServiceLayeredClient) DescribeCallbackExecution( + ctx context.Context, + request *DescribeCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*DescribeCallbackExecutionResponse, error) { + call := func(ctx context.Context) (*DescribeCallbackExecutionResponse, error) { + return c.callDescribeCallbackExecutionNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *CallbackServiceLayeredClient) callPollCallbackExecutionNoRetry( + ctx context.Context, + request *PollCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*PollCallbackExecutionResponse, error) { + var response *PollCallbackExecutionResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("CallbackService.PollCallbackExecution"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetCallbackId(), c.numShards) + op := func(ctx context.Context, client CallbackServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.PollCallbackExecution(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *CallbackServiceLayeredClient) PollCallbackExecution( + ctx context.Context, + request *PollCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*PollCallbackExecutionResponse, error) { + call := func(ctx context.Context) (*PollCallbackExecutionResponse, error) { + return c.callPollCallbackExecutionNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *CallbackServiceLayeredClient) callTerminateCallbackExecutionNoRetry( + ctx context.Context, + request *TerminateCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*TerminateCallbackExecutionResponse, error) { + var response *TerminateCallbackExecutionResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("CallbackService.TerminateCallbackExecution"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetCallbackId(), c.numShards) + op := func(ctx context.Context, client CallbackServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.TerminateCallbackExecution(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *CallbackServiceLayeredClient) TerminateCallbackExecution( + ctx context.Context, + request *TerminateCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*TerminateCallbackExecutionResponse, error) { + call := func(ctx context.Context) (*TerminateCallbackExecutionResponse, error) { + return c.callTerminateCallbackExecutionNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} +func (c *CallbackServiceLayeredClient) callDeleteCallbackExecutionNoRetry( + ctx context.Context, + request *DeleteCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*DeleteCallbackExecutionResponse, error) { + var response *DeleteCallbackExecutionResponse + var err error + startTime := time.Now().UTC() + // the caller is a namespace, hence the tag below. + caller := headers.GetCallerInfo(ctx).CallerName + metricsHandler := c.metricsHandler.WithTags( + metrics.OperationTag("CallbackService.DeleteCallbackExecution"), + metrics.NamespaceTag(caller), + metrics.ServiceRoleTag(metrics.HistoryRoleTagValue), + ) + metrics.ClientRequests.With(metricsHandler).Record(1) + defer func() { + if err != nil { + metrics.ClientFailures.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) + } + metrics.ClientLatency.With(metricsHandler).Record(time.Since(startTime)) + }() + shardID := common.WorkflowIDToHistoryShard(request.GetNamespaceId(), request.GetFrontendRequest().GetCallbackId(), c.numShards) + op := func(ctx context.Context, client CallbackServiceClient) error { + var err error + ctx, cancel := context.WithTimeout(ctx, history.DefaultTimeout) + defer cancel() + response, err = client.DeleteCallbackExecution(ctx, request, opts...) + return err + } + err = c.redirector.Execute(ctx, shardID, op) + return response, err +} +func (c *CallbackServiceLayeredClient) DeleteCallbackExecution( + ctx context.Context, + request *DeleteCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*DeleteCallbackExecutionResponse, error) { + call := func(ctx context.Context) (*DeleteCallbackExecutionResponse, error) { + return c.callDeleteCallbackExecutionNoRetry(ctx, request, opts...) + } + return backoff.ThrottleRetryContextWithReturn(ctx, call, c.retryPolicy, common.IsServiceClientTransientError) +} diff --git a/chasm/lib/callback/gen/callbackpb/v1/service_grpc.pb.go b/chasm/lib/callback/gen/callbackpb/v1/service_grpc.pb.go new file mode 100644 index 00000000000..5323ef5842d --- /dev/null +++ b/chasm/lib/callback/gen/callbackpb/v1/service_grpc.pb.go @@ -0,0 +1,258 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// plugins: +// - protoc-gen-go-grpc +// - protoc +// source: temporal/server/chasm/lib/callback/proto/v1/service.proto + +package callbackspb + +import ( + context "context" + + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + CallbackService_StartCallbackExecution_FullMethodName = "/temporal.server.chasm.lib.callbacks.proto.v1.CallbackService/StartCallbackExecution" + CallbackService_DescribeCallbackExecution_FullMethodName = "/temporal.server.chasm.lib.callbacks.proto.v1.CallbackService/DescribeCallbackExecution" + CallbackService_PollCallbackExecution_FullMethodName = "/temporal.server.chasm.lib.callbacks.proto.v1.CallbackService/PollCallbackExecution" + CallbackService_TerminateCallbackExecution_FullMethodName = "/temporal.server.chasm.lib.callbacks.proto.v1.CallbackService/TerminateCallbackExecution" + CallbackService_DeleteCallbackExecution_FullMethodName = "/temporal.server.chasm.lib.callbacks.proto.v1.CallbackService/DeleteCallbackExecution" +) + +// CallbackServiceClient is the client API for CallbackService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type CallbackServiceClient interface { + StartCallbackExecution(ctx context.Context, in *StartCallbackExecutionRequest, opts ...grpc.CallOption) (*StartCallbackExecutionResponse, error) + DescribeCallbackExecution(ctx context.Context, in *DescribeCallbackExecutionRequest, opts ...grpc.CallOption) (*DescribeCallbackExecutionResponse, error) + PollCallbackExecution(ctx context.Context, in *PollCallbackExecutionRequest, opts ...grpc.CallOption) (*PollCallbackExecutionResponse, error) + TerminateCallbackExecution(ctx context.Context, in *TerminateCallbackExecutionRequest, opts ...grpc.CallOption) (*TerminateCallbackExecutionResponse, error) + DeleteCallbackExecution(ctx context.Context, in *DeleteCallbackExecutionRequest, opts ...grpc.CallOption) (*DeleteCallbackExecutionResponse, error) +} + +type callbackServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewCallbackServiceClient(cc grpc.ClientConnInterface) CallbackServiceClient { + return &callbackServiceClient{cc} +} + +func (c *callbackServiceClient) StartCallbackExecution(ctx context.Context, in *StartCallbackExecutionRequest, opts ...grpc.CallOption) (*StartCallbackExecutionResponse, error) { + out := new(StartCallbackExecutionResponse) + err := c.cc.Invoke(ctx, CallbackService_StartCallbackExecution_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *callbackServiceClient) DescribeCallbackExecution(ctx context.Context, in *DescribeCallbackExecutionRequest, opts ...grpc.CallOption) (*DescribeCallbackExecutionResponse, error) { + out := new(DescribeCallbackExecutionResponse) + err := c.cc.Invoke(ctx, CallbackService_DescribeCallbackExecution_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *callbackServiceClient) PollCallbackExecution(ctx context.Context, in *PollCallbackExecutionRequest, opts ...grpc.CallOption) (*PollCallbackExecutionResponse, error) { + out := new(PollCallbackExecutionResponse) + err := c.cc.Invoke(ctx, CallbackService_PollCallbackExecution_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *callbackServiceClient) TerminateCallbackExecution(ctx context.Context, in *TerminateCallbackExecutionRequest, opts ...grpc.CallOption) (*TerminateCallbackExecutionResponse, error) { + out := new(TerminateCallbackExecutionResponse) + err := c.cc.Invoke(ctx, CallbackService_TerminateCallbackExecution_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *callbackServiceClient) DeleteCallbackExecution(ctx context.Context, in *DeleteCallbackExecutionRequest, opts ...grpc.CallOption) (*DeleteCallbackExecutionResponse, error) { + out := new(DeleteCallbackExecutionResponse) + err := c.cc.Invoke(ctx, CallbackService_DeleteCallbackExecution_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// CallbackServiceServer is the server API for CallbackService service. +// All implementations must embed UnimplementedCallbackServiceServer +// for forward compatibility +type CallbackServiceServer interface { + StartCallbackExecution(context.Context, *StartCallbackExecutionRequest) (*StartCallbackExecutionResponse, error) + DescribeCallbackExecution(context.Context, *DescribeCallbackExecutionRequest) (*DescribeCallbackExecutionResponse, error) + PollCallbackExecution(context.Context, *PollCallbackExecutionRequest) (*PollCallbackExecutionResponse, error) + TerminateCallbackExecution(context.Context, *TerminateCallbackExecutionRequest) (*TerminateCallbackExecutionResponse, error) + DeleteCallbackExecution(context.Context, *DeleteCallbackExecutionRequest) (*DeleteCallbackExecutionResponse, error) + mustEmbedUnimplementedCallbackServiceServer() +} + +// UnimplementedCallbackServiceServer must be embedded to have forward compatible implementations. +type UnimplementedCallbackServiceServer struct { +} + +func (UnimplementedCallbackServiceServer) StartCallbackExecution(context.Context, *StartCallbackExecutionRequest) (*StartCallbackExecutionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartCallbackExecution not implemented") +} +func (UnimplementedCallbackServiceServer) DescribeCallbackExecution(context.Context, *DescribeCallbackExecutionRequest) (*DescribeCallbackExecutionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DescribeCallbackExecution not implemented") +} +func (UnimplementedCallbackServiceServer) PollCallbackExecution(context.Context, *PollCallbackExecutionRequest) (*PollCallbackExecutionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PollCallbackExecution not implemented") +} +func (UnimplementedCallbackServiceServer) TerminateCallbackExecution(context.Context, *TerminateCallbackExecutionRequest) (*TerminateCallbackExecutionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method TerminateCallbackExecution not implemented") +} +func (UnimplementedCallbackServiceServer) DeleteCallbackExecution(context.Context, *DeleteCallbackExecutionRequest) (*DeleteCallbackExecutionResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteCallbackExecution not implemented") +} +func (UnimplementedCallbackServiceServer) mustEmbedUnimplementedCallbackServiceServer() {} + +// UnsafeCallbackServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to CallbackServiceServer will +// result in compilation errors. +type UnsafeCallbackServiceServer interface { + mustEmbedUnimplementedCallbackServiceServer() +} + +func RegisterCallbackServiceServer(s grpc.ServiceRegistrar, srv CallbackServiceServer) { + s.RegisterService(&CallbackService_ServiceDesc, srv) +} + +func _CallbackService_StartCallbackExecution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(StartCallbackExecutionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CallbackServiceServer).StartCallbackExecution(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: CallbackService_StartCallbackExecution_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CallbackServiceServer).StartCallbackExecution(ctx, req.(*StartCallbackExecutionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CallbackService_DescribeCallbackExecution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DescribeCallbackExecutionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CallbackServiceServer).DescribeCallbackExecution(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: CallbackService_DescribeCallbackExecution_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CallbackServiceServer).DescribeCallbackExecution(ctx, req.(*DescribeCallbackExecutionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CallbackService_PollCallbackExecution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PollCallbackExecutionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CallbackServiceServer).PollCallbackExecution(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: CallbackService_PollCallbackExecution_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CallbackServiceServer).PollCallbackExecution(ctx, req.(*PollCallbackExecutionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CallbackService_TerminateCallbackExecution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TerminateCallbackExecutionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CallbackServiceServer).TerminateCallbackExecution(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: CallbackService_TerminateCallbackExecution_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CallbackServiceServer).TerminateCallbackExecution(ctx, req.(*TerminateCallbackExecutionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CallbackService_DeleteCallbackExecution_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteCallbackExecutionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CallbackServiceServer).DeleteCallbackExecution(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: CallbackService_DeleteCallbackExecution_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CallbackServiceServer).DeleteCallbackExecution(ctx, req.(*DeleteCallbackExecutionRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// CallbackService_ServiceDesc is the grpc.ServiceDesc for CallbackService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var CallbackService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "temporal.server.chasm.lib.callbacks.proto.v1.CallbackService", + HandlerType: (*CallbackServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "StartCallbackExecution", + Handler: _CallbackService_StartCallbackExecution_Handler, + }, + { + MethodName: "DescribeCallbackExecution", + Handler: _CallbackService_DescribeCallbackExecution_Handler, + }, + { + MethodName: "PollCallbackExecution", + Handler: _CallbackService_PollCallbackExecution_Handler, + }, + { + MethodName: "TerminateCallbackExecution", + Handler: _CallbackService_TerminateCallbackExecution_Handler, + }, + { + MethodName: "DeleteCallbackExecution", + Handler: _CallbackService_DeleteCallbackExecution_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "temporal/server/chasm/lib/callback/proto/v1/service.proto", +} diff --git a/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go b/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go index a0181447c66..7c0fcca665b 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go @@ -78,3 +78,40 @@ func (this *BackoffTask) Equal(that interface{}) bool { return proto.Equal(this, that1) } + +// Marshal an object of type CompletionScheduleToCloseTimeoutTask to the protobuf v3 wire format +func (val *CompletionScheduleToCloseTimeoutTask) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type CompletionScheduleToCloseTimeoutTask from the protobuf v3 wire format +func (val *CompletionScheduleToCloseTimeoutTask) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *CompletionScheduleToCloseTimeoutTask) Size() int { + return proto.Size(val) +} + +// Equal returns whether two CompletionScheduleToCloseTimeoutTask values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *CompletionScheduleToCloseTimeoutTask) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *CompletionScheduleToCloseTimeoutTask + switch t := that.(type) { + case *CompletionScheduleToCloseTimeoutTask: + that1 = t + case CompletionScheduleToCloseTimeoutTask: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go b/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go index 7354a359c8d..33e29da618d 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go @@ -112,6 +112,43 @@ func (x *BackoffTask) GetAttempt() int32 { return 0 } +// Fired when the callback completion's schedule-to-close timeout expires. +type CompletionScheduleToCloseTimeoutTask struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CompletionScheduleToCloseTimeoutTask) Reset() { + *x = CompletionScheduleToCloseTimeoutTask{} + mi := &file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CompletionScheduleToCloseTimeoutTask) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CompletionScheduleToCloseTimeoutTask) ProtoMessage() {} + +func (x *CompletionScheduleToCloseTimeoutTask) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CompletionScheduleToCloseTimeoutTask.ProtoReflect.Descriptor instead. +func (*CompletionScheduleToCloseTimeoutTask) Descriptor() ([]byte, []int) { + return file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDescGZIP(), []int{2} +} + var File_temporal_server_chasm_lib_callback_proto_v1_tasks_proto protoreflect.FileDescriptor const file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDesc = "" + @@ -120,7 +157,8 @@ const file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDesc = "" "\x0eInvocationTask\x12\x18\n" + "\aattempt\x18\x01 \x01(\x05R\aattempt\"'\n" + "\vBackoffTask\x12\x18\n" + - "\aattempt\x18\x01 \x01(\x05R\aattemptBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" + "\aattempt\x18\x01 \x01(\x05R\aattempt\"&\n" + + "$CompletionScheduleToCloseTimeoutTaskBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" var ( file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDescOnce sync.Once @@ -134,10 +172,11 @@ func file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDescGZIP() return file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDescData } -var file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_goTypes = []any{ - (*InvocationTask)(nil), // 0: temporal.server.chasm.lib.callbacks.proto.v1.InvocationTask - (*BackoffTask)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.BackoffTask + (*InvocationTask)(nil), // 0: temporal.server.chasm.lib.callbacks.proto.v1.InvocationTask + (*BackoffTask)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.BackoffTask + (*CompletionScheduleToCloseTimeoutTask)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.CompletionScheduleToCloseTimeoutTask } var file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -158,7 +197,7 @@ func file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDesc), len(file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDesc)), NumEnums: 0, - NumMessages: 2, + NumMessages: 3, NumExtensions: 0, NumServices: 0, }, diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go new file mode 100644 index 00000000000..b70824a19df --- /dev/null +++ b/chasm/lib/callback/handler.go @@ -0,0 +1,352 @@ +package callback + +import ( + "context" + "errors" + "fmt" + + callbackpb "go.temporal.io/api/callback/v1" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common/contextutil" + "go.temporal.io/server/common/log" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type callbackHandler struct { + callbackspb.UnimplementedCallbackServiceServer + + config *Config + logger log.Logger +} + +func newCallbackHandler(config *Config, logger log.Logger) *callbackHandler { + return &callbackHandler{ + config: config, + logger: logger, + } +} + +func (h *callbackHandler) StartCallbackExecution( + ctx context.Context, + req *callbackspb.StartCallbackExecutionRequest, +) (resp *callbackspb.StartCallbackExecutionResponse, err error) { + frontendReq := req.FrontendRequest + + // Gather all the data necessary to create the Callback component. + input := &createStandaloneCallbackInput{ + CallbackID: frontendReq.GetCallbackId(), + RequestID: frontendReq.GetRequestId(), + CompletionScheduleToCloseTimeout: frontendReq.GetScheduleToCloseTimeout(), + Completion: frontendReq.GetCompletion(), + SearchAttributes: frontendReq.GetSearchAttributes().GetIndexedFields(), + } + + // Convert the API Callback to internal Callback proto. + if nexusCb := frontendReq.GetCallback().GetNexus(); nexusCb != nil { + input.Callback = &callbackspb.Callback{ + Variant: &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{ + Url: nexusCb.GetUrl(), + Header: nexusCb.GetHeader(), + Token: nexusCb.GetToken(), + }, + }, + } + } + + // Create the CHASM Callback in so-called "standalone" mode, where it will be the root + // of the CHASM execution. + result, err := chasm.StartExecution( + ctx, + chasm.ExecutionKey{ + NamespaceID: req.NamespaceId, + BusinessID: frontendReq.GetCallbackId(), + }, + createStandaloneCallback, + input, + chasm.WithRequestID(frontendReq.GetRequestId()), + // Relying on these default policies. No configuration knobs are exposed to users. + chasm.WithBusinessIDPolicy( + chasm.BusinessIDReusePolicyAllowDuplicate, + chasm.BusinessIDConflictPolicyFail, + ), + ) + + // Like Workflow IDs, the Callback ID can be reused. But only one Callback with a given Callback ID + // can be executing at a given time. + var alreadyStartedErr *chasm.ExecutionAlreadyStartedError + if errors.As(err, &alreadyStartedErr) { + svcErr := serviceerror.NewCallbackExecutionAlreadyStarted( + "callback execution already started", + alreadyStartedErr.CurrentRequestID, + alreadyStartedErr.CurrentRunID, + frontendReq.GetCallbackId(), + ) + return nil, svcErr + } + if err != nil { + return nil, err + } + + return &callbackspb.StartCallbackExecutionResponse{ + FrontendResponse: &workflowservice.StartCallbackExecutionResponse{ + RunId: result.ExecutionKey.RunID, + }, + }, nil +} + +func (h *callbackHandler) DescribeCallbackExecution( + ctx context.Context, + req *callbackspb.DescribeCallbackExecutionRequest, +) (*callbackspb.DescribeCallbackExecutionResponse, error) { + + // Build the DescribeCallbackExecution proto. Closes over the req object. + buildDescriptionProto := func( + ctx chasm.Context, + c *Callback, + ) (*callbackspb.DescribeCallbackExecutionResponse, error) { + info, err := c.Describe(ctx) + if err != nil { + return nil, err + } + resp := &workflowservice.DescribeCallbackExecutionResponse{ + Info: info, + } + + if req.FrontendRequest.GetIncludeInput() { + resp.Input = c.SuppliedCompletion.Get(ctx) + } + if req.FrontendRequest.GetIncludeOutcome() { + resp.Outcome = c.Outcome(ctx) + } + + return &callbackspb.DescribeCallbackExecutionResponse{ + FrontendResponse: resp, + }, nil + } + + compRef := chasm.NewComponentRef[*Callback]( + chasm.ExecutionKey{ + NamespaceID: req.GetNamespaceId(), + BusinessID: req.FrontendRequest.GetCallbackId(), + RunID: req.FrontendRequest.GetRunId(), + }, + ) + + // Simple case. If no long-poll token is supplied, we just read and return + // the persisted state. + token := req.GetFrontendRequest().GetLongPollToken() + if len(token) == 0 { + return chasm.ReadComponent( + ctx, + compRef, + func( + c *Callback, + ctx chasm.Context, + req *callbackspb.DescribeCallbackExecutionRequest) (*callbackspb.DescribeCallbackExecutionResponse, error) { + return buildDescriptionProto(ctx, c) + }, + req) + } + + // Below, we send an empty non-error response on context deadline expiry. Here we compute a + // deadline that causes us to send that response before the caller's own deadline (see + // chasm.activity.longPollBuffer). We also cap the caller's deadline at + // chasm.activity.longPollTimeout. + targetNamespace := req.GetFrontendRequest().GetNamespace() + ctx, cancel := contextutil.WithDeadlineBuffer( + ctx, + h.config.LongPollTimeout(targetNamespace), + h.config.LongPollBuffer(targetNamespace), + ) + defer cancel() + + longpollReadFn := func( + c *Callback, + ctx chasm.Context, + req *callbackspb.DescribeCallbackExecutionRequest) (*callbackspb.DescribeCallbackExecutionResponse, bool, error) { + changed, err := chasm.ExecutionStateChanged(c, ctx, token) + if err != nil { + if errors.Is(err, chasm.ErrMalformedComponentRef) { + return nil, false, serviceerror.NewInvalidArgument("invalid long poll token") + } + if errors.Is(err, chasm.ErrInvalidComponentRef) { + return nil, false, serviceerror.NewInvalidArgument("long poll token does not match execution") + } + return nil, false, err + } + if changed { + response, err := buildDescriptionProto(ctx, c) + return response, true, err + } + return nil, false, nil + } + + // Now begin the polling, using our supplied reader. + response, _, err := chasm.PollComponent(ctx, compRef, longpollReadFn, req) + if err != nil && ctx.Err() != nil { + // Send empty non-error response on deadline expiry: caller should continue long-polling. + return &callbackspb.DescribeCallbackExecutionResponse{ + FrontendResponse: &workflowservice.DescribeCallbackExecutionResponse{}, + }, nil + } + return response, err +} + +func (h *callbackHandler) PollCallbackExecution( + ctx context.Context, + req *callbackspb.PollCallbackExecutionRequest, +) (resp *callbackspb.PollCallbackExecutionResponse, err error) { + + ref := chasm.NewComponentRef[*Callback]( + chasm.ExecutionKey{ + NamespaceID: req.NamespaceId, + BusinessID: req.FrontendRequest.GetCallbackId(), + RunID: req.FrontendRequest.GetRunId(), + }, + ) + + ns := req.FrontendRequest.GetNamespace() + ctx, cancel := contextutil.WithDeadlineBuffer( + ctx, + h.config.LongPollTimeout(ns), + h.config.LongPollBuffer(ns), + ) + defer cancel() + + resp, _, err = chasm.PollComponent(ctx, ref, func( + c *Callback, + ctx chasm.Context, + _ *callbackspb.PollCallbackExecutionRequest, + ) (*callbackspb.PollCallbackExecutionResponse, bool, error) { + if !c.LifecycleState(ctx).IsClosed() { + return nil, false, nil + } + return &callbackspb.PollCallbackExecutionResponse{ + FrontendResponse: &workflowservice.PollCallbackExecutionResponse{ + RunId: ctx.ExecutionKey().RunID, + Outcome: c.Outcome(ctx), + }, + }, true, nil + }, req) + + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + // Send an empty non-error response as an invitation to resubmit the long-poll. + return &callbackspb.PollCallbackExecutionResponse{ + FrontendResponse: &workflowservice.PollCallbackExecutionResponse{}, + }, nil + } + return resp, err +} + +func (h *callbackHandler) TerminateCallbackExecution( + ctx context.Context, + req *callbackspb.TerminateCallbackExecutionRequest, +) (resp *callbackspb.TerminateCallbackExecutionResponse, err error) { + + resp, _, err = chasm.UpdateComponent( + ctx, + chasm.NewComponentRef[*Callback]( + chasm.ExecutionKey{ + NamespaceID: req.NamespaceId, + BusinessID: req.FrontendRequest.GetCallbackId(), + RunID: req.FrontendRequest.GetRunId(), + }, + ), + func(c *Callback, ctx chasm.MutableContext, _ *callbackspb.TerminateCallbackExecutionRequest) (*callbackspb.TerminateCallbackExecutionResponse, error) { + if _, err := c.Terminate(ctx, chasm.TerminateComponentRequest{ + Reason: req.FrontendRequest.GetReason(), + RequestID: req.FrontendRequest.GetRequestId(), + }); err != nil { + return nil, err + } + return &callbackspb.TerminateCallbackExecutionResponse{ + FrontendResponse: &workflowservice.TerminateCallbackExecutionResponse{}, + }, nil + }, + req, + ) + return resp, err +} + +func (h *callbackHandler) DeleteCallbackExecution( + ctx context.Context, + req *callbackspb.DeleteCallbackExecutionRequest, +) (resp *callbackspb.DeleteCallbackExecutionResponse, err error) { + + if err = chasm.DeleteExecution[*Callback]( + ctx, + chasm.ExecutionKey{ + NamespaceID: req.NamespaceId, + BusinessID: req.FrontendRequest.GetCallbackId(), + RunID: req.FrontendRequest.GetRunId(), + }, + chasm.DeleteExecutionRequest{ + TerminateComponentRequest: chasm.TerminateComponentRequest{ + Reason: "deleted", + }, + }, + ); err != nil { + return nil, err + } + + return &callbackspb.DeleteCallbackExecutionResponse{ + FrontendResponse: &workflowservice.DeleteCallbackExecutionResponse{}, + }, nil +} + +// createStandaloneCallbackInput is the bundle of inputs to the CHASM execution. +type createStandaloneCallbackInput struct { + RequestID string + Callback *callbackspb.Callback + CallbackID string + CompletionScheduleToCloseTimeout *durationpb.Duration + Completion *callbackpb.CallbackExecutionCompletion + SearchAttributes map[string]*commonpb.Payload +} + +// createStandaloneCallback constructs a new Callback component in "standalone" mode. +// The Callback is immediately transitioned to SCHEDULED state to begin invocation. +func createStandaloneCallback( + ctx chasm.MutableContext, + input *createStandaloneCallbackInput, +) (*Callback, error) { + now := timestamppb.Now() + + // Create child Callback component. + opts := newStandaloneCallbackOpts{ + RequestID: input.RequestID, + RegistrationTime: now, + Callback: input.Callback, + + CallbackID: input.CallbackID, + CompletionScheduleToCloseTimeout: input.CompletionScheduleToCloseTimeout, + Completion: input.Completion, + SearchAttributes: input.SearchAttributes, + } + cb := newStandaloneCallback(ctx, opts) + + // Immediately schedule the callback for invocation. + if err := TransitionScheduled.Apply(cb, ctx, EventScheduled{}); err != nil { + return nil, fmt.Errorf("failed to schedule callback: %w", err) + } + + // Schedule the timeout as applicable. + if durationProto := input.CompletionScheduleToCloseTimeout; durationProto != nil { + if duration := durationProto.AsDuration(); duration > 0 { + timeoutTime := now.AsTime().Add(duration) + ctx.AddTask( + cb, + chasm.TaskAttributes{ScheduledTime: timeoutTime}, + &callbackspb.CompletionScheduleToCloseTimeoutTask{}, + ) + } + } + + return cb, nil +} diff --git a/chasm/lib/callback/invocable_internal.go b/chasm/lib/callback/invocable_internal.go index 273c0a38634..75f465236ae 100644 --- a/chasm/lib/callback/invocable_internal.go +++ b/chasm/lib/callback/invocable_internal.go @@ -58,13 +58,14 @@ func (c invocableInternal) Invoke( task *callbackspb.InvocationTask, taskAttr chasm.TaskAttributes, ) invocationResult { - header := nexus.Header(c.callback.GetHeader()) - if header == nil { - header = nexus.Header{} + // Get the token from the dedicated Token field, falling back to the header for backwards compat. + encodedRef := c.callback.GetToken() + if encodedRef == "" { + header := nexus.Header(c.callback.GetHeader()) + if header != nil { + encodedRef = header.Get(commonnexus.CallbackTokenHeader) + } } - - // Get back the base64-encoded ComponentRef from the header. - encodedRef := header.Get(commonnexus.CallbackTokenHeader) if encodedRef == "" { return invocationResultFail{logInternalError(h.logger, "callback missing token", nil)} } diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 6ed0a65d6dc..f8bafda576c 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -66,8 +66,17 @@ func (n invocableOutbound) Invoke( }) // Make the call and record metrics. startTime := time.Now() - n.completion.Header = n.callback.Header + + // If the outbound call is to a standalone callback, then supply the Nexus + // operation's token in the request. + if n.callback.GetToken() != "" { + if n.completion.Header == nil { + n.completion.Header = nexus.Header{} + } + n.completion.Header.Set(commonnexus.CallbackTokenHeader, n.callback.GetToken()) + } + err := client.CompleteOperation(ctx, n.callback.Url, n.completion) namespaceTag := metrics.NamespaceTag(ns.Name().String()) diff --git a/chasm/lib/callback/library.go b/chasm/lib/callback/library.go index 838a88e1608..90785feb9f0 100644 --- a/chasm/lib/callback/library.go +++ b/chasm/lib/callback/library.go @@ -2,42 +2,60 @@ package callback import ( "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common/namespace" "google.golang.org/grpc" ) -type ( - Library struct { - chasm.UnimplementedLibrary - - InvocationTaskHandler *invocationTaskHandler - BackoffTaskHandler *backoffTaskHandler - } -) +// componentOnlyLibrary only containing the component definitions, but no implementation details. +type componentOnlyLibrary struct { + chasm.UnimplementedLibrary +} -func newLibrary( - InvocationTaskHandler *invocationTaskHandler, - BackoffTaskHandler *backoffTaskHandler, -) *Library { - return &Library{ - InvocationTaskHandler: InvocationTaskHandler, - BackoffTaskHandler: BackoffTaskHandler, - } +func newComponentOnlyLibrary(config *Config, namespaceRegistry namespace.Registry) *componentOnlyLibrary { + return &componentOnlyLibrary{} } -func (l *Library) Name() string { +func (l *componentOnlyLibrary) Name() string { return chasm.CallbackLibraryName } -func (l *Library) Components() []*chasm.RegistrableComponent { +func (l *componentOnlyLibrary) Components() []*chasm.RegistrableComponent { return []*chasm.RegistrableComponent{ chasm.NewRegistrableComponent[*Callback]( chasm.CallbackComponentName, chasm.WithDetached(), + chasm.WithBusinessIDAlias("CallbackId"), + chasm.WithSearchAttributes(executionStatusSearchAttribute), ), } } -func (l *Library) Tasks() []*chasm.RegistrableTask { +type library struct { + componentOnlyLibrary + + config *Config + InvocationTaskHandler *invocationTaskHandler + BackoffTaskHandler *backoffTaskHandler + CompletionScheduleToCloseTimeoutTaskHandler *CompletionScheduleToCloseTimeoutTaskHandler + callbackSvcHandler *callbackHandler +} + +func newLibrary( + InvocationTaskHandler *invocationTaskHandler, + BackoffTaskHandler *backoffTaskHandler, + CompletionScheduleToCloseTimeoutTaskHandler *CompletionScheduleToCloseTimeoutTaskHandler, + callbackSvcHandler *callbackHandler, +) *library { + return &library{ + InvocationTaskHandler: InvocationTaskHandler, + BackoffTaskHandler: BackoffTaskHandler, + CompletionScheduleToCloseTimeoutTaskHandler: CompletionScheduleToCloseTimeoutTaskHandler, + callbackSvcHandler: callbackSvcHandler, + } +} + +func (l *library) Tasks() []*chasm.RegistrableTask { return []*chasm.RegistrableTask{ chasm.NewRegistrableSideEffectTask( "invoke", @@ -47,8 +65,13 @@ func (l *Library) Tasks() []*chasm.RegistrableTask { "backoff", l.BackoffTaskHandler, ), + chasm.NewRegistrablePureTask( + "completionScheduleToCloseTimer", + l.CompletionScheduleToCloseTimeoutTaskHandler, + ), } } -func (l *Library) RegisterServices(server *grpc.Server) { +func (l *library) RegisterServices(server *grpc.Server) { + callbackspb.RegisterCallbackServiceServer(server, l.callbackSvcHandler) } diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index 057e5c470e0..b6d78a6c764 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package temporal.server.chasm.lib.callbacks.proto.v1; +import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "temporal/api/common/v1/message.proto"; import "temporal/api/failure/v1/message.proto"; @@ -33,6 +34,19 @@ message CallbackState { // Request ID that added the callback. string request_id = 9; + // Request ID that terminated the callback, if applicable. Used for idempotency. + string terminate_request_id = 10; + + // The time when the callback reached a terminal state. + google.protobuf.Timestamp close_time = 11; + + // (standalone only) User-supplied business ID set when StartCallbackExecution() is + // called. Used to identify the callback for operations like Describe- or Terminate-. + string callback_id = 12; + + // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() + // is called to when the result gets delivered. + google.protobuf.Duration completion_schedule_to_close_timeout = 13; } // Status of a callback. @@ -49,6 +63,8 @@ enum CallbackStatus { CALLBACK_STATUS_FAILED = 4; // Callback has succeeded. CALLBACK_STATUS_SUCCEEDED = 5; + // Callback was terminated by request. + CALLBACK_STATUS_TERMINATED = 6; } message Callback { @@ -59,6 +75,8 @@ message Callback { string url = 1; // Header to attach to callback request. map header = 2; + // Token identifying the target callback to resolve. + string token = 3; } reserved 1; // For a generic callback mechanism to be added later. diff --git a/chasm/lib/callback/proto/v1/request_response.proto b/chasm/lib/callback/proto/v1/request_response.proto new file mode 100644 index 00000000000..75de45252e7 --- /dev/null +++ b/chasm/lib/callback/proto/v1/request_response.proto @@ -0,0 +1,62 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.callbacks.proto.v1; + +import "temporal/api/workflowservice/v1/request_response.proto"; + +option go_package = "go.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspb"; + +message StartCallbackExecutionRequest { + // Internal namespace ID (UUID). + string namespace_id = 1; + + temporal.api.workflowservice.v1.StartCallbackExecutionRequest frontend_request = 2; +} + +message StartCallbackExecutionResponse { + temporal.api.workflowservice.v1.StartCallbackExecutionResponse frontend_response = 1; +} + +message DescribeCallbackExecutionRequest { + // Internal namespace ID (UUID). + string namespace_id = 1; + + temporal.api.workflowservice.v1.DescribeCallbackExecutionRequest frontend_request = 2; +} + +message DescribeCallbackExecutionResponse { + temporal.api.workflowservice.v1.DescribeCallbackExecutionResponse frontend_response = 1; +} + +message PollCallbackExecutionRequest { + // Internal namespace ID (UUID). + string namespace_id = 1; + + temporal.api.workflowservice.v1.PollCallbackExecutionRequest frontend_request = 2; +} + +message PollCallbackExecutionResponse { + temporal.api.workflowservice.v1.PollCallbackExecutionResponse frontend_response = 1; +} + +message TerminateCallbackExecutionRequest { + // Internal namespace ID (UUID). + string namespace_id = 1; + + temporal.api.workflowservice.v1.TerminateCallbackExecutionRequest frontend_request = 2; +} + +message TerminateCallbackExecutionResponse { + temporal.api.workflowservice.v1.TerminateCallbackExecutionResponse frontend_response = 1; +} + +message DeleteCallbackExecutionRequest { + // Internal namespace ID (UUID). + string namespace_id = 1; + + temporal.api.workflowservice.v1.DeleteCallbackExecutionRequest frontend_request = 2; +} + +message DeleteCallbackExecutionResponse { + temporal.api.workflowservice.v1.DeleteCallbackExecutionResponse frontend_response = 1; +} diff --git a/chasm/lib/callback/proto/v1/service.proto b/chasm/lib/callback/proto/v1/service.proto new file mode 100644 index 00000000000..1413d10ac4b --- /dev/null +++ b/chasm/lib/callback/proto/v1/service.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package temporal.server.chasm.lib.callbacks.proto.v1; + +import "chasm/lib/callback/proto/v1/request_response.proto"; +import "temporal/server/api/common/v1/api_category.proto"; +import "temporal/server/api/routing/v1/extension.proto"; + +option go_package = "go.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspb"; + +service CallbackService { + rpc StartCallbackExecution(StartCallbackExecutionRequest) returns (StartCallbackExecutionResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.callback_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc DescribeCallbackExecution(DescribeCallbackExecutionRequest) returns (DescribeCallbackExecutionResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.callback_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc PollCallbackExecution(PollCallbackExecutionRequest) returns (PollCallbackExecutionResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.callback_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_LONG_POLL; + } + + rpc TerminateCallbackExecution(TerminateCallbackExecutionRequest) returns (TerminateCallbackExecutionResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.callback_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } + + rpc DeleteCallbackExecution(DeleteCallbackExecutionRequest) returns (DeleteCallbackExecutionResponse) { + option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.callback_id"; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + } +} diff --git a/chasm/lib/callback/proto/v1/tasks.proto b/chasm/lib/callback/proto/v1/tasks.proto index f4cb65faa6a..dde93a730f9 100644 --- a/chasm/lib/callback/proto/v1/tasks.proto +++ b/chasm/lib/callback/proto/v1/tasks.proto @@ -13,3 +13,6 @@ message BackoffTask { // The attempt number for this invocation. int32 attempt = 1; } + +// Fired when the callback completion's schedule-to-close timeout expires. +message CompletionScheduleToCloseTimeoutTask {} diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 779b9773989..1d587658864 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -5,6 +5,7 @@ import ( "net/url" "time" + enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" @@ -37,7 +38,7 @@ var TransitionRescheduled = chasm.NewTransition( callbackspb.CALLBACK_STATUS_SCHEDULED, func(cb *Callback, ctx chasm.MutableContext, event EventRescheduled) error { cb.NextAttemptScheduleTime = nil - u, err := url.Parse(cb.Callback.GetNexus().Url) + u, err := url.Parse(cb.Callback.GetNexus().GetUrl()) if err != nil { return fmt.Errorf("failed to parse URL: %v: %w", cb.Callback, err) } @@ -61,7 +62,10 @@ var TransitionAttemptFailed = chasm.NewTransition( []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, callbackspb.CALLBACK_STATUS_BACKING_OFF, func(cb *Callback, ctx chasm.MutableContext, event EventAttemptFailed) error { - cb.recordAttempt(event.Time) + now := ctx.Now(cb) + cb.recordAttempt(now) + cb.CloseTime = timestamppb.New(now) + // Use 0 for elapsed time as we don't limit the retry by time (for now). nextDelay := event.RetryPolicy.ComputeNextDelay(0, int(cb.Attempt), event.Err) nextAttemptScheduleTime := event.Time.Add(nextDelay) @@ -93,8 +97,11 @@ var TransitionFailed = chasm.NewTransition( []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, callbackspb.CALLBACK_STATUS_FAILED, func(cb *Callback, ctx chasm.MutableContext, event EventFailed) error { - cb.recordAttempt(event.Time) - cb.LastAttemptFailure = &failurepb.Failure{ + now := ctx.Now(cb) + cb.recordAttempt(now) + cb.CloseTime = timestamppb.New(now) + + failure := &failurepb.Failure{ Message: event.Err.Error(), FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ @@ -102,6 +109,9 @@ var TransitionFailed = chasm.NewTransition( }, }, } + cb.LastAttemptFailure = failure + cb.TerminalFailure = chasm.NewDataField(ctx, failure) + return nil }, ) @@ -115,8 +125,69 @@ var TransitionSucceeded = chasm.NewTransition( []callbackspb.CallbackStatus{callbackspb.CALLBACK_STATUS_SCHEDULED}, callbackspb.CALLBACK_STATUS_SUCCEEDED, func(cb *Callback, ctx chasm.MutableContext, event EventSucceeded) error { - cb.recordAttempt(event.Time) + now := ctx.Now(cb) + cb.recordAttempt(now) cb.LastAttemptFailure = nil + cb.TerminalFailure = chasm.NewDataField[*failurepb.Failure](ctx, nil) + return nil + }, +) + +// EventTerminated is triggered when the callback is forcefully terminated. +type EventTerminated struct { + Reason string +} + +var TransitionTerminated = chasm.NewTransition( + []callbackspb.CallbackStatus{ + callbackspb.CALLBACK_STATUS_STANDBY, + callbackspb.CALLBACK_STATUS_SCHEDULED, + callbackspb.CALLBACK_STATUS_BACKING_OFF, + }, + callbackspb.CALLBACK_STATUS_TERMINATED, + func(cb *Callback, ctx chasm.MutableContext, event EventTerminated) error { + now := ctx.Now(cb) + cb.CloseTime = timestamppb.New(now) + + reason := event.Reason + if reason == "" { + reason = "callback execution terminated" + } + + failure := &failurepb.Failure{ + Message: reason, + FailureInfo: &failurepb.Failure_TerminatedFailureInfo{}, + } + cb.TerminalFailure = chasm.NewDataField(ctx, failure) + + return nil + }, +) + +// EventTimedOut is triggered when the callback's schedule-to-close timeout fires. +type EventTimedOut struct{} + +var TransitionTimedOut = chasm.NewTransition( + []callbackspb.CallbackStatus{ + callbackspb.CALLBACK_STATUS_STANDBY, + callbackspb.CALLBACK_STATUS_SCHEDULED, + callbackspb.CALLBACK_STATUS_BACKING_OFF, + }, + callbackspb.CALLBACK_STATUS_FAILED, + func(cb *Callback, ctx chasm.MutableContext, event EventTimedOut) error { + now := ctx.Now(cb) + cb.CloseTime = timestamppb.New(now) + + failure := &failurepb.Failure{ + Message: "callback execution timed out", + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ + TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, + }, + }, + } + cb.TerminalFailure = chasm.NewDataField(ctx, failure) + return nil }, ) diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index dbefaf5d96c..63c5fc7e70a 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -20,7 +20,7 @@ func TestValidTransitions(t *testing.T) { Callback: &callbackspb.Callback{ Variant: &callbackspb.Callback_Nexus_{ Nexus: &callbackspb.Callback_Nexus{ - Url: "http://address:666/path/to/callback?query=string", + Url: "http://address:999/path/to/callback?query=string", }, }, }, @@ -30,6 +30,8 @@ func TestValidTransitions(t *testing.T) { // AttemptFailed mctx := &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + err := TransitionAttemptFailed.Apply(callback, mctx, EventAttemptFailed{ Time: currentTime, Err: errors.New("test"), @@ -76,6 +78,8 @@ func TestValidTransitions(t *testing.T) { // Succeeded currentTime = currentTime.Add(time.Second) mctx = &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + err = TransitionSucceeded.Apply(callback, mctx, EventSucceeded{Time: currentTime}) require.NoError(t, err) @@ -96,6 +100,8 @@ func TestValidTransitions(t *testing.T) { // failed mctx = &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + err = TransitionFailed.Apply(callback, mctx, EventFailed{Time: currentTime, Err: errors.New("failed")}) require.NoError(t, err) @@ -110,3 +116,49 @@ func TestValidTransitions(t *testing.T) { // Assert task is not generated, failed is terminal require.Empty(t, mctx.Tasks) } + +func TestTerminatedTransition(t *testing.T) { + callback := &Callback{ + CallbackState: &callbackspb.CallbackState{ + Callback: &callbackspb.Callback{ + Variant: &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{ + Url: "http://address:999/path", + }, + }, + }, + }, + } + + for _, src := range []callbackspb.CallbackStatus{ + callbackspb.CALLBACK_STATUS_STANDBY, + callbackspb.CALLBACK_STATUS_SCHEDULED, + callbackspb.CALLBACK_STATUS_BACKING_OFF, + } { + t.Run("from_"+src.String(), func(t *testing.T) { + cb := &Callback{CallbackState: proto.Clone(callback.CallbackState).(*callbackspb.CallbackState)} + cb.SetStateMachineState(src) + mctx := &chasm.MockMutableContext{} + err := TransitionTerminated.Apply(cb, mctx, EventTerminated{}) + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_TERMINATED, cb.StateMachineState()) + }) + } +} + +func TestSaveResult_TerminatedWhileInFlight(t *testing.T) { + // If the callback was terminated while an invocation was in-flight, + // saveResult should drop the result silently. + cb := &Callback{ + CallbackState: &callbackspb.CallbackState{ + Status: callbackspb.CALLBACK_STATUS_TERMINATED, + }, + } + mctx := &chasm.MockMutableContext{} + _, err := cb.saveResult(mctx, saveResultInput{ + result: invocationResultOK{}, + retryPolicy: backoff.NewExponentialRetryPolicy(time.Second), + }) + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_TERMINATED, cb.StateMachineState()) +} diff --git a/chasm/lib/callback/tasks.go b/chasm/lib/callback/tasks.go index b53efafea19..406b2468045 100644 --- a/chasm/lib/callback/tasks.go +++ b/chasm/lib/callback/tasks.go @@ -180,3 +180,30 @@ func (h *backoffTaskHandler) Validate( ) (bool, error) { return callback.Status == callbackspb.CALLBACK_STATUS_BACKING_OFF && callback.Attempt == task.Attempt, nil } + +// CompletionScheduleToCloseTimeoutTaskHandler handles schedule-to-close timeout for standalone callback executions. +type CompletionScheduleToCloseTimeoutTaskHandler struct { + chasm.PureTaskHandlerBase +} + +func NewCompletionScheduleToCloseTimeoutTaskHandler() *CompletionScheduleToCloseTimeoutTaskHandler { + return &CompletionScheduleToCloseTimeoutTaskHandler{} +} + +func (h *CompletionScheduleToCloseTimeoutTaskHandler) Validate( + _ chasm.Context, + callback *Callback, + _ chasm.TaskAttributes, + _ *callbackspb.CompletionScheduleToCloseTimeoutTask, +) (bool, error) { + return TransitionTimedOut.Possible(callback), nil +} + +func (h *CompletionScheduleToCloseTimeoutTaskHandler) Execute( + ctx chasm.MutableContext, + callback *Callback, + _ chasm.TaskAttributes, + _ *callbackspb.CompletionScheduleToCloseTimeoutTask, +) error { + return TransitionTimedOut.Apply(callback, ctx, EventTimedOut{}) +} diff --git a/chasm/lib/callback/tasks_test.go b/chasm/lib/callback/tasks_test.go index 71e5c881a64..c0793ed7961 100644 --- a/chasm/lib/callback/tasks_test.go +++ b/chasm/lib/callback/tasks_test.go @@ -192,7 +192,7 @@ func TestExecuteInvocationTaskNexus_Outcomes(t *testing.T) { } chasmRegistry := chasm.NewRegistry(logger) - err = chasmRegistry.Register(&Library{ + err = chasmRegistry.Register(&library{ InvocationTaskHandler: handler, }) require.NoError(t, err) @@ -547,7 +547,7 @@ func TestExecuteInvocationTaskChasm_Outcomes(t *testing.T) { } chasmRegistry := chasm.NewRegistry(logger) - err = chasmRegistry.Register(&Library{ + err = chasmRegistry.Register(&library{ InvocationTaskHandler: handler, }) require.NoError(t, err) diff --git a/chasm/lib/callback/validator.go b/chasm/lib/callback/validator.go index d9de4ea607b..d9b872b98ce 100644 --- a/chasm/lib/callback/validator.go +++ b/chasm/lib/callback/validator.go @@ -47,9 +47,17 @@ func (v *validator) Validate(_ context.Context, namespaceName string, cbs []*com } for _, cb := range cbs { + if cb == nil { + return serviceerror.NewInvalidArgument("Callback is not set") + } + switch variant := cb.GetVariant().(type) { case *commonpb.Callback_Nexus_: rawURL := variant.Nexus.GetUrl() + if rawURL == "" { + return serviceerror.NewInvalidArgument("Callback URL is not set") + } + if len(rawURL) > v.urlMaxLength(namespaceName) { return serviceerror.NewInvalidArgumentf( "invalid url: url length longer than max length allowed of %d", v.urlMaxLength(namespaceName), diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index df0355886bb..b79d244f926 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -110,7 +110,7 @@ func (w *Workflow) AddCompletionCallbacks( id := fmt.Sprintf("%s-%d", requestID, idx) // Create and add callback - callbackObj := callback.NewCallback(requestID, eventTime, &callbackspb.CallbackState{}, chasmCB) + callbackObj := callback.NewEmbeddedCallback(ctx, requestID, eventTime, chasmCB) w.Callbacks[id] = chasm.NewComponentField(ctx, callbackObj) } return nil diff --git a/client/frontend/client_gen.go b/client/frontend/client_gen.go index c72793f75b2..df0363fd252 100644 --- a/client/frontend/client_gen.go +++ b/client/frontend/client_gen.go @@ -19,6 +19,16 @@ func (c *clientImpl) CountActivityExecutions( return c.client.CountActivityExecutions(ctx, request, opts...) } +func (c *clientImpl) CountCallbackExecutions( + ctx context.Context, + request *workflowservice.CountCallbackExecutionsRequest, + opts ...grpc.CallOption, +) (*workflowservice.CountCallbackExecutionsResponse, error) { + ctx, cancel := c.createContext(ctx) + defer cancel() + return c.client.CountCallbackExecutions(ctx, request, opts...) +} + func (c *clientImpl) CountNexusOperationExecutions( ctx context.Context, request *workflowservice.CountNexusOperationExecutionsRequest, @@ -99,6 +109,16 @@ func (c *clientImpl) DeleteActivityExecution( return c.client.DeleteActivityExecution(ctx, request, opts...) } +func (c *clientImpl) DeleteCallbackExecution( + ctx context.Context, + request *workflowservice.DeleteCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.DeleteCallbackExecutionResponse, error) { + ctx, cancel := c.createContext(ctx) + defer cancel() + return c.client.DeleteCallbackExecution(ctx, request, opts...) +} + func (c *clientImpl) DeleteNexusOperationExecution( ctx context.Context, request *workflowservice.DeleteNexusOperationExecutionRequest, @@ -189,6 +209,16 @@ func (c *clientImpl) DescribeBatchOperation( return c.client.DescribeBatchOperation(ctx, request, opts...) } +func (c *clientImpl) DescribeCallbackExecution( + ctx context.Context, + request *workflowservice.DescribeCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.DescribeCallbackExecutionResponse, error) { + ctx, cancel := c.createContext(ctx) + defer cancel() + return c.client.DescribeCallbackExecution(ctx, request, opts...) +} + func (c *clientImpl) DescribeDeployment( ctx context.Context, request *workflowservice.DescribeDeploymentRequest, @@ -439,6 +469,16 @@ func (c *clientImpl) ListBatchOperations( return c.client.ListBatchOperations(ctx, request, opts...) } +func (c *clientImpl) ListCallbackExecutions( + ctx context.Context, + request *workflowservice.ListCallbackExecutionsRequest, + opts ...grpc.CallOption, +) (*workflowservice.ListCallbackExecutionsResponse, error) { + ctx, cancel := c.createContext(ctx) + defer cancel() + return c.client.ListCallbackExecutions(ctx, request, opts...) +} + func (c *clientImpl) ListClosedWorkflowExecutions( ctx context.Context, request *workflowservice.ListClosedWorkflowExecutionsRequest, @@ -619,6 +659,16 @@ func (c *clientImpl) PollActivityTaskQueue( return c.client.PollActivityTaskQueue(ctx, request, opts...) } +func (c *clientImpl) PollCallbackExecution( + ctx context.Context, + request *workflowservice.PollCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.PollCallbackExecutionResponse, error) { + ctx, cancel := c.createContext(ctx) + defer cancel() + return c.client.PollCallbackExecution(ctx, request, opts...) +} + func (c *clientImpl) PollNexusOperationExecution( ctx context.Context, request *workflowservice.PollNexusOperationExecutionRequest, @@ -989,6 +1039,16 @@ func (c *clientImpl) StartBatchOperation( return c.client.StartBatchOperation(ctx, request, opts...) } +func (c *clientImpl) StartCallbackExecution( + ctx context.Context, + request *workflowservice.StartCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.StartCallbackExecutionResponse, error) { + ctx, cancel := c.createContext(ctx) + defer cancel() + return c.client.StartCallbackExecution(ctx, request, opts...) +} + func (c *clientImpl) StartNexusOperationExecution( ctx context.Context, request *workflowservice.StartNexusOperationExecutionRequest, @@ -1029,6 +1089,16 @@ func (c *clientImpl) TerminateActivityExecution( return c.client.TerminateActivityExecution(ctx, request, opts...) } +func (c *clientImpl) TerminateCallbackExecution( + ctx context.Context, + request *workflowservice.TerminateCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.TerminateCallbackExecutionResponse, error) { + ctx, cancel := c.createContext(ctx) + defer cancel() + return c.client.TerminateCallbackExecution(ctx, request, opts...) +} + func (c *clientImpl) TerminateNexusOperationExecution( ctx context.Context, request *workflowservice.TerminateNexusOperationExecutionRequest, diff --git a/client/frontend/metric_client_gen.go b/client/frontend/metric_client_gen.go index 247c90a2ed9..b87c42e8169 100644 --- a/client/frontend/metric_client_gen.go +++ b/client/frontend/metric_client_gen.go @@ -23,6 +23,20 @@ func (c *metricClient) CountActivityExecutions( return c.client.CountActivityExecutions(ctx, request, opts...) } +func (c *metricClient) CountCallbackExecutions( + ctx context.Context, + request *workflowservice.CountCallbackExecutionsRequest, + opts ...grpc.CallOption, +) (_ *workflowservice.CountCallbackExecutionsResponse, retError error) { + + metricsHandler, startTime := c.startMetricsRecording(ctx, "FrontendClientCountCallbackExecutions") + defer func() { + c.finishMetricsRecording(metricsHandler, startTime, retError) + }() + + return c.client.CountCallbackExecutions(ctx, request, opts...) +} + func (c *metricClient) CountNexusOperationExecutions( ctx context.Context, request *workflowservice.CountNexusOperationExecutionsRequest, @@ -135,6 +149,20 @@ func (c *metricClient) DeleteActivityExecution( return c.client.DeleteActivityExecution(ctx, request, opts...) } +func (c *metricClient) DeleteCallbackExecution( + ctx context.Context, + request *workflowservice.DeleteCallbackExecutionRequest, + opts ...grpc.CallOption, +) (_ *workflowservice.DeleteCallbackExecutionResponse, retError error) { + + metricsHandler, startTime := c.startMetricsRecording(ctx, "FrontendClientDeleteCallbackExecution") + defer func() { + c.finishMetricsRecording(metricsHandler, startTime, retError) + }() + + return c.client.DeleteCallbackExecution(ctx, request, opts...) +} + func (c *metricClient) DeleteNexusOperationExecution( ctx context.Context, request *workflowservice.DeleteNexusOperationExecutionRequest, @@ -261,6 +289,20 @@ func (c *metricClient) DescribeBatchOperation( return c.client.DescribeBatchOperation(ctx, request, opts...) } +func (c *metricClient) DescribeCallbackExecution( + ctx context.Context, + request *workflowservice.DescribeCallbackExecutionRequest, + opts ...grpc.CallOption, +) (_ *workflowservice.DescribeCallbackExecutionResponse, retError error) { + + metricsHandler, startTime := c.startMetricsRecording(ctx, "FrontendClientDescribeCallbackExecution") + defer func() { + c.finishMetricsRecording(metricsHandler, startTime, retError) + }() + + return c.client.DescribeCallbackExecution(ctx, request, opts...) +} + func (c *metricClient) DescribeDeployment( ctx context.Context, request *workflowservice.DescribeDeploymentRequest, @@ -611,6 +653,20 @@ func (c *metricClient) ListBatchOperations( return c.client.ListBatchOperations(ctx, request, opts...) } +func (c *metricClient) ListCallbackExecutions( + ctx context.Context, + request *workflowservice.ListCallbackExecutionsRequest, + opts ...grpc.CallOption, +) (_ *workflowservice.ListCallbackExecutionsResponse, retError error) { + + metricsHandler, startTime := c.startMetricsRecording(ctx, "FrontendClientListCallbackExecutions") + defer func() { + c.finishMetricsRecording(metricsHandler, startTime, retError) + }() + + return c.client.ListCallbackExecutions(ctx, request, opts...) +} + func (c *metricClient) ListClosedWorkflowExecutions( ctx context.Context, request *workflowservice.ListClosedWorkflowExecutionsRequest, @@ -863,6 +919,20 @@ func (c *metricClient) PollActivityTaskQueue( return c.client.PollActivityTaskQueue(ctx, request, opts...) } +func (c *metricClient) PollCallbackExecution( + ctx context.Context, + request *workflowservice.PollCallbackExecutionRequest, + opts ...grpc.CallOption, +) (_ *workflowservice.PollCallbackExecutionResponse, retError error) { + + metricsHandler, startTime := c.startMetricsRecording(ctx, "FrontendClientPollCallbackExecution") + defer func() { + c.finishMetricsRecording(metricsHandler, startTime, retError) + }() + + return c.client.PollCallbackExecution(ctx, request, opts...) +} + func (c *metricClient) PollNexusOperationExecution( ctx context.Context, request *workflowservice.PollNexusOperationExecutionRequest, @@ -1381,6 +1451,20 @@ func (c *metricClient) StartBatchOperation( return c.client.StartBatchOperation(ctx, request, opts...) } +func (c *metricClient) StartCallbackExecution( + ctx context.Context, + request *workflowservice.StartCallbackExecutionRequest, + opts ...grpc.CallOption, +) (_ *workflowservice.StartCallbackExecutionResponse, retError error) { + + metricsHandler, startTime := c.startMetricsRecording(ctx, "FrontendClientStartCallbackExecution") + defer func() { + c.finishMetricsRecording(metricsHandler, startTime, retError) + }() + + return c.client.StartCallbackExecution(ctx, request, opts...) +} + func (c *metricClient) StartNexusOperationExecution( ctx context.Context, request *workflowservice.StartNexusOperationExecutionRequest, @@ -1437,6 +1521,20 @@ func (c *metricClient) TerminateActivityExecution( return c.client.TerminateActivityExecution(ctx, request, opts...) } +func (c *metricClient) TerminateCallbackExecution( + ctx context.Context, + request *workflowservice.TerminateCallbackExecutionRequest, + opts ...grpc.CallOption, +) (_ *workflowservice.TerminateCallbackExecutionResponse, retError error) { + + metricsHandler, startTime := c.startMetricsRecording(ctx, "FrontendClientTerminateCallbackExecution") + defer func() { + c.finishMetricsRecording(metricsHandler, startTime, retError) + }() + + return c.client.TerminateCallbackExecution(ctx, request, opts...) +} + func (c *metricClient) TerminateNexusOperationExecution( ctx context.Context, request *workflowservice.TerminateNexusOperationExecutionRequest, diff --git a/client/frontend/retryable_client_gen.go b/client/frontend/retryable_client_gen.go index 7a5a63cf4de..89b4bc9f5ca 100644 --- a/client/frontend/retryable_client_gen.go +++ b/client/frontend/retryable_client_gen.go @@ -26,6 +26,21 @@ func (c *retryableClient) CountActivityExecutions( return resp, err } +func (c *retryableClient) CountCallbackExecutions( + ctx context.Context, + request *workflowservice.CountCallbackExecutionsRequest, + opts ...grpc.CallOption, +) (*workflowservice.CountCallbackExecutionsResponse, error) { + var resp *workflowservice.CountCallbackExecutionsResponse + op := func(ctx context.Context) error { + var err error + resp, err = c.client.CountCallbackExecutions(ctx, request, opts...) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable) + return resp, err +} + func (c *retryableClient) CountNexusOperationExecutions( ctx context.Context, request *workflowservice.CountNexusOperationExecutionsRequest, @@ -146,6 +161,21 @@ func (c *retryableClient) DeleteActivityExecution( return resp, err } +func (c *retryableClient) DeleteCallbackExecution( + ctx context.Context, + request *workflowservice.DeleteCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.DeleteCallbackExecutionResponse, error) { + var resp *workflowservice.DeleteCallbackExecutionResponse + op := func(ctx context.Context) error { + var err error + resp, err = c.client.DeleteCallbackExecution(ctx, request, opts...) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable) + return resp, err +} + func (c *retryableClient) DeleteNexusOperationExecution( ctx context.Context, request *workflowservice.DeleteNexusOperationExecutionRequest, @@ -281,6 +311,21 @@ func (c *retryableClient) DescribeBatchOperation( return resp, err } +func (c *retryableClient) DescribeCallbackExecution( + ctx context.Context, + request *workflowservice.DescribeCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.DescribeCallbackExecutionResponse, error) { + var resp *workflowservice.DescribeCallbackExecutionResponse + op := func(ctx context.Context) error { + var err error + resp, err = c.client.DescribeCallbackExecution(ctx, request, opts...) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable) + return resp, err +} + func (c *retryableClient) DescribeDeployment( ctx context.Context, request *workflowservice.DescribeDeploymentRequest, @@ -656,6 +701,21 @@ func (c *retryableClient) ListBatchOperations( return resp, err } +func (c *retryableClient) ListCallbackExecutions( + ctx context.Context, + request *workflowservice.ListCallbackExecutionsRequest, + opts ...grpc.CallOption, +) (*workflowservice.ListCallbackExecutionsResponse, error) { + var resp *workflowservice.ListCallbackExecutionsResponse + op := func(ctx context.Context) error { + var err error + resp, err = c.client.ListCallbackExecutions(ctx, request, opts...) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable) + return resp, err +} + func (c *retryableClient) ListClosedWorkflowExecutions( ctx context.Context, request *workflowservice.ListClosedWorkflowExecutionsRequest, @@ -926,6 +986,21 @@ func (c *retryableClient) PollActivityTaskQueue( return resp, err } +func (c *retryableClient) PollCallbackExecution( + ctx context.Context, + request *workflowservice.PollCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.PollCallbackExecutionResponse, error) { + var resp *workflowservice.PollCallbackExecutionResponse + op := func(ctx context.Context) error { + var err error + resp, err = c.client.PollCallbackExecution(ctx, request, opts...) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable) + return resp, err +} + func (c *retryableClient) PollNexusOperationExecution( ctx context.Context, request *workflowservice.PollNexusOperationExecutionRequest, @@ -1481,6 +1556,21 @@ func (c *retryableClient) StartBatchOperation( return resp, err } +func (c *retryableClient) StartCallbackExecution( + ctx context.Context, + request *workflowservice.StartCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.StartCallbackExecutionResponse, error) { + var resp *workflowservice.StartCallbackExecutionResponse + op := func(ctx context.Context) error { + var err error + resp, err = c.client.StartCallbackExecution(ctx, request, opts...) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable) + return resp, err +} + func (c *retryableClient) StartNexusOperationExecution( ctx context.Context, request *workflowservice.StartNexusOperationExecutionRequest, @@ -1541,6 +1631,21 @@ func (c *retryableClient) TerminateActivityExecution( return resp, err } +func (c *retryableClient) TerminateCallbackExecution( + ctx context.Context, + request *workflowservice.TerminateCallbackExecutionRequest, + opts ...grpc.CallOption, +) (*workflowservice.TerminateCallbackExecutionResponse, error) { + var resp *workflowservice.TerminateCallbackExecutionResponse + op := func(ctx context.Context) error { + var err error + resp, err = c.client.TerminateCallbackExecution(ctx, request, opts...) + return err + } + err := backoff.ThrottleRetryContext(ctx, op, c.policy, c.isRetryable) + return resp, err +} + func (c *retryableClient) TerminateNexusOperationExecution( ctx context.Context, request *workflowservice.TerminateNexusOperationExecutionRequest, diff --git a/common/rpc/interceptor/logtags/workflow_service_server_gen.go b/common/rpc/interceptor/logtags/workflow_service_server_gen.go index a03c1e7c1f1..5fb3c1c6f3a 100644 --- a/common/rpc/interceptor/logtags/workflow_service_server_gen.go +++ b/common/rpc/interceptor/logtags/workflow_service_server_gen.go @@ -13,6 +13,10 @@ func (wt *WorkflowTags) extractFromWorkflowServiceServerMessage(message any) []t return nil case *workflowservice.CountActivityExecutionsResponse: return nil + case *workflowservice.CountCallbackExecutionsRequest: + return nil + case *workflowservice.CountCallbackExecutionsResponse: + return nil case *workflowservice.CountNexusOperationExecutionsRequest: return nil case *workflowservice.CountNexusOperationExecutionsResponse: @@ -48,6 +52,12 @@ func (wt *WorkflowTags) extractFromWorkflowServiceServerMessage(message any) []t } case *workflowservice.DeleteActivityExecutionResponse: return nil + case *workflowservice.DeleteCallbackExecutionRequest: + return []tag.Tag{ + tag.WorkflowRunID(r.GetRunId()), + } + case *workflowservice.DeleteCallbackExecutionResponse: + return nil case *workflowservice.DeleteNexusOperationExecutionRequest: return []tag.Tag{ tag.OperationID(r.GetOperationId()), @@ -95,6 +105,12 @@ func (wt *WorkflowTags) extractFromWorkflowServiceServerMessage(message any) []t return nil case *workflowservice.DescribeBatchOperationResponse: return nil + case *workflowservice.DescribeCallbackExecutionRequest: + return []tag.Tag{ + tag.WorkflowRunID(r.GetRunId()), + } + case *workflowservice.DescribeCallbackExecutionResponse: + return nil case *workflowservice.DescribeDeploymentRequest: return nil case *workflowservice.DescribeDeploymentResponse: @@ -209,6 +225,10 @@ func (wt *WorkflowTags) extractFromWorkflowServiceServerMessage(message any) []t return nil case *workflowservice.ListBatchOperationsResponse: return nil + case *workflowservice.ListCallbackExecutionsRequest: + return nil + case *workflowservice.ListCallbackExecutionsResponse: + return nil case *workflowservice.ListClosedWorkflowExecutionsRequest: return nil case *workflowservice.ListClosedWorkflowExecutionsResponse: @@ -299,6 +319,14 @@ func (wt *WorkflowTags) extractFromWorkflowServiceServerMessage(message any) []t tag.WorkflowID(r.GetWorkflowExecution().GetWorkflowId()), tag.WorkflowRunID(r.GetWorkflowExecution().GetRunId()), } + case *workflowservice.PollCallbackExecutionRequest: + return []tag.Tag{ + tag.WorkflowRunID(r.GetRunId()), + } + case *workflowservice.PollCallbackExecutionResponse: + return []tag.Tag{ + tag.WorkflowRunID(r.GetRunId()), + } case *workflowservice.PollNexusOperationExecutionRequest: return []tag.Tag{ tag.OperationID(r.GetOperationId()), @@ -515,6 +543,12 @@ func (wt *WorkflowTags) extractFromWorkflowServiceServerMessage(message any) []t return nil case *workflowservice.StartBatchOperationResponse: return nil + case *workflowservice.StartCallbackExecutionRequest: + return nil + case *workflowservice.StartCallbackExecutionResponse: + return []tag.Tag{ + tag.WorkflowRunID(r.GetRunId()), + } case *workflowservice.StartNexusOperationExecutionRequest: return []tag.Tag{ tag.OperationID(r.GetOperationId()), @@ -542,6 +576,12 @@ func (wt *WorkflowTags) extractFromWorkflowServiceServerMessage(message any) []t } case *workflowservice.TerminateActivityExecutionResponse: return nil + case *workflowservice.TerminateCallbackExecutionRequest: + return []tag.Tag{ + tag.WorkflowRunID(r.GetRunId()), + } + case *workflowservice.TerminateCallbackExecutionResponse: + return nil case *workflowservice.TerminateNexusOperationExecutionRequest: return []tag.Tag{ tag.OperationID(r.GetOperationId()), diff --git a/common/testing/mockapi/workflowservicemock/v1/service_grpc.pb.mock.go b/common/testing/mockapi/workflowservicemock/v1/service_grpc.pb.mock.go index 34a676ad5a9..42996b2cb90 100644 --- a/common/testing/mockapi/workflowservicemock/v1/service_grpc.pb.mock.go +++ b/common/testing/mockapi/workflowservicemock/v1/service_grpc.pb.mock.go @@ -62,6 +62,26 @@ func (mr *MockWorkflowServiceClientMockRecorder) CountActivityExecutions(ctx, in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountActivityExecutions", reflect.TypeOf((*MockWorkflowServiceClient)(nil).CountActivityExecutions), varargs...) } +// CountCallbackExecutions mocks base method. +func (m *MockWorkflowServiceClient) CountCallbackExecutions(ctx context.Context, in *workflowservice.CountCallbackExecutionsRequest, opts ...grpc.CallOption) (*workflowservice.CountCallbackExecutionsResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CountCallbackExecutions", varargs...) + ret0, _ := ret[0].(*workflowservice.CountCallbackExecutionsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CountCallbackExecutions indicates an expected call of CountCallbackExecutions. +func (mr *MockWorkflowServiceClientMockRecorder) CountCallbackExecutions(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountCallbackExecutions", reflect.TypeOf((*MockWorkflowServiceClient)(nil).CountCallbackExecutions), varargs...) +} + // CountNexusOperationExecutions mocks base method. func (m *MockWorkflowServiceClient) CountNexusOperationExecutions(ctx context.Context, in *workflowservice.CountNexusOperationExecutionsRequest, opts ...grpc.CallOption) (*workflowservice.CountNexusOperationExecutionsResponse, error) { m.ctrl.T.Helper() @@ -222,6 +242,26 @@ func (mr *MockWorkflowServiceClientMockRecorder) DeleteActivityExecution(ctx, in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteActivityExecution", reflect.TypeOf((*MockWorkflowServiceClient)(nil).DeleteActivityExecution), varargs...) } +// DeleteCallbackExecution mocks base method. +func (m *MockWorkflowServiceClient) DeleteCallbackExecution(ctx context.Context, in *workflowservice.DeleteCallbackExecutionRequest, opts ...grpc.CallOption) (*workflowservice.DeleteCallbackExecutionResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DeleteCallbackExecution", varargs...) + ret0, _ := ret[0].(*workflowservice.DeleteCallbackExecutionResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteCallbackExecution indicates an expected call of DeleteCallbackExecution. +func (mr *MockWorkflowServiceClientMockRecorder) DeleteCallbackExecution(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteCallbackExecution", reflect.TypeOf((*MockWorkflowServiceClient)(nil).DeleteCallbackExecution), varargs...) +} + // DeleteNexusOperationExecution mocks base method. func (m *MockWorkflowServiceClient) DeleteNexusOperationExecution(ctx context.Context, in *workflowservice.DeleteNexusOperationExecutionRequest, opts ...grpc.CallOption) (*workflowservice.DeleteNexusOperationExecutionResponse, error) { m.ctrl.T.Helper() @@ -402,6 +442,26 @@ func (mr *MockWorkflowServiceClientMockRecorder) DescribeBatchOperation(ctx, in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeBatchOperation", reflect.TypeOf((*MockWorkflowServiceClient)(nil).DescribeBatchOperation), varargs...) } +// DescribeCallbackExecution mocks base method. +func (m *MockWorkflowServiceClient) DescribeCallbackExecution(ctx context.Context, in *workflowservice.DescribeCallbackExecutionRequest, opts ...grpc.CallOption) (*workflowservice.DescribeCallbackExecutionResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeCallbackExecution", varargs...) + ret0, _ := ret[0].(*workflowservice.DescribeCallbackExecutionResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeCallbackExecution indicates an expected call of DescribeCallbackExecution. +func (mr *MockWorkflowServiceClientMockRecorder) DescribeCallbackExecution(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeCallbackExecution", reflect.TypeOf((*MockWorkflowServiceClient)(nil).DescribeCallbackExecution), varargs...) +} + // DescribeDeployment mocks base method. func (m *MockWorkflowServiceClient) DescribeDeployment(ctx context.Context, in *workflowservice.DescribeDeploymentRequest, opts ...grpc.CallOption) (*workflowservice.DescribeDeploymentResponse, error) { m.ctrl.T.Helper() @@ -902,6 +962,26 @@ func (mr *MockWorkflowServiceClientMockRecorder) ListBatchOperations(ctx, in any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListBatchOperations", reflect.TypeOf((*MockWorkflowServiceClient)(nil).ListBatchOperations), varargs...) } +// ListCallbackExecutions mocks base method. +func (m *MockWorkflowServiceClient) ListCallbackExecutions(ctx context.Context, in *workflowservice.ListCallbackExecutionsRequest, opts ...grpc.CallOption) (*workflowservice.ListCallbackExecutionsResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListCallbackExecutions", varargs...) + ret0, _ := ret[0].(*workflowservice.ListCallbackExecutionsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListCallbackExecutions indicates an expected call of ListCallbackExecutions. +func (mr *MockWorkflowServiceClientMockRecorder) ListCallbackExecutions(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListCallbackExecutions", reflect.TypeOf((*MockWorkflowServiceClient)(nil).ListCallbackExecutions), varargs...) +} + // ListClosedWorkflowExecutions mocks base method. func (m *MockWorkflowServiceClient) ListClosedWorkflowExecutions(ctx context.Context, in *workflowservice.ListClosedWorkflowExecutionsRequest, opts ...grpc.CallOption) (*workflowservice.ListClosedWorkflowExecutionsResponse, error) { m.ctrl.T.Helper() @@ -1262,6 +1342,26 @@ func (mr *MockWorkflowServiceClientMockRecorder) PollActivityTaskQueue(ctx, in a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PollActivityTaskQueue", reflect.TypeOf((*MockWorkflowServiceClient)(nil).PollActivityTaskQueue), varargs...) } +// PollCallbackExecution mocks base method. +func (m *MockWorkflowServiceClient) PollCallbackExecution(ctx context.Context, in *workflowservice.PollCallbackExecutionRequest, opts ...grpc.CallOption) (*workflowservice.PollCallbackExecutionResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "PollCallbackExecution", varargs...) + ret0, _ := ret[0].(*workflowservice.PollCallbackExecutionResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PollCallbackExecution indicates an expected call of PollCallbackExecution. +func (mr *MockWorkflowServiceClientMockRecorder) PollCallbackExecution(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PollCallbackExecution", reflect.TypeOf((*MockWorkflowServiceClient)(nil).PollCallbackExecution), varargs...) +} + // PollNexusOperationExecution mocks base method. func (m *MockWorkflowServiceClient) PollNexusOperationExecution(ctx context.Context, in *workflowservice.PollNexusOperationExecutionRequest, opts ...grpc.CallOption) (*workflowservice.PollNexusOperationExecutionResponse, error) { m.ctrl.T.Helper() @@ -2002,6 +2102,26 @@ func (mr *MockWorkflowServiceClientMockRecorder) StartBatchOperation(ctx, in any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartBatchOperation", reflect.TypeOf((*MockWorkflowServiceClient)(nil).StartBatchOperation), varargs...) } +// StartCallbackExecution mocks base method. +func (m *MockWorkflowServiceClient) StartCallbackExecution(ctx context.Context, in *workflowservice.StartCallbackExecutionRequest, opts ...grpc.CallOption) (*workflowservice.StartCallbackExecutionResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "StartCallbackExecution", varargs...) + ret0, _ := ret[0].(*workflowservice.StartCallbackExecutionResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StartCallbackExecution indicates an expected call of StartCallbackExecution. +func (mr *MockWorkflowServiceClientMockRecorder) StartCallbackExecution(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartCallbackExecution", reflect.TypeOf((*MockWorkflowServiceClient)(nil).StartCallbackExecution), varargs...) +} + // StartNexusOperationExecution mocks base method. func (m *MockWorkflowServiceClient) StartNexusOperationExecution(ctx context.Context, in *workflowservice.StartNexusOperationExecutionRequest, opts ...grpc.CallOption) (*workflowservice.StartNexusOperationExecutionResponse, error) { m.ctrl.T.Helper() @@ -2082,6 +2202,26 @@ func (mr *MockWorkflowServiceClientMockRecorder) TerminateActivityExecution(ctx, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateActivityExecution", reflect.TypeOf((*MockWorkflowServiceClient)(nil).TerminateActivityExecution), varargs...) } +// TerminateCallbackExecution mocks base method. +func (m *MockWorkflowServiceClient) TerminateCallbackExecution(ctx context.Context, in *workflowservice.TerminateCallbackExecutionRequest, opts ...grpc.CallOption) (*workflowservice.TerminateCallbackExecutionResponse, error) { + m.ctrl.T.Helper() + varargs := []any{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "TerminateCallbackExecution", varargs...) + ret0, _ := ret[0].(*workflowservice.TerminateCallbackExecutionResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// TerminateCallbackExecution indicates an expected call of TerminateCallbackExecution. +func (mr *MockWorkflowServiceClientMockRecorder) TerminateCallbackExecution(ctx, in any, opts ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateCallbackExecution", reflect.TypeOf((*MockWorkflowServiceClient)(nil).TerminateCallbackExecution), varargs...) +} + // TerminateNexusOperationExecution mocks base method. func (m *MockWorkflowServiceClient) TerminateNexusOperationExecution(ctx context.Context, in *workflowservice.TerminateNexusOperationExecutionRequest, opts ...grpc.CallOption) (*workflowservice.TerminateNexusOperationExecutionResponse, error) { m.ctrl.T.Helper() diff --git a/go.mod b/go.mod index dbe232a6248..6e6dbdca09f 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 - go.temporal.io/api v1.62.12-0.20260430203359-15c391664683 + go.temporal.io/api v1.62.12-0.20260506203937-27ab43932052 // DO NOT SUBMIT -- Branch chrsmith/standalone-callbacks go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 go.temporal.io/sdk v1.41.1 go.uber.org/fx v1.24.0 diff --git a/go.sum b/go.sum index 957bed1529f..9d452e6683c 100644 --- a/go.sum +++ b/go.sum @@ -469,8 +469,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.62.12-0.20260430203359-15c391664683 h1:GtwQjX9hN0pRjuneBpl/xvcu9Xl9llAt4GjKrlpP0sg= -go.temporal.io/api v1.62.12-0.20260430203359-15c391664683/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= +go.temporal.io/api v1.62.12-0.20260506203937-27ab43932052 h1:zTB6uMLzdBsLksMH073JTuLfcS0S53+Bm5Kxestwnz4= +go.temporal.io/api v1.62.12-0.20260506203937-27ab43932052/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 h1:1hKeH3GyR6YD6LKMHGCZ76t6h1Sgha0hXVQBxWi3dlQ= go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2/go.mod h1:T8dnzVPeO+gaUTj9eDgm/lT2lZH4+JXNvrGaQGyVi50= go.temporal.io/sdk v1.41.1 h1:yOpvsHyDD1lNuwlGBv/SUodCPhjv9nDeC9lLHW/fJUA= diff --git a/service/frontend/configs/quotas.go b/service/frontend/configs/quotas.go index 4d8dcc42c3a..ba8bd19806f 100644 --- a/service/frontend/configs/quotas.go +++ b/service/frontend/configs/quotas.go @@ -92,6 +92,7 @@ var ( "/temporal.api.workflowservice.v1.WorkflowService/CreateSchedule": 1, "/temporal.api.workflowservice.v1.WorkflowService/StartBatchOperation": 1, "/temporal.api.workflowservice.v1.WorkflowService/StartActivityExecution": 1, + "/temporal.api.workflowservice.v1.WorkflowService/StartCallbackExecution": 1, "/temporal.api.workflowservice.v1.WorkflowService/StartNexusOperationExecution": 1, DispatchNexusTaskByNamespaceAndTaskQueueAPIName: 1, DispatchNexusTaskByEndpointAPIName: 1, @@ -145,7 +146,9 @@ var ( "/temporal.api.workflowservice.v1.WorkflowService/UpdateTaskQueueConfig": 2, "/temporal.api.workflowservice.v1.WorkflowService/RequestCancelActivityExecution": 2, "/temporal.api.workflowservice.v1.WorkflowService/TerminateActivityExecution": 2, + "/temporal.api.workflowservice.v1.WorkflowService/TerminateCallbackExecution": 2, "/temporal.api.workflowservice.v1.WorkflowService/DeleteActivityExecution": 2, + "/temporal.api.workflowservice.v1.WorkflowService/DeleteCallbackExecution": 2, "/temporal.api.workflowservice.v1.WorkflowService/RequestCancelNexusOperationExecution": 2, "/temporal.api.workflowservice.v1.WorkflowService/TerminateNexusOperationExecution": 2, "/temporal.api.workflowservice.v1.WorkflowService/DeleteNexusOperationExecution": 2, @@ -155,6 +158,7 @@ var ( // P3: Status Querying APIs "/temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution": 3, "/temporal.api.workflowservice.v1.WorkflowService/DescribeActivityExecution": 3, + "/temporal.api.workflowservice.v1.WorkflowService/DescribeCallbackExecution": 3, "/temporal.api.workflowservice.v1.WorkflowService/DescribeTaskQueue": 3, "/temporal.api.workflowservice.v1.WorkflowService/GetWorkerBuildIdCompatibility": 3, "/temporal.api.workflowservice.v1.WorkflowService/GetWorkerVersioningRules": 3, @@ -181,7 +185,8 @@ var ( // P4: Poll APIs and other low priority APIs "/temporal.api.workflowservice.v1.WorkflowService/PollNexusOperationExecution": 4, - "/temporal.api.workflowservice.v1.WorkflowService/PollActivityExecution": 4, // TODO(saa-preview): should it be 4 or 3? + "/temporal.api.workflowservice.v1.WorkflowService/PollActivityExecution": 4, // TODO(saa-preview): Should it be 4 or 3? Same PollCallback. + "/temporal.api.workflowservice.v1.WorkflowService/PollCallbackExecution": 4, "/temporal.api.workflowservice.v1.WorkflowService/PollWorkflowTaskQueue": 4, "/temporal.api.workflowservice.v1.WorkflowService/PollActivityTaskQueue": 4, "/temporal.api.workflowservice.v1.WorkflowService/PollWorkflowExecutionUpdate": 4, @@ -216,7 +221,9 @@ var ( "/temporal.api.workflowservice.v1.WorkflowService/ListWorkers": 1, "/temporal.api.workflowservice.v1.WorkflowService/DescribeWorker": 1, "/temporal.api.workflowservice.v1.WorkflowService/CountActivityExecutions": 1, + "/temporal.api.workflowservice.v1.WorkflowService/CountCallbackExecutions": 1, "/temporal.api.workflowservice.v1.WorkflowService/ListActivityExecutions": 1, + "/temporal.api.workflowservice.v1.WorkflowService/ListCallbackExecutions": 1, "/temporal.api.workflowservice.v1.WorkflowService/CountNexusOperationExecutions": 1, "/temporal.api.workflowservice.v1.WorkflowService/ListNexusOperationExecutions": 1, diff --git a/service/frontend/configs/quotas_test.go b/service/frontend/configs/quotas_test.go index 173b28f2cda..615ac6eb974 100644 --- a/service/frontend/configs/quotas_test.go +++ b/service/frontend/configs/quotas_test.go @@ -106,6 +106,8 @@ func (s *quotasSuite) TestVisibilityAPIs() { "/temporal.api.workflowservice.v1.WorkflowService/CountActivityExecutions": {}, "/temporal.api.workflowservice.v1.WorkflowService/ListActivityExecutions": {}, + "/temporal.api.workflowservice.v1.WorkflowService/CountCallbackExecutions": {}, + "/temporal.api.workflowservice.v1.WorkflowService/ListCallbackExecutions": {}, "/temporal.api.workflowservice.v1.WorkflowService/CountNexusOperationExecutions": {}, "/temporal.api.workflowservice.v1.WorkflowService/ListNexusOperationExecutions": {}, } diff --git a/service/frontend/fx.go b/service/frontend/fx.go index a40077a8aa8..45dd0127083 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -130,6 +130,7 @@ var Module = fx.Options( chasmnexus.Module, chasmworkflow.Module, activity.FrontendModule, + callback.FrontendModule, fx.Provide(visibility.ChasmVisibilityManagerProvider), fx.Provide(chasm.ChasmVisibilityInterceptorProvider), ) @@ -888,6 +889,7 @@ func HandlerProvider( healthInterceptor *interceptor.HealthInterceptor, scheduleSpecBuilder *scheduler.SpecBuilder, activityHandler activity.FrontendHandler, + callbackHandler callback.FrontendHandler, callbackValidator callback.Validator, nexusOperationHandler chasmnexus.FrontendHandler, registry *chasm.Registry, @@ -927,6 +929,7 @@ func HandlerProvider( scheduleSpecBuilder, httpEnabled(cfg, serviceName), activityHandler, + callbackHandler, nexusOperationHandler, registry, workerDeploymentReadRateLimiter, diff --git a/service/frontend/service.go b/service/frontend/service.go index ac561bec1a1..954385c7e3b 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -10,6 +10,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/adminservice/v1" "go.temporal.io/server/chasm/lib/activity" + "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" @@ -237,6 +238,7 @@ type Config struct { // CHASM archetypes Activity *activity.Config + Callback *callback.Config } // IsExperimentAllowed checks if an experiment is enabled for a given namespace in the dynamic config. @@ -404,7 +406,9 @@ func NewConfig( HTTPAllowedHosts: dynamicconfig.FrontendHTTPAllowedHosts.Get(dc), AllowedExperiments: dynamicconfig.FrontendAllowedExperiments.Get(dc), + // CHASM component configurations. Activity: activity.ConfigProvider(dc), + Callback: callback.ConfigProvider(dc), } } diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index d98a304e687..f331f5580ad 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -111,15 +111,18 @@ const ( ) type ( - // ActivityHandler is the activity frontend handler, aliased to avoid embedding name collision. - ActivityHandler = activity.FrontendHandler - // NexusOperationHandler is the nexus operation frontend handler, aliased to avoid embedding name collision. + // Aliases for CHASM components to avoid name collisions. + + ActivityHandler = activity.FrontendHandler + CallbackHandler = callback.FrontendHandler NexusOperationHandler = chasmnexus.FrontendHandler // WorkflowHandler - gRPC handler interface for workflowservice WorkflowHandler struct { workflowservice.UnsafeWorkflowServiceServer + ActivityHandler + CallbackHandler NexusOperationHandler status int32 @@ -330,12 +333,14 @@ func NewWorkflowHandler( scheduleSpecBuilder *scheduler.SpecBuilder, httpEnabled bool, activityHandler activity.FrontendHandler, + callbackHandler callback.FrontendHandler, nexusOperationHandler chasmnexus.FrontendHandler, registry *chasm.Registry, workerDeploymentReadRateLimiter quotas.RequestRateLimiter, ) *WorkflowHandler { handler := &WorkflowHandler{ ActivityHandler: activityHandler, + CallbackHandler: callbackHandler, NexusOperationHandler: nexusOperationHandler, status: common.DaemonStatusInitialized, callbackValidator: callbackValidator, diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index 5ed7bcba34b..1ddf147d24b 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -210,6 +210,7 @@ func (s *WorkflowHandlerSuite) getWorkflowHandler(config *Config) *WorkflowHandl scheduler.NewSpecBuilder(), true, nil, // Not testing activity handler here + nil, // Not testing callback handler here nexusoperation.NewFrontendHandler( nil, nil, diff --git a/service/history/fx.go b/service/history/fx.go index 313f29fd45d..2b89ed4ee88 100644 --- a/service/history/fx.go +++ b/service/history/fx.go @@ -8,6 +8,7 @@ import ( "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity" + "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" @@ -101,6 +102,7 @@ var Module = fx.Options( hsmnexusoperations.Module, fx.Invoke(hsmnexusworkflow.RegisterCommandHandlers), activity.HistoryModule, + callback.HistoryModule, chasmnexus.Module, chasmworkflow.Module, ) diff --git a/temporal/fx.go b/temporal/fx.go index 8905c1b5567..1b35c13493b 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -21,7 +21,6 @@ import ( "go.temporal.io/api/serviceerror" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" - chasmcallback "go.temporal.io/server/chasm/lib/callback" chasmscheduler "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/client" "go.temporal.io/server/common/archiver" @@ -155,7 +154,6 @@ var ( ChasmLibraryOptions = fx.Options( chasm.Module, chasmscheduler.Module, - chasmcallback.Module, ) ) diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go new file mode 100644 index 00000000000..ded87ac6215 --- /dev/null +++ b/tests/standalone_callbacks_test.go @@ -0,0 +1,1228 @@ +package tests + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" + + "github.com/google/uuid" + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + callbackpb "go.temporal.io/api/callback/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + historypb "go.temporal.io/api/history/v1" + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/api/operatorservice/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/chasm/lib/callback" + "go.temporal.io/server/common/dynamicconfig" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexustest" + hsmcallbacks "go.temporal.io/server/components/callbacks" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" +) + +// Test suite for the Nexus "Standalone Callbacks". Which are Nexus operations corresponding to +// aysynchronous actions that take place outside of Temporal. (e.g. waiting for a payment to +// be processed, or webhook to be delivered, etc.) + +// Minimal information that an external service would need to report the results of a callback. +type externalRequestInfo struct { + Namespace string + Token string + URL string + + // Result of the callback, success or failure. + Result *callbackpb.CallbackExecutionCompletion +} + +// Result of the fake service after receiving a request. If the `StartCallbackExecution` request +// was accepted, reports the CallbackID, RunID. Otherwise the error. +type externalRequestResult struct { + CallbackID string + + // Mutually exclusive with Error. + RunID string + Error error +} + +// fakeExternalService simulates a service doing work asynchronously outside of Temporal. +// Test cases will create a Nexus handler that will start a Nexus operation, and then +// pass the context information to the fake service (via externalRequestInfo). The fake +// service will then notify Temporal the work is done (via StartCallbackExecution), and +// then put metadata into the `requestResults` channel. +type fakeExternalService struct { + incommingRequests chan<- externalRequestInfo + requestResults <-chan externalRequestResult +} + +// startFakeExternalService starts a new fake service Goroutine, using the supplied client. +// Will shut down and close its channels when the given context is complete. +func startFakeExternalService(ctx context.Context, c workflowservice.WorkflowServiceClient) *fakeExternalService { + // Channels are buffered so that tests don't block on reads/writes. + input := make(chan externalRequestInfo, 4) + output := make(chan externalRequestResult, 4) + + // Logic of the actual faux return, processing requests. Reads requests to do work (from + // a Nexus handler), and then reports the results out-of-band from the Nexus operation. + go func() { + defer close(input) + defer close(output) + + for { + select { + case incommingRequest := <-input: + // Uniquely identify the callback execution. + callbackID := "faux-svc-callback-" + uuid.NewString() + + targetCallback := &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: incommingRequest.URL, + Token: incommingRequest.Token, + }, + }, + } + + resp, err := c.StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ + Namespace: incommingRequest.Namespace, + Identity: "faux-external-service", + RequestId: uuid.NewString(), + CallbackId: callbackID, + Callback: targetCallback, + Input: &workflowservice.StartCallbackExecutionRequest_Completion{ + Completion: incommingRequest.Result, + }, + ScheduleToCloseTimeout: durationpb.New(10 * time.Second), + }) + + // Make the result available to the testcase. + output <- externalRequestResult{ + CallbackID: callbackID, + RunID: resp.GetRunId(), + Error: err, + } + case <-ctx.Done(): + return + } + } + }() + + return &fakeExternalService{ + incommingRequests: input, + requestResults: output, + } +} + +func TestStandaloneCallbackSuite(t *testing.T) { + t.Parallel() + suite.Run(t, new(StandaloneCallbackSuite)) +} + +type StandaloneCallbackSuite struct { + testcore.FunctionalTestBase +} + +// Expose a helper to enable "require" assertions, to fail the test immediately instead of +// allowing cascading failures. e.g. s.Require().NotNil(err, "unable to do critical thing") +func (s *StandaloneCallbackSuite) Require() *require.Assertions { + return require.New(s.T()) +} + +func (s *StandaloneCallbackSuite) SetupSuite() { + s.SetupSuiteWithCluster( + testcore.WithDynamicConfigOverrides(map[dynamicconfig.Key]any{ + dynamicconfig.EnableChasm.Key(): true, + dynamicconfig.EnableCHASMCallbacks.Key(): true, + callback.EnableStandaloneExecutions.Key(): true, + + // QUIRK: The configuration for setting the callback allow list + // is in the HSM code. + hsmcallbacks.AllowedAddresses.Key(): []any{ + map[string]any{"Pattern": "*", "AllowInsecure": true}, + }, + }), + ) +} + +// Calls StartCallbackExecution, reporting the result of a callback that doesn't exist. +// +// This isn't expected to return an error, since the API is written to support the external +// operation having completed before the Temporal/Nexus side registration has completed. +// +// However, since the Nexus callback isn't even registered, the callback execution will +// aways result in timing out. +func (s *StandaloneCallbackSuite) callStartCallbackExecutionToBogusCallback( + ctx context.Context, + callbackID string, + timeout time.Duration, +) *workflowservice.StartCallbackExecutionResponse { + s.T().Helper() + cb := &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "http://localhost:1/nonexistent", + }, + }, + } + + completion := &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), "some-result"), + }, + } + + return s.callStartCallbackExecution(ctx, callbackID, cb, completion, timeout) +} + +// Call the StartCallbackExecution API with the given parameters. +func (s *StandaloneCallbackSuite) callStartCallbackExecution( + ctx context.Context, + callbackID string, + cb *commonpb.Callback, + completion *callbackpb.CallbackExecutionCompletion, + timeout time.Duration, +) *workflowservice.StartCallbackExecutionResponse { + s.T().Helper() + + resp, err := s.FrontendClient().StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + Identity: "startCallbackExecution", + RequestId: uuid.NewString(), + CallbackId: callbackID, + Callback: cb, + Input: &workflowservice.StartCallbackExecutionRequest_Completion{ + Completion: completion, + }, + ScheduleToCloseTimeout: durationpb.New(timeout), + }) + s.Require().NoError(err, "Error calling StartCallbackExecution to bogus callback") + + return resp +} + +// TestBasicaOperation tests that a Nexus operation started by a workflow can be completed using the +// StartCallbackExecution API to deliver the Nexus completion to the operation's callback URL. +// +// Flow: +// 1. A caller workflow (`callerWf`) starts a Nexus operation via an external endpoint. +// 2. The Nexus handler (`nexusHandler`) starts the operation asynchronously and captures +// the callback URL and token. +// 3. The Nexus handler passes the data to a fake external service (`fakeSvc`), which then +// StartCallbackExecution API with a successful payload. +// 4. The CHASM callback execution delivers the Nexus completion to the callback URL. +// 5. The caller workflow receives the operation's result and completes. +// +// The test is ran in two variants. First, where the out-of-band service reports a successful +// compoetion result. The second reports a failure. Causing the calling workflow to fail. +func (s *StandaloneCallbackSuite) TestBasicOperation() { + + // Implementation of the test scenario, standing up the workflow, Nexus operation, + // external service, etc. + // + // Takes a CallbackExecutionCompletion to be reported by the external service, and a + // verification function to test the calling Workflow behaved as expected. + runStandaloneCallbackScenario := func( + ctx context.Context, + completionResult *callbackpb.CallbackExecutionCompletion, + workflowRunVerificationFn func(client.WorkflowRun), + ) { + taskQueue := testcore.RandomizeStr(s.T().Name()) + + // Fake External Service + // + // Start a fake external service to handle async requests, and report + // their results to Temporal. + fakeSvc := startFakeExternalService(ctx, s.FrontendClient()) + + // Nexus Handler + // + // Set up an external Nexus handler that starts operations asynchronously + // and sends the callback URL and token to the fake service. The Nexus + // handler terminates, while the fake service reports results out-of-band. + nexusEndpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) + const ( + nexusSvcName = "nexus-service" + nexusSvcOp = "nexus-operation" + ) + + nexusHandler := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + s.Equal(nexusSvcName, service) + s.Equal(nexusSvcOp, operation) + + // Send the request to the external service to do the work. + fakeSvc.incommingRequests <- externalRequestInfo{ + Namespace: s.Namespace().String(), + Token: options.CallbackHeader.Get(commonnexus.CallbackTokenHeader), + URL: options.CallbackURL, + + Result: completionResult, + } + + // End the Nexus operation. + return &nexus.HandlerStartOperationResultAsync{ + OperationToken: fmt.Sprintf("operation-token-%s", uuid.NewString()), + }, nil + }, + } + + // Start the Nexus server. + listenAddr := nexustest.AllocListenAddress() + nexustest.NewNexusServer(s.T(), listenAddr, nexusHandler) + + // Register the Nexus endpoint with the Temporal service. + createNexusEndpointReq := &operatorservice.CreateNexusEndpointRequest{ + Spec: &nexuspb.EndpointSpec{ + Name: nexusEndpointName, + Target: &nexuspb.EndpointTarget{ + Variant: &nexuspb.EndpointTarget_External_{ + External: &nexuspb.EndpointTarget_External{ + Url: "http://" + listenAddr, + }, + }, + }, + }, + } + _, err := s.OperatorClient().CreateNexusEndpoint(ctx, createNexusEndpointReq) + s.Require().NoError(err, "Error registering Nexus endpoint") + + // Calling Workflow + // + // This will create the Nexus operation, invoking the Handler workflow. The + // workflow will then block until the Nexus operation completes, which will + // not be until the fake external service has reported the callback's result. + callerWf := func(ctx workflow.Context) (string, error) { + c := workflow.NewNexusClient(nexusEndpointName, nexusSvcName) + fut := c.ExecuteOperation(ctx, nexusSvcOp, "input", workflow.NexusOperationOptions{}) + + var nexusOpResult string + err := fut.Get(ctx, &nexusOpResult) + return nexusOpResult, err + } + + // Run the Test + // + // Construct and start the calling workflow Worker. Then wait for completion. + callerWfWorker := worker.New(s.SdkClient(), taskQueue, worker.Options{}) + callerWfWorker.RegisterWorkflow(callerWf) + s.Require().NoError(callerWfWorker.Start(), "Error starting calling workflow Worker") + defer callerWfWorker.Stop() + + // Start + startOpts := client.StartWorkflowOptions{ + TaskQueue: taskQueue, + } + callerWfRun, err := s.SdkClient().ExecuteWorkflow(ctx, startOpts, callerWf) + s.Require().NoError(err, "Error running the caller Workflow") + + // Defer to a caller-supplied verification function to wait on the caller + // workflow's result and verify the success/failure as applicable. + workflowRunVerificationFn(callerWfRun) + + // If the fake service was called correctly, we expect to see the result + // of it doing the out-of-band work. + fakeSvcResult := <-fakeSvc.requestResults + s.Require().NoError(fakeSvcResult.Error) + + // Additional Verification + // + // Use the Describe and Poll APIs to fetch the callback execution. + descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: fakeSvcResult.CallbackID, + IncludeInput: true, + IncludeOutcome: false, + }) + s.Require().NoError(err, "Error describing the callback execution") + s.True(proto.Equal(completionResult, descResp.Input)) + s.Nil(descResp.Outcome) + + gotInfo := descResp.GetInfo() + s.Require().NotNil(gotInfo, "Got nil Info in response") + s.Equal(fakeSvcResult.CallbackID, gotInfo.GetCallbackId()) + s.NotNil(gotInfo.GetCreateTime()) + + // Confirm the outcome of the callback was a successful delivery. + // (Even if it was for a failed completion.) + s.Equal(enumspb.CALLBACK_EXECUTION_STATUS_SUCCEEDED, gotInfo.GetStatus()) + s.Equal(enumspb.CALLBACK_STATE_SUCCEEDED, gotInfo.GetState()) + + // Poll to verify the outcome as well. + pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: fakeSvcResult.CallbackID, + }) + s.Require().NoError(err, "Error polling completed callback execution") + + outcome := pollResp.GetOutcome() + s.Require().NotNil(outcome, "Got nil Outcome") + + // Confirm the outcome of the callback was a successful delivery. + // (Even if it was for a failed completion.) + s.NotNil(outcome.GetSuccess()) + s.Nil(outcome.GetFailure()) + } + + s.Run("success", func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + wantPayloadStr := "successfully delivered payload via external svc" + successCompletion := &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), wantPayloadStr), + }, + } + + verifyWorkflowRunFn := func(workflowRun client.WorkflowRun) { + var gotPayload string + s.NoError(workflowRun.Get(ctx, &gotPayload)) + s.Equal(wantPayloadStr, gotPayload) + } + + runStandaloneCallbackScenario(ctx, successCompletion, verifyWorkflowRunFn) + }) + + s.Run("failure", func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + failureCompletion := &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Failure{ + Failure: &failurepb.Failure{ + Message: "operation failed from standalone callback", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + NonRetryable: true, + }, + }, + }, + }, + } + + verifyWorkflowRunFn := func(workflowRun client.WorkflowRun) { + // The workflow should fail with a NexusOperationError wrapping the failure. + var unusedResult string + err := workflowRun.Get(ctx, &unusedResult) + + // Confirm the (super-long) error contains the key information. + s.ErrorContains(err, "workflow execution error") + s.ErrorContains(err, workflowRun.GetRunID()) + s.ErrorContains(err, "nexus operation completed unsuccessfully") + s.ErrorContains(err, "operation failed from standalone callback") + + // Confirm the error's type is correct as well. + var wee *temporal.WorkflowExecutionError + s.ErrorAs(err, &wee) + + var noe *temporal.NexusOperationError + s.ErrorAs(wee, &noe) + s.Contains(noe.Error(), "operation failed from standalone callback") + } + + runStandaloneCallbackScenario(ctx, failureCompletion, verifyWorkflowRunFn) + }) +} + +// TestPollCallbackExecution tests that PollCallbackExecution long-polls for the outcome +// of a callback execution. It returns an empty response when the poll times out, and +// the CallbackExecutionOutcome when the callback reaches a terminal state. +func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + s.Run("returns_empty_for_non_terminal", func() { + callbackID := "poll-test-" + uuid.NewString() + + // Report the result of a non-existent, non-routable callback. The CallbackExecution + // will linger for 1m before timing out. + s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + + // Poll a non-terminal callback with a short timeout. Should return an empty response. + // NOTE: Passing a shorter timeout like 1s will cause failure with "Workflow is busy." + shortCtx, shortCancel := context.WithTimeout(ctx, 2*time.Second) + defer shortCancel() + pollResp, err := s.FrontendClient().PollCallbackExecution(shortCtx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + }) + s.NoError(err) + s.Nil(pollResp.GetOutcome()) + + // Terminate the CallbackExecution resource, then poll should return the outcome. + _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + Identity: s.T().Name(), + RequestId: uuid.NewString(), + Reason: "testing poll behavior", + }) + s.Require().NoError(err, "Unable to terminate CallbackExecution") + + pollResp, err = s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + }) + s.NoError(err) + s.NotNil(pollResp.GetOutcome().GetFailure()) + s.Equal("testing poll behavior", pollResp.GetOutcome().GetFailure().GetMessage()) + }) + + s.Run("blocks_until_complete", func() { + callbackID := "poll-blocks-" + uuid.NewString() + s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + + // Start a long-poll in a goroutine. + type pollResult struct { + resp *workflowservice.PollCallbackExecutionResponse + err error + } + resultCh := make(chan pollResult, 1) + go func() { + resp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + }) + resultCh <- pollResult{resp: resp, err: err} + }() + + // Verify the poll is still blocking and hasn't returned yet. + select { + case <-resultCh: + s.Fail("expected poll to block, but it returned before terminate") + case <-time.After(500 * time.Millisecond): + } + + // Terminate the CallbackExecution. Confirm that the poll result (from Goroutine) has completed. + _, err := s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + Identity: "test", + RequestId: uuid.NewString(), + Reason: "testing poll blocks", + }) + s.NoError(err) + + result := <-resultCh + s.NoError(result.err) + s.NotNil(result.resp.GetOutcome().GetFailure()) + s.Equal("testing poll blocks", result.resp.GetOutcome().GetFailure().GetMessage()) + }) + + s.Run("returns_run_id", func() { + callbackID := "poll-runid-" + uuid.NewString() + startResp := s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + + gotRunID := startResp.GetRunId() + + // Terminate so poll returns immediately. + _, err := s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + Identity: "test", + RequestId: uuid.NewString(), + Reason: "testing run_id", + }) + s.NoError(err) + + // Confirm the Poll result includes the RunID of the initial StartCallbackExecution request. + pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + }) + s.NoError(err) + s.Equal(gotRunID, pollResp.GetRunId()) + s.NotNil(pollResp.GetOutcome().GetFailure()) + }) + + s.Run("poll_after_timeout", func() { + callbackID := "poll-timeout-" + uuid.NewString() + // Start with a very short schedule-to-close timeout so it times out quickly. + s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, 500*time.Millisecond) + + // Wait for the callback to time out, then poll for the outcome. + const ( + waitUpTo = 3 * time.Second + checkInterval = 200 * time.Millisecond + ) + s.EventuallyWithT(func(t *assert.CollectT) { + pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + }) + require.NoError(t, err) + require.NotNil(t, pollResp.GetOutcome()) + + require.NotNil(t, pollResp.GetOutcome().GetFailure()) + require.NotNil(t, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo()) + require.Equal(t, enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) + }, waitUpTo, checkInterval) + }) + + // Context: The tests are failing because I'm not tracking the terminal failure. + // Part of that is because I don't understand the PR feedback, asking to put it + // in a "separate data field" from + // package temporal.server.chasm.lib.callbacks.proto.v1; + // CallbackState.Failure +} + +// TestDeleteCallbackExecution verifies that a standalone callback execution can be deleted. +// Delete terminates the callback if it's still running, then marks it for cleanup. +func (s *StandaloneCallbackSuite) TestDeleteCallbackExecution() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Create a callback that points to a non-existent URL so it won't complete on its own. + // The callback will be in SCHEDULED/BACKING_OFF state when we delete it. + callbackID := "delete-test-" + uuid.NewString() + startResp := s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + runID := startResp.GetRunId() + + // Describe using run_id to verify it was created. + descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: runID, + }) + s.NoError(err) + s.Equal(callbackID, descResp.GetInfo().GetCallbackId()) + s.Equal(runID, descResp.GetInfo().GetRunId()) + s.Equal(enumspb.CALLBACK_EXECUTION_STATUS_RUNNING, descResp.GetInfo().GetStatus()) + + // Delete with wrong run_id should fail. + _, err = s.FrontendClient().DeleteCallbackExecution(ctx, &workflowservice.DeleteCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: uuid.NewString(), + }) + s.ErrorContains(err, fmt.Sprintf("callback not found for ID: %s", callbackID)) + + // Delete the callback execution using correct run_id. + _, err = s.FrontendClient().DeleteCallbackExecution(ctx, &workflowservice.DeleteCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: runID, + }) + s.NoError(err) + + // Describe after delete — the callback should eventually be not found. + const ( + waitUpTo = 3 * time.Second + checkInterval = 100 * time.Millisecond + ) + s.EventuallyWithT(func(t *assert.CollectT) { + _, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: runID, + }) + require.ErrorContains(t, err, "not found") + }, waitUpTo, checkInterval) +} + +// TestStartCallbackExecution_InvalidArguments verifies request validation. +func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + validCallback := &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "http://localhost:1/callback", + }, + }, + } + validCompletion := &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), "result"), + }, + } + + tests := []struct { + name string + mutate func(req *workflowservice.StartCallbackExecutionRequest) + errMsg string + }{ + { + name: "missing callback_id", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.CallbackId = "" + }, + errMsg: "CallbackId is not set", + }, + { + name: "missing callback", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.Callback = nil + }, + errMsg: "Callback is not set", + }, + { + name: "missing callback URL", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.Callback = &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{Url: ""}, + }, + } + }, + errMsg: "Callback URL is not set", + }, + { + name: "invalid callback URL scheme", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.Callback = &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{Url: "ftp://example.com/callback"}, + }, + } + }, + errMsg: "unknown scheme", + }, + { + name: "callback URL missing host", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.Callback = &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{Url: "http:///callback"}, + }, + } + }, + errMsg: "missing host", + }, + { + name: "missing completion", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.Input = nil + }, + errMsg: "Completion is not set", + }, + { + name: "empty completion", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.Input = &workflowservice.StartCallbackExecutionRequest_Completion{Completion: &callbackpb.CallbackExecutionCompletion{}} + }, + errMsg: "Completion must have either success or failure set", + }, + { + name: "missing schedule_to_close_timeout", + mutate: func(req *workflowservice.StartCallbackExecutionRequest) { + req.ScheduleToCloseTimeout = nil + }, + errMsg: "ScheduleToCloseTimeout must be set", + }, + } + + for _, tc := range tests { + s.Run(tc.name, func() { + req := &workflowservice.StartCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + Identity: "test", + RequestId: uuid.NewString(), + CallbackId: "validation-test-" + uuid.NewString(), + Callback: validCallback, + Input: &workflowservice.StartCallbackExecutionRequest_Completion{Completion: validCompletion}, + ScheduleToCloseTimeout: durationpb.New(time.Minute), + } + tc.mutate(req) + _, err := s.FrontendClient().StartCallbackExecution(ctx, req) + s.Error(err) + s.Contains(err.Error(), tc.errMsg) + }) + } +} + +// TestStartCallbackExecution_DuplicateID verifies that starting a callback with +// an already-used callback_id returns an AlreadyExists error with a different request_id, +// and that the same request_id is idempotent (returns the existing run_id without error). +func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + callbackID := "dup-test-" + uuid.NewString() + requestID := uuid.NewString() + + // Build the request explicitly so we can reuse the same request_id. + req := &workflowservice.StartCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + Identity: "test", + RequestId: requestID, + CallbackId: callbackID, + Callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "http://localhost:1/nonexistent", + }, + }, + }, + Input: &workflowservice.StartCallbackExecutionRequest_Completion{Completion: &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), "some-result"), + }, + }}, + ScheduleToCloseTimeout: durationpb.New(time.Minute), + } + + // First call succeeds. + startResp, err := s.FrontendClient().StartCallbackExecution(ctx, req) + s.NoError(err) + existingRunID := startResp.GetRunId() + s.NotEmpty(existingRunID) + + // Same callback_id + same request_id should be idempotent (return existing run_id). + dupResp, err := s.FrontendClient().StartCallbackExecution(ctx, req) + s.NoError(err) + s.Equal(existingRunID, dupResp.GetRunId()) + + // Same callback_id + different request_id should fail with serviceerror.CallbackExecutionAlreadyStarted. + req.RequestId = uuid.NewString() + _, err = s.FrontendClient().StartCallbackExecution(ctx, req) + s.Error(err) + s.Contains(err.Error(), "callback execution already started") +} + +// TestListAndCountCallbackExecutions tests that standalone callback executions +// can be listed and counted via the visibility APIs, and verifies the returned data. +func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) + defer cancel() + + // Create two callback executions with known IDs. + callbackIDs := make([]string, 2) + for i := range 2 { + callbackIDs[i] = fmt.Sprintf("list-test-%d-%s", i, uuid.NewString()) + s.callStartCallbackExecutionToBogusCallback(ctx, callbackIDs[i], time.Minute) + } + + // List callback executions. Visibility indexing happens be async, so use EventuallyWithT. + const ( + waitUpTo = 5 * time.Second + checkInterval = 200 * time.Millisecond + ) + + // Verify returned data includes our callback IDs and has valid fields. + s.EventuallyWithT(func(t *assert.CollectT) { + listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: s.Namespace().String(), + PageSize: 10, + }) + require.NoError(t, err) + require.GreaterOrEqual(t, len(listResp.GetExecutions()), 2) + + // Collect returned callback IDs and verify fields. + foundIDs := make(map[string]bool) + for _, exec := range listResp.GetExecutions() { + foundIDs[exec.GetCallbackId()] = true + require.NotEmpty(t, exec.GetCallbackId()) + require.NotNil(t, exec.GetCreateTime()) + } + for _, id := range callbackIDs { + require.True(t, foundIDs[id], "expected callback %s in list response", id) + } + }, waitUpTo, checkInterval, "Didn't find expected results from ListCallbackExecutions") + + // List with ExecutionStatus query filter — newly started callbacks should be "Running". + s.EventuallyWithT(func(t *assert.CollectT) { + listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: s.Namespace().String(), + PageSize: 10, + Query: fmt.Sprintf(`ExecutionStatus = "Running" AND CallbackId = %q`, callbackIDs[0]), + }) + require.NoError(t, err) + require.Len(t, listResp.GetExecutions(), 1) + require.Equal(t, callbackIDs[0], listResp.GetExecutions()[0].GetCallbackId()) + }, waitUpTo, checkInterval, "Didn't find Running callback") + + // Terminate one callback to test filtering by terminal status. + _, err := s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackIDs[1], + Identity: "test", + RequestId: uuid.NewString(), + Reason: "testing list filter", + }) + s.NoError(err) + + // List with ExecutionStatus = "Terminated" should find the terminated callback. + s.EventuallyWithT(func(t *assert.CollectT) { + listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: s.Namespace().String(), + PageSize: 10, + Query: fmt.Sprintf(`ExecutionStatus = "Terminated" AND CallbackId = %q`, callbackIDs[1]), + }) + require.NoError(t, err) + require.Len(t, listResp.GetExecutions(), 1) + require.Equal(t, callbackIDs[1], listResp.GetExecutions()[0].GetCallbackId()) + }, waitUpTo, checkInterval, "Didn't find Terminated callbacks") + + // Count callback executions. + s.EventuallyWithT(func(t *assert.CollectT) { + countResp, err := s.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ + Namespace: s.Namespace().String(), + }) + require.NoError(t, err) + require.GreaterOrEqual(t, countResp.GetCount(), int64(2)) + }, 10*time.Second, 200*time.Millisecond) + + // Count with ExecutionStatus filter should only count scheduled callbacks. + s.EventuallyWithT(func(t *assert.CollectT) { + countResp, err := s.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ + Namespace: s.Namespace().String(), + Query: `ExecutionStatus = "Running"`, + }) + require.NoError(t, err) + require.GreaterOrEqual(t, countResp.GetCount(), int64(1)) + }, waitUpTo, checkInterval) +} + +// TestStartCallbackExecution_SearchAttributes tests that search attributes provided at start +// are persisted and can be used to query callback executions via list filtering. +func (s *StandaloneCallbackSuite) TestStartCallbackExecution_SearchAttributes() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + callbackID := "sa-test-" + uuid.NewString() + saValue := "sa-test-value-" + uuid.NewString() + + _, err := s.FrontendClient().StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + Identity: "test", + RequestId: uuid.NewString(), + CallbackId: callbackID, + Callback: &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "http://localhost:1/nonexistent", + }, + }, + }, + Input: &workflowservice.StartCallbackExecutionRequest_Completion{Completion: &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), "some-result"), + }, + }}, + ScheduleToCloseTimeout: durationpb.New(time.Minute), + SearchAttributes: &commonpb.SearchAttributes{ + IndexedFields: map[string]*commonpb.Payload{ + "CustomKeywordField": testcore.MustToPayload(s.T(), saValue), + }, + }, + }) + s.NoError(err) + + // Verify the search attribute is queryable via list. + s.EventuallyWithT(func(t *assert.CollectT) { + listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: s.Namespace().String(), + PageSize: 10, + Query: fmt.Sprintf(`CustomKeywordField = %q AND CallbackId = %q`, saValue, callbackID), + }) + require.NoError(t, err) + require.Len(t, listResp.GetExecutions(), 1) + require.Equal(t, callbackID, listResp.GetExecutions()[0].GetCallbackId()) + }, 10*time.Second, 200*time.Millisecond) +} + +// TestTerminateCallbackExecution tests terminate, run_id validation, and request ID idempotency. +func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) + defer cancel() + + callbackID := "terminate-test-" + uuid.NewString() + startResp := s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + requestID := uuid.NewString() + runID := startResp.GetRunId() + + // Wrong run_id should fail for describe, poll, and terminate. + wrongRunID := uuid.NewString() + + _, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: wrongRunID, + }) + s.Error(err) + + shortCtx, shortCancel := context.WithTimeout(ctx, time.Second*2) + defer shortCancel() + _, err = s.FrontendClient().PollCallbackExecution(shortCtx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: wrongRunID, + }) + s.Error(err) + + _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: wrongRunID, + Identity: "test", + RequestId: uuid.NewString(), + Reason: "wrong run_id", + }) + s.Error(err) + + // Terminate with correct run_id and known request ID. + _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: runID, + Identity: "test", + RequestId: requestID, + Reason: "testing terminate", + }) + s.NoError(err) + + // Describe after terminate — should be TERMINATED with correct run_id. + descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: runID, + }) + s.NoError(err) + s.Equal(enumspb.CALLBACK_EXECUTION_STATUS_TERMINATED, descResp.GetInfo().GetStatus()) + s.Equal(enumspb.CALLBACK_STATE_TERMINATED, descResp.GetInfo().GetState()) + s.NotNil(descResp.GetInfo().GetCloseTime()) + s.Equal(runID, descResp.GetInfo().GetRunId()) + + // Poll to verify the outcome. + pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + RunId: runID, + }) + s.NoError(err) + s.NotNil(pollResp.GetOutcome().GetFailure()) + s.Equal("testing terminate", pollResp.GetOutcome().GetFailure().GetMessage()) + + // Same request ID should be a no-op (idempotent). + _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + Identity: "test", + RequestId: requestID, + Reason: "testing terminate", + }) + s.NoError(err) + + // Different request ID should return FailedPrecondition. + _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + Identity: "test", + RequestId: uuid.NewString(), + Reason: "different request", + }) + s.Error(err) + s.Contains(err.Error(), "already terminated with request ID") +} + +// TestCallbackExecutionFailedOutcome tests that when a callback fails with a non-retryable error +// (e.g., a 400 response from the target), the poll outcome contains the failure details. +func (s *StandaloneCallbackSuite) TestCallbackExecutionFailedOutcome() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + // Start an HTTP server that always returns 400 Bad Request (non-retryable). + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + })) + defer srv.Close() + + callbackID := "failed-outcome-test-" + uuid.NewString() + cb := &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{Url: srv.URL + "/callback"}, + }, + } + completion := &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), "some-result"), + }, + } + s.callStartCallbackExecution(ctx, callbackID, cb, completion, time.Minute) + + // Poll for the outcome — the callback should eventually fail with a non-retryable error. + pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + }) + s.NoError(err) + s.NotNil(pollResp.GetOutcome().GetFailure()) + s.Contains(pollResp.GetOutcome().GetFailure().GetMessage(), "handler error (BAD_REQUEST)") + s.True(pollResp.GetOutcome().GetFailure().GetApplicationFailureInfo().GetNonRetryable()) +} + +// TestNexusOperationCompletionBeforeStartHandlerReturns tests that a standalone callback can +// complete a Nexus operation even when the callback execution is started *before* the Nexus +// start handler returns to the caller. This exercises the race where the completion arrives +// while the operation is still in SCHEDULED state (i.e., before transitioning to STARTED). +// +// Flow: +// 1. A caller workflow starts a Nexus operation via an external endpoint. +// 2. The external Nexus handler captures the callback URL/token and calls +// StartCallbackExecution to deliver the completion *before* returning async. +// 3. The handler then returns an async result. +// 4. The operation can be completed from SCHEDULED state directly, so the workflow +// receives the result regardless of the start handler timing. +func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandlerReturns() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) + defer cancel() + + taskQueue := testcore.RandomizeStr(s.T().Name()) + endpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) + + // Set up an external Nexus handler that starts the standalone callback *inside* + // the start handler — before returning the async result — to simulate a race + // where the completion is delivered before the caller processes the start response. + h := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + token := options.CallbackHeader.Get(commonnexus.CallbackTokenHeader) + callbackURL := options.CallbackURL + + // Start the standalone callback execution to deliver the completion + // BEFORE returning from this handler. + callbackID := "race-callback-" + uuid.NewString() + cb := &commonpb.Callback{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: callbackURL, + Token: token, + }, + }, + } + completion := &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), "result-before-start-returns"), + }, + } + s.callStartCallbackExecution(ctx, callbackID, cb, completion, time.Minute) + + // Now return the async result — the completion has already been + // delivered and the operation should already be completed. + return &nexus.HandlerStartOperationResultAsync{ + OperationToken: "test", + }, nil + }, + } + listenAddr := nexustest.AllocListenAddress() + nexustest.NewNexusServer(s.T(), listenAddr, h) + + _, err := s.OperatorClient().CreateNexusEndpoint(ctx, &operatorservice.CreateNexusEndpointRequest{ + Spec: &nexuspb.EndpointSpec{ + Name: endpointName, + Target: &nexuspb.EndpointTarget{ + Variant: &nexuspb.EndpointTarget_External_{ + External: &nexuspb.EndpointTarget_External{ + Url: "http://" + listenAddr, + }, + }, + }, + }, + }) + s.NoError(err) + + // Register a caller workflow that starts a Nexus operation and waits for its result. + callerWf := func(ctx workflow.Context) (string, error) { + c := workflow.NewNexusClient(endpointName, "service") + fut := c.ExecuteOperation(ctx, "operation", "input", workflow.NexusOperationOptions{}) + var result string + err := fut.Get(ctx, &result) + return result, err + } + + w := worker.New(s.SdkClient(), taskQueue, worker.Options{}) + w.RegisterWorkflow(callerWf) + s.NoError(w.Start()) + defer w.Stop() + + // Start the caller workflow. + run, err := s.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{ + TaskQueue: taskQueue, + }, callerWf) + s.NoError(err) + + // The standalone callback delivers the completion even though it was started + // before the start handler returned. The operation transitions directly from + // SCHEDULED to SUCCEEDED. + var result string + s.NoError(run.Get(ctx, &result)) + s.Equal("result-before-start-returns", result) + + // Verify the operation token is recorded in the caller workflow's history. + histResp, err := s.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: run.GetID(), + RunId: run.GetRunID(), + }, + }) + s.NoError(err) + startedIdx := slices.IndexFunc(histResp.History.Events, func(e *historypb.HistoryEvent) bool { + return e.GetNexusOperationStartedEventAttributes() != nil + }) + s.NotEqual(-1, startedIdx, "expected NexusOperationStarted event in history") + s.Equal("test", histResp.History.Events[startedIdx].GetNexusOperationStartedEventAttributes().GetOperationToken()) +} + +// TestScheduleToCloseTimeout verifies that a callback execution transitions to FAILED +// when its schedule-to-close timeout expires before the callback succeeds. +func (s *StandaloneCallbackSuite) TestScheduleToCloseTimeout() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + // Short timeout so it fires quickly during the test. + callbackID := "timeout-test-" + uuid.NewString() + s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, 2*time.Second) + + // Poll until the callback reaches a terminal state due to timeout. + pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + }) + s.NoError(err) + s.NotNil(pollResp.GetOutcome(), "expected terminal outcome after timeout") + s.NotNil(pollResp.GetOutcome().GetFailure(), "expected failure outcome after timeout") + s.Contains(pollResp.GetOutcome().GetFailure().GetMessage(), "timed out") + s.NotNil(pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo()) + s.Equal(enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) + + // Describe should show FAILED state with timeout failure. + descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: s.Namespace().String(), + CallbackId: callbackID, + IncludeOutcome: true, + }) + s.NoError(err) + s.Equal(enumspb.CALLBACK_EXECUTION_STATUS_FAILED, descResp.GetInfo().GetStatus()) + s.Equal(enumspb.CALLBACK_STATE_FAILED, descResp.GetInfo().GetState()) + s.NotNil(descResp.GetInfo().GetCloseTime()) + s.NotNil(descResp.GetOutcome().GetFailure()) + s.NotNil(descResp.GetOutcome().GetFailure().GetTimeoutFailureInfo()) + s.Equal(enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, descResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) +} From 047a6e8220810bea869e537c16e8d9f10ee969fd Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 8 May 2026 11:08:18 -0700 Subject: [PATCH 02/52] Test updates --- common/api/metadata.go | 7 +++++++ common/rpc/interceptor/redirection.go | 8 ++++++++ common/rpc/interceptor/redirection_test.go | 8 ++++++++ 3 files changed, 23 insertions(+) diff --git a/common/api/metadata.go b/common/api/metadata.go index bb584d38000..9962280693e 100644 --- a/common/api/metadata.go +++ b/common/api/metadata.go @@ -89,20 +89,27 @@ var ( "RespondActivityTaskCanceled": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "RespondActivityTaskCanceledById": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "CountActivityExecutions": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingNone}, + "CountCallbackExecutions": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingNone}, "CountNexusOperationExecutions": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingNone}, "DeleteActivityExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, + "DeleteCallbackExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "DeleteNexusOperationExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "DescribeActivityExecution": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingCapable}, + "DescribeCallbackExecution": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingCapable}, "DescribeNexusOperationExecution": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingCapable}, "PollActivityExecution": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingAlways}, + "PollCallbackExecution": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingAlways}, "PollNexusOperationExecution": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingAlways}, "ListActivityExecutions": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingNone}, + "ListCallbackExecutions": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingNone}, "ListNexusOperationExecutions": {Scope: ScopeNamespace, Access: AccessReadOnly, Polling: PollingNone}, "RequestCancelActivityExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "RequestCancelNexusOperationExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "StartActivityExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, + "StartCallbackExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "StartNexusOperationExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "TerminateActivityExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, + "TerminateCallbackExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "TerminateNexusOperationExecution": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, "PollNexusTaskQueue": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingAlways}, "RespondNexusTaskCompleted": {Scope: ScopeNamespace, Access: AccessWrite, Polling: PollingNone}, diff --git a/common/rpc/interceptor/redirection.go b/common/rpc/interceptor/redirection.go index 05e3bc73cf5..276662e2e84 100644 --- a/common/rpc/interceptor/redirection.go +++ b/common/rpc/interceptor/redirection.go @@ -156,6 +156,14 @@ var ( "TerminateActivityExecution": func() any { return &workflowservice.TerminateActivityExecutionResponse{} }, "DeleteActivityExecution": func() any { return &workflowservice.DeleteActivityExecutionResponse{} }, + "StartCallbackExecution": func() any { return &workflowservice.StartCallbackExecutionResponse{} }, + "DescribeCallbackExecution": func() any { return &workflowservice.DescribeCallbackExecutionResponse{} }, + "PollCallbackExecution": func() any { return &workflowservice.PollCallbackExecutionResponse{} }, + "CountCallbackExecutions": func() any { return &workflowservice.CountCallbackExecutionsResponse{} }, + "ListCallbackExecutions": func() any { return &workflowservice.ListCallbackExecutionsResponse{} }, + "TerminateCallbackExecution": func() any { return &workflowservice.TerminateCallbackExecutionResponse{} }, + "DeleteCallbackExecution": func() any { return &workflowservice.DeleteCallbackExecutionResponse{} }, + "CountNexusOperationExecutions": func() any { return &workflowservice.CountNexusOperationExecutionsResponse{} }, "DeleteNexusOperationExecution": func() any { return &workflowservice.DeleteNexusOperationExecutionResponse{} }, "DescribeNexusOperationExecution": func() any { return &workflowservice.DescribeNexusOperationExecutionResponse{} }, diff --git a/common/rpc/interceptor/redirection_test.go b/common/rpc/interceptor/redirection_test.go index ddc180b4b00..a1a46a68510 100644 --- a/common/rpc/interceptor/redirection_test.go +++ b/common/rpc/interceptor/redirection_test.go @@ -213,6 +213,14 @@ func (s *redirectionInterceptorSuite) TestGlobalAPI() { "TerminateActivityExecution": {}, "DeleteActivityExecution": {}, + "StartCallbackExecution": {}, + "DescribeCallbackExecution": {}, + "PollCallbackExecution": {}, + "ListCallbackExecutions": {}, + "CountCallbackExecutions": {}, + "TerminateCallbackExecution": {}, + "DeleteCallbackExecution": {}, + "CountNexusOperationExecutions": {}, "DeleteNexusOperationExecution": {}, "DescribeNexusOperationExecution": {}, From 2410731d5e76a7b946d1afae3c52e510948de83a Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 8 May 2026 11:57:51 -0700 Subject: [PATCH 03/52] Refactor standalone_callbacks_test.go to use testcore.NewEnv --- tests/standalone_callbacks_test.go | 308 +++++++++++++++-------------- 1 file changed, 162 insertions(+), 146 deletions(-) diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index ded87ac6215..7c7bf9e2e57 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -13,7 +13,6 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -30,6 +29,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/testing/parallelsuite" hsmcallbacks "go.temporal.io/server/components/callbacks" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/proto" @@ -129,32 +129,24 @@ func startFakeExternalService(ctx context.Context, c workflowservice.WorkflowSer } func TestStandaloneCallbackSuite(t *testing.T) { - t.Parallel() - suite.Run(t, new(StandaloneCallbackSuite)) + parallelsuite.Run(t, &StandaloneCallbackSuite{}) } type StandaloneCallbackSuite struct { - testcore.FunctionalTestBase + parallelsuite.Suite[*StandaloneCallbackSuite] } -// Expose a helper to enable "require" assertions, to fail the test immediately instead of -// allowing cascading failures. e.g. s.Require().NotNil(err, "unable to do critical thing") -func (s *StandaloneCallbackSuite) Require() *require.Assertions { - return require.New(s.T()) -} - -func (s *StandaloneCallbackSuite) SetupSuite() { - s.SetupSuiteWithCluster( - testcore.WithDynamicConfigOverrides(map[dynamicconfig.Key]any{ - dynamicconfig.EnableChasm.Key(): true, - dynamicconfig.EnableCHASMCallbacks.Key(): true, - callback.EnableStandaloneExecutions.Key(): true, - - // QUIRK: The configuration for setting the callback allow list - // is in the HSM code. - hsmcallbacks.AllowedAddresses.Key(): []any{ - map[string]any{"Pattern": "*", "AllowInsecure": true}, - }, +func (s *StandaloneCallbackSuite) newEnv() *testcore.TestEnv { + return testcore.NewEnv(s.T(), + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + testcore.WithDynamicConfig(callback.EnableStandaloneExecutions, true), + + // QUIRK: The configuration for setting the callback allow list + // is in the HSM code. The chasm/lib/callback AllowedAddresses + // field isn't used. + testcore.WithDynamicConfig(hsmcallbacks.AllowedAddresses, []any{ + map[string]any{"Pattern": "*", "AllowInsecure": true}, }), ) } @@ -168,6 +160,7 @@ func (s *StandaloneCallbackSuite) SetupSuite() { // aways result in timing out. func (s *StandaloneCallbackSuite) callStartCallbackExecutionToBogusCallback( ctx context.Context, + env *testcore.TestEnv, callbackID string, timeout time.Duration, ) *workflowservice.StartCallbackExecutionResponse { @@ -186,12 +179,13 @@ func (s *StandaloneCallbackSuite) callStartCallbackExecutionToBogusCallback( }, } - return s.callStartCallbackExecution(ctx, callbackID, cb, completion, timeout) + return s.callStartCallbackExecution(ctx, env, callbackID, cb, completion, timeout) } // Call the StartCallbackExecution API with the given parameters. func (s *StandaloneCallbackSuite) callStartCallbackExecution( ctx context.Context, + env *testcore.TestEnv, callbackID string, cb *commonpb.Callback, completion *callbackpb.CallbackExecutionCompletion, @@ -199,8 +193,11 @@ func (s *StandaloneCallbackSuite) callStartCallbackExecution( ) *workflowservice.StartCallbackExecutionResponse { s.T().Helper() - resp, err := s.FrontendClient().StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + frontend := env.FrontendClient() + namespace := env.Namespace() + + resp, err := frontend.StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ + Namespace: namespace.String(), Identity: "startCallbackExecution", RequestId: uuid.NewString(), CallbackId: callbackID, @@ -210,7 +207,7 @@ func (s *StandaloneCallbackSuite) callStartCallbackExecution( }, ScheduleToCloseTimeout: durationpb.New(timeout), }) - s.Require().NoError(err, "Error calling StartCallbackExecution to bogus callback") + s.NoError(err, "Error calling StartCallbackExecution to bogus callback") return resp } @@ -237,6 +234,8 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // Takes a CallbackExecutionCompletion to be reported by the external service, and a // verification function to test the calling Workflow behaved as expected. runStandaloneCallbackScenario := func( + s *StandaloneCallbackSuite, + env *testcore.TestEnv, ctx context.Context, completionResult *callbackpb.CallbackExecutionCompletion, workflowRunVerificationFn func(client.WorkflowRun), @@ -247,7 +246,7 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // // Start a fake external service to handle async requests, and report // their results to Temporal. - fakeSvc := startFakeExternalService(ctx, s.FrontendClient()) + fakeSvc := startFakeExternalService(ctx, env.FrontendClient()) // Nexus Handler // @@ -272,7 +271,7 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // Send the request to the external service to do the work. fakeSvc.incommingRequests <- externalRequestInfo{ - Namespace: s.Namespace().String(), + Namespace: env.Namespace().String(), Token: options.CallbackHeader.Get(commonnexus.CallbackTokenHeader), URL: options.CallbackURL, @@ -303,8 +302,8 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { }, }, } - _, err := s.OperatorClient().CreateNexusEndpoint(ctx, createNexusEndpointReq) - s.Require().NoError(err, "Error registering Nexus endpoint") + _, err := env.OperatorClient().CreateNexusEndpoint(ctx, createNexusEndpointReq) + s.NoError(err, "Error registering Nexus endpoint") // Calling Workflow // @@ -323,17 +322,17 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // Run the Test // // Construct and start the calling workflow Worker. Then wait for completion. - callerWfWorker := worker.New(s.SdkClient(), taskQueue, worker.Options{}) + callerWfWorker := worker.New(env.SdkClient(), taskQueue, worker.Options{}) callerWfWorker.RegisterWorkflow(callerWf) - s.Require().NoError(callerWfWorker.Start(), "Error starting calling workflow Worker") + s.NoError(callerWfWorker.Start(), "Error starting calling workflow Worker") defer callerWfWorker.Stop() // Start startOpts := client.StartWorkflowOptions{ TaskQueue: taskQueue, } - callerWfRun, err := s.SdkClient().ExecuteWorkflow(ctx, startOpts, callerWf) - s.Require().NoError(err, "Error running the caller Workflow") + callerWfRun, err := env.SdkClient().ExecuteWorkflow(ctx, startOpts, callerWf) + s.NoError(err, "Error running the caller Workflow") // Defer to a caller-supplied verification function to wait on the caller // workflow's result and verify the success/failure as applicable. @@ -342,23 +341,23 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // If the fake service was called correctly, we expect to see the result // of it doing the out-of-band work. fakeSvcResult := <-fakeSvc.requestResults - s.Require().NoError(fakeSvcResult.Error) + s.NoError(fakeSvcResult.Error) // Additional Verification // // Use the Describe and Poll APIs to fetch the callback execution. - descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + descResp, err := env.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: fakeSvcResult.CallbackID, IncludeInput: true, IncludeOutcome: false, }) - s.Require().NoError(err, "Error describing the callback execution") + s.NoError(err, "Error describing the callback execution") s.True(proto.Equal(completionResult, descResp.Input)) s.Nil(descResp.Outcome) gotInfo := descResp.GetInfo() - s.Require().NotNil(gotInfo, "Got nil Info in response") + s.NotNil(gotInfo, "Got nil Info in response") s.Equal(fakeSvcResult.CallbackID, gotInfo.GetCallbackId()) s.NotNil(gotInfo.GetCreateTime()) @@ -368,14 +367,14 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { s.Equal(enumspb.CALLBACK_STATE_SUCCEEDED, gotInfo.GetState()) // Poll to verify the outcome as well. - pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: fakeSvcResult.CallbackID, }) - s.Require().NoError(err, "Error polling completed callback execution") + s.NoError(err, "Error polling completed callback execution") outcome := pollResp.GetOutcome() - s.Require().NotNil(outcome, "Got nil Outcome") + s.NotNil(outcome, "Got nil Outcome") // Confirm the outcome of the callback was a successful delivery. // (Even if it was for a failed completion.) @@ -383,7 +382,8 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { s.Nil(outcome.GetFailure()) } - s.Run("success", func() { + s.Run("success", func(s *StandaloneCallbackSuite) { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -400,10 +400,11 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { s.Equal(wantPayloadStr, gotPayload) } - runStandaloneCallbackScenario(ctx, successCompletion, verifyWorkflowRunFn) + runStandaloneCallbackScenario(s, env, ctx, successCompletion, verifyWorkflowRunFn) }) - s.Run("failure", func() { + s.Run("failure", func(s *StandaloneCallbackSuite) { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -440,7 +441,7 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { s.Contains(noe.Error(), "operation failed from standalone callback") } - runStandaloneCallbackScenario(ctx, failureCompletion, verifyWorkflowRunFn) + runStandaloneCallbackScenario(s, env, ctx, failureCompletion, verifyWorkflowRunFn) }) } @@ -448,39 +449,41 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // of a callback execution. It returns an empty response when the poll times out, and // the CallbackExecutionOutcome when the callback reaches a terminal state. func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + env := s.newEnv() + + s.Run("returns_empty_for_non_terminal", func(s *StandaloneCallbackSuite) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() - s.Run("returns_empty_for_non_terminal", func() { callbackID := "poll-test-" + uuid.NewString() // Report the result of a non-existent, non-routable callback. The CallbackExecution // will linger for 1m before timing out. - s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) // Poll a non-terminal callback with a short timeout. Should return an empty response. // NOTE: Passing a shorter timeout like 1s will cause failure with "Workflow is busy." shortCtx, shortCancel := context.WithTimeout(ctx, 2*time.Second) defer shortCancel() - pollResp, err := s.FrontendClient().PollCallbackExecution(shortCtx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err := env.FrontendClient().PollCallbackExecution(shortCtx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, }) s.NoError(err) s.Nil(pollResp.GetOutcome()) // Terminate the CallbackExecution resource, then poll should return the outcome. - _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, Identity: s.T().Name(), RequestId: uuid.NewString(), Reason: "testing poll behavior", }) - s.Require().NoError(err, "Unable to terminate CallbackExecution") + s.NoError(err, "Unable to terminate CallbackExecution") - pollResp, err = s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err = env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, }) s.NoError(err) @@ -488,9 +491,12 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.Equal("testing poll behavior", pollResp.GetOutcome().GetFailure().GetMessage()) }) - s.Run("blocks_until_complete", func() { + s.Run("blocks_until_complete", func(s *StandaloneCallbackSuite) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + callbackID := "poll-blocks-" + uuid.NewString() - s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) // Start a long-poll in a goroutine. type pollResult struct { @@ -499,8 +505,8 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { } resultCh := make(chan pollResult, 1) go func() { - resp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + resp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, }) resultCh <- pollResult{resp: resp, err: err} @@ -514,8 +520,8 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { } // Terminate the CallbackExecution. Confirm that the poll result (from Goroutine) has completed. - _, err := s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err := env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, Identity: "test", RequestId: uuid.NewString(), @@ -529,15 +535,18 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.Equal("testing poll blocks", result.resp.GetOutcome().GetFailure().GetMessage()) }) - s.Run("returns_run_id", func() { + s.Run("returns_run_id", func(s *StandaloneCallbackSuite) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + callbackID := "poll-runid-" + uuid.NewString() - startResp := s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + startResp := s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) gotRunID := startResp.GetRunId() // Terminate so poll returns immediately. - _, err := s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err := env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, Identity: "test", RequestId: uuid.NewString(), @@ -546,8 +555,8 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.NoError(err) // Confirm the Poll result includes the RunID of the initial StartCallbackExecution request. - pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, }) s.NoError(err) @@ -555,10 +564,13 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.NotNil(pollResp.GetOutcome().GetFailure()) }) - s.Run("poll_after_timeout", func() { + s.Run("poll_after_timeout", func(s *StandaloneCallbackSuite) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + callbackID := "poll-timeout-" + uuid.NewString() // Start with a very short schedule-to-close timeout so it times out quickly. - s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, 500*time.Millisecond) + s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, 500*time.Millisecond) // Wait for the callback to time out, then poll for the outcome. const ( @@ -566,8 +578,8 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { checkInterval = 200 * time.Millisecond ) s.EventuallyWithT(func(t *assert.CollectT) { - pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, }) require.NoError(t, err) @@ -578,29 +590,24 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { require.Equal(t, enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) }, waitUpTo, checkInterval) }) - - // Context: The tests are failing because I'm not tracking the terminal failure. - // Part of that is because I don't understand the PR feedback, asking to put it - // in a "separate data field" from - // package temporal.server.chasm.lib.callbacks.proto.v1; - // CallbackState.Failure } // TestDeleteCallbackExecution verifies that a standalone callback execution can be deleted. // Delete terminates the callback if it's still running, then marks it for cleanup. func (s *StandaloneCallbackSuite) TestDeleteCallbackExecution() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() // Create a callback that points to a non-existent URL so it won't complete on its own. // The callback will be in SCHEDULED/BACKING_OFF state when we delete it. callbackID := "delete-test-" + uuid.NewString() - startResp := s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + startResp := s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) runID := startResp.GetRunId() // Describe using run_id to verify it was created. - descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + descResp, err := env.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: runID, }) @@ -610,16 +617,16 @@ func (s *StandaloneCallbackSuite) TestDeleteCallbackExecution() { s.Equal(enumspb.CALLBACK_EXECUTION_STATUS_RUNNING, descResp.GetInfo().GetStatus()) // Delete with wrong run_id should fail. - _, err = s.FrontendClient().DeleteCallbackExecution(ctx, &workflowservice.DeleteCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().DeleteCallbackExecution(ctx, &workflowservice.DeleteCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: uuid.NewString(), }) s.ErrorContains(err, fmt.Sprintf("callback not found for ID: %s", callbackID)) // Delete the callback execution using correct run_id. - _, err = s.FrontendClient().DeleteCallbackExecution(ctx, &workflowservice.DeleteCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().DeleteCallbackExecution(ctx, &workflowservice.DeleteCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: runID, }) @@ -631,8 +638,8 @@ func (s *StandaloneCallbackSuite) TestDeleteCallbackExecution() { checkInterval = 100 * time.Millisecond ) s.EventuallyWithT(func(t *assert.CollectT) { - _, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err := env.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: runID, }) @@ -642,8 +649,7 @@ func (s *StandaloneCallbackSuite) TestDeleteCallbackExecution() { // TestStartCallbackExecution_InvalidArguments verifies request validation. func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() + env := s.newEnv() validCallback := &commonpb.Callback{ Variant: &commonpb.Callback_Nexus_{ @@ -734,9 +740,12 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() } for _, tc := range tests { - s.Run(tc.name, func() { + s.Run(tc.name, func(s *StandaloneCallbackSuite) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + req := &workflowservice.StartCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + Namespace: env.Namespace().String(), Identity: "test", RequestId: uuid.NewString(), CallbackId: "validation-test-" + uuid.NewString(), @@ -745,7 +754,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() ScheduleToCloseTimeout: durationpb.New(time.Minute), } tc.mutate(req) - _, err := s.FrontendClient().StartCallbackExecution(ctx, req) + _, err := env.FrontendClient().StartCallbackExecution(ctx, req) s.Error(err) s.Contains(err.Error(), tc.errMsg) }) @@ -756,6 +765,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() // an already-used callback_id returns an AlreadyExists error with a different request_id, // and that the same request_id is idempotent (returns the existing run_id without error). func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() @@ -764,7 +774,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { // Build the request explicitly so we can reuse the same request_id. req := &workflowservice.StartCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + Namespace: env.Namespace().String(), Identity: "test", RequestId: requestID, CallbackId: callbackID, @@ -784,19 +794,19 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { } // First call succeeds. - startResp, err := s.FrontendClient().StartCallbackExecution(ctx, req) + startResp, err := env.FrontendClient().StartCallbackExecution(ctx, req) s.NoError(err) existingRunID := startResp.GetRunId() s.NotEmpty(existingRunID) // Same callback_id + same request_id should be idempotent (return existing run_id). - dupResp, err := s.FrontendClient().StartCallbackExecution(ctx, req) + dupResp, err := env.FrontendClient().StartCallbackExecution(ctx, req) s.NoError(err) s.Equal(existingRunID, dupResp.GetRunId()) // Same callback_id + different request_id should fail with serviceerror.CallbackExecutionAlreadyStarted. req.RequestId = uuid.NewString() - _, err = s.FrontendClient().StartCallbackExecution(ctx, req) + _, err = env.FrontendClient().StartCallbackExecution(ctx, req) s.Error(err) s.Contains(err.Error(), "callback execution already started") } @@ -804,6 +814,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { // TestListAndCountCallbackExecutions tests that standalone callback executions // can be listed and counted via the visibility APIs, and verifies the returned data. func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) defer cancel() @@ -811,7 +822,7 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { callbackIDs := make([]string, 2) for i := range 2 { callbackIDs[i] = fmt.Sprintf("list-test-%d-%s", i, uuid.NewString()) - s.callStartCallbackExecutionToBogusCallback(ctx, callbackIDs[i], time.Minute) + s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackIDs[i], time.Minute) } // List callback executions. Visibility indexing happens be async, so use EventuallyWithT. @@ -822,8 +833,8 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { // Verify returned data includes our callback IDs and has valid fields. s.EventuallyWithT(func(t *assert.CollectT) { - listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ - Namespace: s.Namespace().String(), + listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: env.Namespace().String(), PageSize: 10, }) require.NoError(t, err) @@ -843,8 +854,8 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { // List with ExecutionStatus query filter — newly started callbacks should be "Running". s.EventuallyWithT(func(t *assert.CollectT) { - listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ - Namespace: s.Namespace().String(), + listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: env.Namespace().String(), PageSize: 10, Query: fmt.Sprintf(`ExecutionStatus = "Running" AND CallbackId = %q`, callbackIDs[0]), }) @@ -854,8 +865,8 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { }, waitUpTo, checkInterval, "Didn't find Running callback") // Terminate one callback to test filtering by terminal status. - _, err := s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err := env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackIDs[1], Identity: "test", RequestId: uuid.NewString(), @@ -865,8 +876,8 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { // List with ExecutionStatus = "Terminated" should find the terminated callback. s.EventuallyWithT(func(t *assert.CollectT) { - listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ - Namespace: s.Namespace().String(), + listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: env.Namespace().String(), PageSize: 10, Query: fmt.Sprintf(`ExecutionStatus = "Terminated" AND CallbackId = %q`, callbackIDs[1]), }) @@ -877,8 +888,8 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { // Count callback executions. s.EventuallyWithT(func(t *assert.CollectT) { - countResp, err := s.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ - Namespace: s.Namespace().String(), + countResp, err := env.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ + Namespace: env.Namespace().String(), }) require.NoError(t, err) require.GreaterOrEqual(t, countResp.GetCount(), int64(2)) @@ -886,8 +897,8 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { // Count with ExecutionStatus filter should only count scheduled callbacks. s.EventuallyWithT(func(t *assert.CollectT) { - countResp, err := s.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ - Namespace: s.Namespace().String(), + countResp, err := env.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ + Namespace: env.Namespace().String(), Query: `ExecutionStatus = "Running"`, }) require.NoError(t, err) @@ -898,14 +909,15 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { // TestStartCallbackExecution_SearchAttributes tests that search attributes provided at start // are persisted and can be used to query callback executions via list filtering. func (s *StandaloneCallbackSuite) TestStartCallbackExecution_SearchAttributes() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() callbackID := "sa-test-" + uuid.NewString() saValue := "sa-test-value-" + uuid.NewString() - _, err := s.FrontendClient().StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err := env.FrontendClient().StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ + Namespace: env.Namespace().String(), Identity: "test", RequestId: uuid.NewString(), CallbackId: callbackID, @@ -932,8 +944,8 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_SearchAttributes() // Verify the search attribute is queryable via list. s.EventuallyWithT(func(t *assert.CollectT) { - listResp, err := s.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ - Namespace: s.Namespace().String(), + listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ + Namespace: env.Namespace().String(), PageSize: 10, Query: fmt.Sprintf(`CustomKeywordField = %q AND CallbackId = %q`, saValue, callbackID), }) @@ -945,19 +957,20 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_SearchAttributes() // TestTerminateCallbackExecution tests terminate, run_id validation, and request ID idempotency. func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() callbackID := "terminate-test-" + uuid.NewString() - startResp := s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, time.Minute) + startResp := s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) requestID := uuid.NewString() runID := startResp.GetRunId() // Wrong run_id should fail for describe, poll, and terminate. wrongRunID := uuid.NewString() - _, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err := env.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: wrongRunID, }) @@ -965,15 +978,15 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { shortCtx, shortCancel := context.WithTimeout(ctx, time.Second*2) defer shortCancel() - _, err = s.FrontendClient().PollCallbackExecution(shortCtx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().PollCallbackExecution(shortCtx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: wrongRunID, }) s.Error(err) - _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: wrongRunID, Identity: "test", @@ -983,8 +996,8 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { s.Error(err) // Terminate with correct run_id and known request ID. - _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: runID, Identity: "test", @@ -994,8 +1007,8 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { s.NoError(err) // Describe after terminate — should be TERMINATED with correct run_id. - descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + descResp, err := env.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: runID, }) @@ -1006,8 +1019,8 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { s.Equal(runID, descResp.GetInfo().GetRunId()) // Poll to verify the outcome. - pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: runID, }) @@ -1016,8 +1029,8 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { s.Equal("testing terminate", pollResp.GetOutcome().GetFailure().GetMessage()) // Same request ID should be a no-op (idempotent). - _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, Identity: "test", RequestId: requestID, @@ -1026,8 +1039,8 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { s.NoError(err) // Different request ID should return FailedPrecondition. - _, err = s.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + _, err = env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, Identity: "test", RequestId: uuid.NewString(), @@ -1040,6 +1053,7 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { // TestCallbackExecutionFailedOutcome tests that when a callback fails with a non-retryable error // (e.g., a 400 response from the target), the poll outcome contains the failure details. func (s *StandaloneCallbackSuite) TestCallbackExecutionFailedOutcome() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -1060,11 +1074,11 @@ func (s *StandaloneCallbackSuite) TestCallbackExecutionFailedOutcome() { Success: testcore.MustToPayload(s.T(), "some-result"), }, } - s.callStartCallbackExecution(ctx, callbackID, cb, completion, time.Minute) + s.callStartCallbackExecution(ctx, env, callbackID, cb, completion, time.Minute) // Poll for the outcome — the callback should eventually fail with a non-retryable error. - pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, }) s.NoError(err) @@ -1086,6 +1100,7 @@ func (s *StandaloneCallbackSuite) TestCallbackExecutionFailedOutcome() { // 4. The operation can be completed from SCHEDULED state directly, so the workflow // receives the result regardless of the start handler timing. func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandlerReturns() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) defer cancel() @@ -1121,7 +1136,7 @@ func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandler Success: testcore.MustToPayload(s.T(), "result-before-start-returns"), }, } - s.callStartCallbackExecution(ctx, callbackID, cb, completion, time.Minute) + s.callStartCallbackExecution(ctx, env, callbackID, cb, completion, time.Minute) // Now return the async result — the completion has already been // delivered and the operation should already be completed. @@ -1133,7 +1148,7 @@ func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandler listenAddr := nexustest.AllocListenAddress() nexustest.NewNexusServer(s.T(), listenAddr, h) - _, err := s.OperatorClient().CreateNexusEndpoint(ctx, &operatorservice.CreateNexusEndpointRequest{ + _, err := env.OperatorClient().CreateNexusEndpoint(ctx, &operatorservice.CreateNexusEndpointRequest{ Spec: &nexuspb.EndpointSpec{ Name: endpointName, Target: &nexuspb.EndpointTarget{ @@ -1156,13 +1171,13 @@ func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandler return result, err } - w := worker.New(s.SdkClient(), taskQueue, worker.Options{}) + w := worker.New(env.SdkClient(), taskQueue, worker.Options{}) w.RegisterWorkflow(callerWf) s.NoError(w.Start()) defer w.Stop() // Start the caller workflow. - run, err := s.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{ + run, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{ TaskQueue: taskQueue, }, callerWf) s.NoError(err) @@ -1175,8 +1190,8 @@ func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandler s.Equal("result-before-start-returns", result) // Verify the operation token is recorded in the caller workflow's history. - histResp, err := s.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ - Namespace: s.Namespace().String(), + histResp, err := env.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), Execution: &commonpb.WorkflowExecution{ WorkflowId: run.GetID(), RunId: run.GetRunID(), @@ -1193,16 +1208,17 @@ func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandler // TestScheduleToCloseTimeout verifies that a callback execution transitions to FAILED // when its schedule-to-close timeout expires before the callback succeeds. func (s *StandaloneCallbackSuite) TestScheduleToCloseTimeout() { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() // Short timeout so it fires quickly during the test. callbackID := "timeout-test-" + uuid.NewString() - s.callStartCallbackExecutionToBogusCallback(ctx, callbackID, 2*time.Second) + s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, 2*time.Second) // Poll until the callback reaches a terminal state due to timeout. - pollResp, err := s.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + pollResp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, }) s.NoError(err) @@ -1213,8 +1229,8 @@ func (s *StandaloneCallbackSuite) TestScheduleToCloseTimeout() { s.Equal(enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) // Describe should show FAILED state with timeout failure. - descResp, err := s.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ - Namespace: s.Namespace().String(), + descResp, err := env.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: env.Namespace().String(), CallbackId: callbackID, IncludeOutcome: true, }) From e7bd633ae8856964818ab9ba35c738dfcf8012bb Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 8 May 2026 13:21:40 -0700 Subject: [PATCH 04/52] Expanding statemachine_test.go coverage --- chasm/lib/callback/statemachine_test.go | 222 +++++++++++++++--------- 1 file changed, 143 insertions(+), 79 deletions(-) diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index 63c5fc7e70a..6b7a4f9d40f 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -10,6 +10,7 @@ import ( callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/backoff" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestValidTransitions(t *testing.T) { @@ -28,107 +29,131 @@ func TestValidTransitions(t *testing.T) { } callback.SetStateMachineState(callbackspb.CALLBACK_STATUS_SCHEDULED) - // AttemptFailed - mctx := &chasm.MockMutableContext{} - mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + t.Run("TransitionAttemptFailed", func(t *testing.T) { + mctx := &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } - err := TransitionAttemptFailed.Apply(callback, mctx, EventAttemptFailed{ - Time: currentTime, - Err: errors.New("test"), - RetryPolicy: backoff.NewExponentialRetryPolicy(time.Second), + err := TransitionAttemptFailed.Apply(callback, mctx, EventAttemptFailed{ + Time: currentTime, + Err: errors.New("error message"), + RetryPolicy: backoff.NewExponentialRetryPolicy(time.Second), + }) + require.NoError(t, err) + + // Assert info object is updated. + require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, callback.StateMachineState()) + require.Equal(t, int32(1), callback.Attempt) + require.Equal(t, "error message", callback.LastAttemptFailure.Message) + require.False(t, callback.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) + require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) + dt := currentTime.Add(time.Second).Sub(callback.NextAttemptScheduleTime.AsTime()) + require.Less(t, dt, time.Millisecond*200) + + // Because of the retry policy, the first failure isn't terminal. + _, hasTermFailure := callback.TerminalFailure.TryGet(mctx) + require.False(t, hasTermFailure) + + // Assert backoff task is generated + require.Len(t, mctx.Tasks, 1) + require.IsType(t, &callbackspb.BackoffTask{}, mctx.Tasks[0].Payload) }) - require.NoError(t, err) - - // Assert info object is updated - require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, callback.StateMachineState()) - require.Equal(t, int32(1), callback.Attempt) - require.Equal(t, "test", callback.LastAttemptFailure.Message) - require.False(t, callback.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) - require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) - dt := currentTime.Add(time.Second).Sub(callback.NextAttemptScheduleTime.AsTime()) - require.Less(t, dt, time.Millisecond*200) - - // Assert backoff task is generated - require.Len(t, mctx.Tasks, 1) - require.IsType(t, &callbackspb.BackoffTask{}, mctx.Tasks[0].Payload) - - // Rescheduled - mctx = &chasm.MockMutableContext{} - err = TransitionRescheduled.Apply(callback, mctx, EventRescheduled{}) - require.NoError(t, err) - // Assert info object is updated only where needed - require.Equal(t, callbackspb.CALLBACK_STATUS_SCHEDULED, callback.StateMachineState()) - require.Equal(t, int32(1), callback.Attempt) - require.Equal(t, "test", callback.LastAttemptFailure.Message) - // Remains unmodified - require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) - require.Nil(t, callback.NextAttemptScheduleTime) - - // Assert callback task is generated - require.Len(t, mctx.Tasks, 1) - require.IsType(t, &callbackspb.InvocationTask{}, mctx.Tasks[0].Payload) + t.Run("TransitionRescheduled", func(t *testing.T) { + mctx := &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + + err := TransitionRescheduled.Apply(callback, mctx, EventRescheduled{}) + require.NoError(t, err) + + // Assert info object is only partially updated. + require.Equal(t, callbackspb.CALLBACK_STATUS_SCHEDULED, callback.StateMachineState()) + // Unmodified + require.Equal(t, int32(1), callback.Attempt) + require.Equal(t, "error message", callback.LastAttemptFailure.Message) + require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) + require.Nil(t, callback.NextAttemptScheduleTime) + _, hasTermFailure := callback.TerminalFailure.TryGet(mctx) + require.False(t, hasTermFailure) + + // Assert callback task is generated. + require.Len(t, mctx.Tasks, 1) + require.IsType(t, &callbackspb.InvocationTask{}, mctx.Tasks[0].Payload) + }) - // Store the pre-succeeded state to test Failed later + // Store the pre-succeeded state to test Failed later. dup := &Callback{ CallbackState: proto.Clone(callback.CallbackState).(*callbackspb.CallbackState), } dup.Status = callback.StateMachineState() - // Succeeded - currentTime = currentTime.Add(time.Second) - mctx = &chasm.MockMutableContext{} - mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + t.Run("TransitionSucceeded", func(t *testing.T) { + currentTime = currentTime.Add(time.Second) + mctx := &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } - err = TransitionSucceeded.Apply(callback, mctx, EventSucceeded{Time: currentTime}) - require.NoError(t, err) + err := TransitionSucceeded.Apply(callback, mctx, EventSucceeded{Time: currentTime}) + require.NoError(t, err) + + // Assert info object is updated. + require.Equal(t, callbackspb.CALLBACK_STATUS_SUCCEEDED, callback.StateMachineState()) + require.Equal(t, int32(2), callback.Attempt) + require.Nil(t, callback.LastAttemptFailure) + require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) + require.Nil(t, callback.NextAttemptScheduleTime) - // Assert info object is updated only where needed - require.Equal(t, callbackspb.CALLBACK_STATUS_SUCCEEDED, callback.StateMachineState()) - require.Equal(t, int32(2), callback.Attempt) - require.Nil(t, callback.LastAttemptFailure) - require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) - require.Nil(t, callback.NextAttemptScheduleTime) + // TerminalFailure may explicitly be set to nil. + termFailureValue, hasTermFailure := callback.TerminalFailure.TryGet(mctx) + require.True(t, !hasTermFailure || termFailureValue == nil) - // Assert no task is generated on success transition - require.Empty(t, mctx.Tasks) + // Assert no task is generated on success transition + require.Empty(t, mctx.Tasks) + }) - // Reset back to scheduled + // Reset back to the scheduled state. callback = dup // Increment the time to ensure it's updated in the transition currentTime = currentTime.Add(time.Second) - // failed - mctx = &chasm.MockMutableContext{} - mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } - - err = TransitionFailed.Apply(callback, mctx, EventFailed{Time: currentTime, Err: errors.New("failed")}) - require.NoError(t, err) - - // Assert info object is updated only where needed - require.Equal(t, callbackspb.CALLBACK_STATUS_FAILED, callback.StateMachineState()) - require.Equal(t, int32(2), callback.Attempt) - require.Equal(t, "failed", callback.LastAttemptFailure.Message) - require.True(t, callback.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) - require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) - require.Nil(t, callback.NextAttemptScheduleTime) - - // Assert task is not generated, failed is terminal - require.Empty(t, mctx.Tasks) + t.Run("TransitionFailed", func(t *testing.T) { + mctx := &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + + err := TransitionFailed.Apply(callback, mctx, EventFailed{Time: currentTime, Err: errors.New("failed")}) + require.NoError(t, err) + + // Assert info object is updated. + require.Equal(t, callbackspb.CALLBACK_STATUS_FAILED, callback.StateMachineState()) + require.Equal(t, int32(2), callback.Attempt) + require.Equal(t, "failed", callback.LastAttemptFailure.Message) + require.True(t, callback.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) + require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) + require.Nil(t, callback.NextAttemptScheduleTime) + + // Check the TerminalFailure field is set. + require.NotNil(t, callback.TerminalFailure, "TerminalFailure not set") + got := callback.TerminalFailure.Get(mctx) + want := callback.LastAttemptFailure + require.True(t, proto.Equal(want, got), "TerminalFailure not as expected. Got %v, want %v.", got, want) + + // Assert no tasks generated. In terminal state. + require.Empty(t, mctx.Tasks) + }) } func TestTerminatedTransition(t *testing.T) { - callback := &Callback{ - CallbackState: &callbackspb.CallbackState{ - Callback: &callbackspb.Callback{ - Variant: &callbackspb.Callback_Nexus_{ - Nexus: &callbackspb.Callback_Nexus{ - Url: "http://address:999/path", - }, + initialCallbackState := &callbackspb.CallbackState{ + RegistrationTime: timestamppb.New(time.Now()), + Callback: &callbackspb.Callback{ + Variant: &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{ + Url: "http://address:999/path", }, }, }, } + initialCallback := &Callback{ + CallbackState: initialCallbackState, + } for _, src := range []callbackspb.CallbackStatus{ callbackspb.CALLBACK_STATUS_STANDBY, @@ -136,12 +161,51 @@ func TestTerminatedTransition(t *testing.T) { callbackspb.CALLBACK_STATUS_BACKING_OFF, } { t.Run("from_"+src.String(), func(t *testing.T) { - cb := &Callback{CallbackState: proto.Clone(callback.CallbackState).(*callbackspb.CallbackState)} - cb.SetStateMachineState(src) + currentTime := time.Now().UTC() mctx := &chasm.MockMutableContext{} + mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + + cb := &Callback{ + CallbackState: proto.Clone(initialCallbackState).(*callbackspb.CallbackState), + } + cb.SetStateMachineState(src) + err := TransitionTerminated.Apply(cb, mctx, EventTerminated{}) require.NoError(t, err) + + // Confirm expected state changes. require.Equal(t, callbackspb.CALLBACK_STATUS_TERMINATED, cb.StateMachineState()) + require.Equal(t, currentTime, cb.GetCloseTime().AsTime()) + // Other fields remain the same. + require.True(t, proto.Equal(initialCallbackState.Callback, cb.Callback)) + require.True(t, proto.Equal(initialCallbackState.RegistrationTime, cb.RegistrationTime)) + require.Equal(t, initialCallback.LastAttemptFailure, cb.LastAttemptFailure) + + // Confirm the Callback's terminal failure reason is set. + termFailure := cb.TerminalFailure.Get(mctx) + require.Equal(t, "callback execution terminated", termFailure.Message) + + // No new tasks were generated. + require.Empty(t, mctx.Tasks) + }) + } + + // Terminal states. Confirm the request should be rejected by CHASM. + for _, src := range []callbackspb.CallbackStatus{ + callbackspb.CALLBACK_STATUS_SUCCEEDED, + callbackspb.CALLBACK_STATUS_FAILED, + callbackspb.CALLBACK_STATUS_TERMINATED, + } { + t.Run("from_"+src.String(), func(t *testing.T) { + mctx := &chasm.MockMutableContext{} + + cb := &Callback{ + CallbackState: proto.Clone(initialCallbackState).(*callbackspb.CallbackState), + } + cb.SetStateMachineState(src) + + err := TransitionTerminated.Apply(cb, mctx, EventTerminated{}) + require.ErrorContains(t, err, "invalid transition from") }) } } From 9aea1a60874eda6b43c806cf7ea027d96c555ab8 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 8 May 2026 14:32:24 -0700 Subject: [PATCH 05/52] Address race when completing an unstarted Nexus operation --- chasm/lib/callback/component_test.go | 52 ++++++++++++++++++++++++ chasm/lib/callback/invocable_outbound.go | 25 ++++++++++++ components/nexusoperations/completion.go | 13 ++++++ 3 files changed, 90 insertions(+) create mode 100644 chasm/lib/callback/component_test.go diff --git a/chasm/lib/callback/component_test.go b/chasm/lib/callback/component_test.go new file mode 100644 index 00000000000..7cf38f19042 --- /dev/null +++ b/chasm/lib/callback/component_test.go @@ -0,0 +1,52 @@ +package callback + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common/backoff" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/components/nexusoperations" +) + +// Confirm that callback delivery failures due to the Nexus operation not having +// started will be retried and not trigger circuit breakers. +func TestCallbacksToUnstartedNexusOperations(t *testing.T) { + cb := &Callback{ + CallbackState: &callbackspb.CallbackState{ + Callback: &callbackspb.Callback{ + Variant: &callbackspb.Callback_Nexus_{ + Nexus: &callbackspb.Callback_Nexus{ + Url: commonnexus.SystemCallbackURL, + }, + }, + }, + Status: callbackspb.CALLBACK_STATUS_SCHEDULED, + }, + } + + // Simulate the InvocationTask being executed, which ends with the invocation's + // result being saved on the Callback. + mctx := &chasm.MockMutableContext{} + _, err := cb.saveResult(mctx, saveResultInput{ + result: invocationResultRetry{err: nexusoperations.ErrOperationNotStarted}, + retryPolicy: backoff.NewExponentialRetryPolicy(time.Second), + }) + + require.NoError(t, err) + require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, cb.StateMachineState()) + require.Equal(t, int32(1), cb.Attempt) + require.Equal(t, "nexus operation not started", cb.LastAttemptFailure.Message) + require.False(t, cb.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) + require.NotNil(t, cb.NextAttemptScheduleTime) + + _, ok := cb.TerminalFailure.TryGet(mctx) + require.False(t, ok) + + // Confirm backoff task was generated. + require.Len(t, mctx.Tasks, 1) + require.IsType(t, &callbackspb.BackoffTask{}, mctx.Tasks[0].Payload) +} diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index f8bafda576c..ccac789e159 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -15,6 +15,7 @@ import ( "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/components/nexusoperations" queuescommon "go.temporal.io/server/service/history/queues/common" queueserrors "go.temporal.io/server/service/history/queues/errors" ) @@ -28,7 +29,22 @@ type invocableOutbound struct { attempt int32 } +func (n invocableOutbound) isSystemCallback() bool { + c := n.callback + if c == nil { + return false + } + return c.Url == commonnexus.SystemCallbackURL +} + func (n invocableOutbound) WrapError(result invocationResult, err error) error { + // If the error is due to a completion of a Nexus operation being delivered before the + // operation has officially started, we want to avoid triggering the circuit breakers. + // Since the actual destination is working fine, and the failure is due to a data race. + if errors.Is(err, nexusoperations.ErrOperationNotStarted) { + return err + } + if retry, ok := result.(invocationResultRetry); ok { return queueserrors.NewDestinationDownError(retry.err.Error(), err) } @@ -88,6 +104,15 @@ func (n invocableOutbound) Invoke( if err != nil { retryable := isRetryableCallError(err) h.logger.Error("Callback request failed", tag.Error(err), tag.Bool("retryable", retryable)) + + // If the error from trying to resolve a Nexus operation that hasn't yet been marked + // as started, it is safe to retry. (n.WrapError will ensure repeated failures of this + // kind won't cause the SystemCallback to trip the circuit breaker.) + isErrNotStarted := errors.Is(err, nexusoperations.ErrOperationNotStarted) + if n.isSystemCallback() && isErrNotStarted { + retryable = true + } + if retryable { return invocationResultRetry{err} } diff --git a/components/nexusoperations/completion.go b/components/nexusoperations/completion.go index cf37247030b..ec10497f7ca 100644 --- a/components/nexusoperations/completion.go +++ b/components/nexusoperations/completion.go @@ -16,6 +16,11 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +// ErrOperationNotStarted is returned when a completion arrives before the operation has +// started and no operation token is provided. This error is used by the callback invocation +// layer to detect this specific condition and retry without triggering the circuit breaker. +var ErrOperationNotStarted = serviceerror.NewFailedPrecondition("nexus operation not started") + func handleSuccessfulOperationResult( node *hsm.Node, operation Operation, @@ -136,6 +141,14 @@ func fabricateStartedEventIfMissing( return nil } + // If the operation hasn't started yet and the completion doesn't include an operation token, + // reject the request. This handles the race where a completion arrives before the start + // handler returns with the operation token. The caller will retry and by then the start + // handler will have returned and recorded the token. + if operationToken == "" { + return ErrOperationNotStarted + } + eventID, err := hsm.EventIDFromToken(operation.ScheduledEventToken) if err != nil { return err From 9788a8256e45a1feb46adfa02d673d1bfb7ccc4b Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 8 May 2026 15:27:32 -0700 Subject: [PATCH 06/52] Address PR feedback --- chasm/lib/callback/library.go | 3 +-- chasm/lib/callback/statemachine.go | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/chasm/lib/callback/library.go b/chasm/lib/callback/library.go index 90785feb9f0..e993ceccda4 100644 --- a/chasm/lib/callback/library.go +++ b/chasm/lib/callback/library.go @@ -3,7 +3,6 @@ package callback import ( "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" - "go.temporal.io/server/common/namespace" "google.golang.org/grpc" ) @@ -12,7 +11,7 @@ type componentOnlyLibrary struct { chasm.UnimplementedLibrary } -func newComponentOnlyLibrary(config *Config, namespaceRegistry namespace.Registry) *componentOnlyLibrary { +func newComponentOnlyLibrary() *componentOnlyLibrary { return &componentOnlyLibrary{} } diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 1d587658864..bf9b7a39226 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -64,7 +64,6 @@ var TransitionAttemptFailed = chasm.NewTransition( func(cb *Callback, ctx chasm.MutableContext, event EventAttemptFailed) error { now := ctx.Now(cb) cb.recordAttempt(now) - cb.CloseTime = timestamppb.New(now) // Use 0 for elapsed time as we don't limit the retry by time (for now). nextDelay := event.RetryPolicy.ComputeNextDelay(0, int(cb.Attempt), event.Err) From 34ca85f8c4a88fc7fde40286f619fd7bbb9b73c3 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 9 May 2026 10:28:48 -0700 Subject: [PATCH 07/52] Count DescribeCallbackExecution as long poll as applicable --- .../rpc/interceptor/namespace_rate_limit.go | 37 +++++----- .../interceptor/namespace_rate_limit_test.go | 74 +++++++++++++++++++ service/frontend/configs/quotas.go | 25 +++++-- 3 files changed, 110 insertions(+), 26 deletions(-) diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index 1bcf3b26a20..3561b7932f0 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -76,11 +76,13 @@ func (ni *NamespaceRateLimitInterceptorImpl) Intercept( ) (any, error) { if ns := MustGetNamespaceName(ni.namespaceRegistry, req); ns != namespace.EmptyName { method := info.FullMethod - if IsLongPollGetWorkflowExecutionHistoryRequest(req) { - method = configs.PollWorkflowHistoryAPIName - } else if IsLongPollDescribeActivityExecutionRequest(req) { - method = configs.PollActivityExecutionAPIName + + // Potentially consider the request as something else, for cases where a typically + // normal API is called in a long polling-mode. + if longPollingName, ok := mapToLongPollingRequest(req); ok { + method = longPollingName } + if ni.pollWaitForToken(ns.String()) { if _, ok := ni.pollMethods[info.FullMethod]; ok { if err := ni.Wait(ctx, ns, method, headers.NewGRPCHeaderGetter(ctx)); err != nil { @@ -158,22 +160,21 @@ func (ni *NamespaceRateLimitInterceptorImpl) Allow(namespaceName namespace.Name, return nil } -func IsLongPollGetWorkflowExecutionHistoryRequest( - req any, -) bool { +// mapToLongPollingRequest returns the endpoint name that should be used instead, if the +// request is a non-standard long polling operation. +// +// e.g. DescribeActivityExecution is usually sync. But takes an optional LongPollToken, +// which if set, then the request should be treated as the dedicated long polling version +// (configs.PollActivityExecutionAPIName) instead. +func mapToLongPollingRequest(req any) (string, bool) { switch request := req.(type) { case *workflowservice.GetWorkflowExecutionHistoryRequest: - return request.GetWaitNewEvent() - } - return false -} - -func IsLongPollDescribeActivityExecutionRequest( - req any, -) bool { - switch request := req.(type) { + return configs.PollWorkflowHistoryAPIName, request.GetWaitNewEvent() case *workflowservice.DescribeActivityExecutionRequest: - return len(request.GetLongPollToken()) > 0 + return configs.PollActivityExecutionAPIName, len(request.GetLongPollToken()) > 0 + case *workflowservice.DescribeCallbackExecutionRequest: + return configs.PollCallbackExecutionAPIName, len(request.GetLongPollToken()) > 0 } - return false + + return "", false } diff --git a/common/rpc/interceptor/namespace_rate_limit_test.go b/common/rpc/interceptor/namespace_rate_limit_test.go index dc8d4f90be3..3788e8ad760 100644 --- a/common/rpc/interceptor/namespace_rate_limit_test.go +++ b/common/rpc/interceptor/namespace_rate_limit_test.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/quotas" + "go.temporal.io/server/service/frontend/configs" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) @@ -233,6 +234,79 @@ func (s *namespaceRateLimitInterceptorSuite) TestIntercept_PollMethod_WaitEnable s.ErrorIs(err, ErrNamespaceRateLimitServerBusy) } +func TestMapToLongPollingRequest(t *testing.T) { + emptyToken := []byte{} + validToken := []byte("long-poll-token") + + tests := []struct { + name string + req any + wantMethod string + wantOk bool + }{ + // GetWorkflowExecutionHistory + { + name: "GetWorkflowExecutionHistory(WaitNewEvent=true)", + req: &workflowservice.GetWorkflowExecutionHistoryRequest{WaitNewEvent: true}, + wantMethod: configs.PollWorkflowHistoryAPIName, + wantOk: true, + }, + { + name: "GetWorkflowExecutionHistory(WaitNewEvent=false)", + req: &workflowservice.GetWorkflowExecutionHistoryRequest{WaitNewEvent: false}, + wantMethod: configs.PollWorkflowHistoryAPIName, + wantOk: false, + }, + // DescribeActivityExecution + { + name: "DescribeActivityExecution(LongPollToken='')", + req: &workflowservice.DescribeActivityExecutionRequest{LongPollToken: emptyToken}, + wantMethod: configs.PollActivityExecutionAPIName, + wantOk: false, + }, + { + name: "DescribeActivityExecution(LongPollToken='...')", + req: &workflowservice.DescribeActivityExecutionRequest{LongPollToken: validToken}, + wantMethod: configs.PollActivityExecutionAPIName, + wantOk: true, + }, + // DescribeCallbackExecution + { + name: "DescribeCallbackExecution(LongPollToken='')", + req: &workflowservice.DescribeCallbackExecutionRequest{LongPollToken: emptyToken}, + wantMethod: configs.PollCallbackExecutionAPIName, + wantOk: false, + }, + { + name: "DescribeCallbackExecution(LongPollToken='...')", + req: &workflowservice.DescribeCallbackExecutionRequest{LongPollToken: validToken}, + wantMethod: configs.PollCallbackExecutionAPIName, + wantOk: true, + }, + // Other + { + name: "Unrelated request type", + req: &workflowservice.StartWorkflowExecutionRequest{}, + wantMethod: "", + wantOk: false, + }, + { + name: "Nil request", + req: nil, + wantMethod: "", + wantOk: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotMethod, gotOk := mapToLongPollingRequest(tc.req) + require.Equal(t, tc.wantMethod, gotMethod) + require.Equal(t, tc.wantOk, gotOk) + }) + } +} + // noopHeaderGetter implements headers.HeaderGetter with empty values. type noopHeaderGetter struct{} diff --git a/service/frontend/configs/quotas.go b/service/frontend/configs/quotas.go index ba8bd19806f..555bb94fb16 100644 --- a/service/frontend/configs/quotas.go +++ b/service/frontend/configs/quotas.go @@ -20,8 +20,11 @@ const ( CompleteNexusOperation = "/temporal.api.nexusservice.v1.NexusService/CompleteNexusOperation" // PollWorkflowHistoryAPIName is used instead of GetWorkflowExecutionHistory if WaitNewEvent is true in request. PollWorkflowHistoryAPIName = "/temporal.api.workflowservice.v1.WorkflowService/PollWorkflowExecutionHistory" - // PollActivityExecutionAPIName is used instead of DescribeActivityExecution if LongPollToken is set in request. + + // Poll{resource}ExecutionAPIName is used instead of Describe{resource}Execution if LongPollToken is set in request, + // since that impacts how the operation would get billed. PollActivityExecutionAPIName = "/temporal.api.workflowservice.v1.WorkflowService/PollActivityExecutionDescription" + PollCallbackExecutionAPIName = "/temporal.api.workflowservice.v1.WorkflowService/PollCallbackExecutionDescription" ) var ( @@ -29,11 +32,13 @@ var ( // from their corresponding quota, which is determined by // dynamicconfig.FrontendMaxConcurrentLongRunningRequestsPerInstance. If the value is not set, // then the method is not considered a long-running request and the number of concurrent - // requests will not be throttled. The Poll* methods here are long-running because they block - // until there is a task available. GetWorkflowExecutionHistory and DescribeActivityExecution - // methods are blocking only if WaitNewEvent/LongPollToken are set, otherwise they are not - // long-running. The QueryWorkflow and UpdateWorkflowExecution methods are long-running because - // they both block until a background WFT is complete. + // requests will not be throttled. + // + // The Poll* methods here are long-running because they block until there is a task available. + // Other methods, like GetWorkflowExecutionHistory or DescribeActivityExecution, are blocking only + // if WaitNewEvent/LongPollToken are set, otherwise they are not long-running. The QueryWorkflow + // and UpdateWorkflowExecution methods are long-running because they both block until a background + // WFT is complete. ExecutionAPICountLimitOverride = map[string]int{ // These methods here are long-running because they block until there is a task available. "/temporal.api.workflowservice.v1.WorkflowService/PollActivityTaskQueue": 1, @@ -42,8 +47,9 @@ var ( "/temporal.api.workflowservice.v1.WorkflowService/PollNexusTaskQueue": 1, "/temporal.api.workflowservice.v1.WorkflowService/PollNexusOperationExecution": 1, - // Long-running if activity outcome is not already available + // Long-running if the outcome is the resources isn't in a terminal state. "/temporal.api.workflowservice.v1.WorkflowService/PollActivityExecution": 1, + "/temporal.api.workflowservice.v1.WorkflowService/PollCallbackExecution": 1, // These methods are long-running because they block until a background WFT is complete. "/temporal.api.workflowservice.v1.WorkflowService/QueryWorkflow": 1, @@ -53,6 +59,7 @@ var ( "/temporal.api.workflowservice.v1.WorkflowService/DescribeNexusOperationExecution": 1, "/temporal.api.workflowservice.v1.WorkflowService/GetWorkflowExecutionHistory": 1, "/temporal.api.workflowservice.v1.WorkflowService/DescribeActivityExecution": 1, + "/temporal.api.workflowservice.v1.WorkflowService/DescribeCallbackExecution": 1, // Potentially long-running, depending on the operations. "/temporal.api.workflowservice.v1.WorkflowService/ExecuteMultiOperation": 1, @@ -200,10 +207,12 @@ var ( // P5: Low priority APIs // GetWorkflowExecutionHistory with WaitNewEvent set to true is a long poll API. - // Similarly, DescribeActivityExecution is a long poll API if LongPollToken is set. + // Similarly, `Describe{resource}Execution` are a long poll API if LongPollToken is set. // Treat these as long-poll but lower priority (5) so spikes don’t block Poll* APIs. PollWorkflowHistoryAPIName: 5, PollActivityExecutionAPIName: 5, + PollCallbackExecutionAPIName: 5, + // Informational API that aren't required for the temporal service to function OpenAPIV3APIName: 5, OpenAPIV2APIName: 5, From e4cad82a8c30e2425bd54229a5aad2f9eb18d502 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 9 May 2026 10:31:55 -0700 Subject: [PATCH 08/52] Address PR feedback --- chasm/lib/callback/frontend.go | 13 +- chasm/lib/callback/frontend_validation.go | 113 ++++++++++-------- .../lib/callback/frontend_validation_test.go | 42 +++++++ chasm/lib/callback/library.go | 1 - chasm/lib/callback/statemachine.go | 1 + chasm/lib/callback/statemachine_test.go | 4 + tests/standalone_callbacks_test.go | 13 +- 7 files changed, 133 insertions(+), 54 deletions(-) create mode 100644 chasm/lib/callback/frontend_validation_test.go diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go index b2d79d6942f..40d5aef5fa4 100644 --- a/chasm/lib/callback/frontend.go +++ b/chasm/lib/callback/frontend.go @@ -2,6 +2,7 @@ package callback import ( "context" + "time" callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" @@ -273,12 +274,20 @@ func (h *frontendHandler) ListCallbackExecutions( statusStr, _ := chasm.SearchAttributeValue(exec.ChasmSearchAttributes, executionStatusSearchAttribute) status, _ := enumspb.CallbackExecutionStatusFromString(statusStr) + // Map time.Time(0) to nil, e.g. for running Callbacks. + convertTime := func(t time.Time) *timestamppb.Timestamp { + if !t.IsZero() { + return timestamppb.New(t) + } + return nil + } + info := callbackpb.CallbackExecutionListInfo{ CallbackId: exec.BusinessID, RunId: exec.RunID, Status: status, - CreateTime: timestamppb.New(exec.StartTime), - CloseTime: timestamppb.New(exec.CloseTime), + CreateTime: convertTime(exec.StartTime), + CloseTime: convertTime(exec.CloseTime), SearchAttributes: &commonpb.SearchAttributes{IndexedFields: exec.CustomSearchAttributes}, StateTransitionCount: exec.StateTransitionCount, } diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index 69ad72df517..e9c69489326 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -47,6 +47,32 @@ func verifyCallbackIDLength(reqProto CallbackIDer, config *Config) error { return nil } +// requiredField is a tuple of a required field name and its value. +// Used instead of a map[string]string to provide deterministic +// errors if multiple fields aren't set. +type requiredField struct { + FieldName string + Value string +} + +func (rf requiredField) Validate() error { + if rf.Value == "" { + return missingRequiredFieldError(rf.FieldName) + } + return nil +} + +type requiredFields []requiredField + +func (fields requiredFields) Validate() error { + for _, rf := range fields { + if err := rf.Validate(); err != nil { + return err + } + } + return nil +} + // frontendRequestValidator bundles the configuration data for validating an incomming request. // // IMPORTANT: Validation methods MAY mutate the incomming request, in order to ensure they all have @@ -65,17 +91,15 @@ func (rv *frontendRequestValidator) ValidateStartCallbackExecution(req *workflow req.RequestId = uuid.NewString() } - // Required fields. - requiredFields := map[string]string{ - "Namespace": req.GetNamespace(), - "Identity": req.GetIdentity(), - "RequestId": req.GetRequestId(), - "CallbackId": req.GetCallbackId(), + // Check required fields each have a value. + requiredFields := requiredFields{ + {"Namespace", req.GetNamespace()}, + {"Identity", req.GetIdentity()}, + {"RequestId", req.GetRequestId()}, + {"CallbackId", req.GetCallbackId()}, } - for k, v := range requiredFields { - if v == "" { - return missingRequiredFieldError(k) - } + if err := requiredFields.Validate(); err != nil { + return err } // Field lengths @@ -162,15 +186,13 @@ func (rv *frontendRequestValidator) validateSearchAttributes(req Namespacer, saT } func (rv *frontendRequestValidator) ValidateDescribeCallbackExecution(req *workflowservice.DescribeCallbackExecutionRequest) error { - // Required fields. - requiredFields := map[string]string{ - "Namespace": req.GetNamespace(), - "CallbackId": req.GetCallbackId(), - } - for k, v := range requiredFields { - if v == "" { - return missingRequiredFieldError(k) - } + // Check required fields each have a value. + requiredFields := requiredFields{ + {"Namespace", req.GetNamespace()}, + {"CallbackId", req.GetCallbackId()}, + } + if err := requiredFields.Validate(); err != nil { + return err } // Field lengths @@ -187,15 +209,13 @@ func (rv *frontendRequestValidator) ValidateDescribeCallbackExecution(req *workf } func (rv *frontendRequestValidator) ValidatePollCallbackExecution(req *workflowservice.PollCallbackExecutionRequest) error { - // Required fields. - requiredFields := map[string]string{ - "Namespace": req.GetNamespace(), - "CallbackId": req.GetCallbackId(), - } - for k, v := range requiredFields { - if v == "" { - return missingRequiredFieldError(k) - } + // Check required fields each have a value. + requiredFields := requiredFields{ + {"Namespace", req.GetNamespace()}, + {"CallbackId", req.GetCallbackId()}, + } + if err := requiredFields.Validate(); err != nil { + return err } // Field lengths @@ -208,19 +228,16 @@ func (rv *frontendRequestValidator) ValidateTerminateCallbackExecution(req *work req.RequestId = uuid.NewString() } - // Required fields. - requiredFields := map[string]string{ - "RequestId": req.GetRequestId(), - "Namespace": req.GetNamespace(), - "CallbackId": req.GetCallbackId(), - - // NOTE: We don't require the Identity or Reason fields to be set, - // and just set reasonable defaults. + // Check required fields each have a value. + // NOTE: We don't require the Identity or Reason fields to be set, + // and just set reasonable defaults. + requiredFields := requiredFields{ + {"RequestId", req.GetRequestId()}, + {"Namespace", req.GetNamespace()}, + {"CallbackId", req.GetCallbackId()}, } - for k, v := range requiredFields { - if v == "" { - return missingRequiredFieldError(k) - } + if err := requiredFields.Validate(); err != nil { + return err } // Field lengths @@ -231,15 +248,13 @@ func (rv *frontendRequestValidator) ValidateTerminateCallbackExecution(req *work } func (rv *frontendRequestValidator) ValidateDeleteCallbackExecution(req *workflowservice.DeleteCallbackExecutionRequest) error { - // Required fields. - requiredFields := map[string]string{ - "Namespace": req.GetNamespace(), - "CallbackId": req.GetCallbackId(), - } - for k, v := range requiredFields { - if v == "" { - return missingRequiredFieldError(k) - } + // Check required fields each have a value. + requiredFields := requiredFields{ + {"Namespace", req.GetNamespace()}, + {"CallbackId", req.GetCallbackId()}, + } + if err := requiredFields.Validate(); err != nil { + return err } // Field lengths diff --git a/chasm/lib/callback/frontend_validation_test.go b/chasm/lib/callback/frontend_validation_test.go new file mode 100644 index 00000000000..d4f9d7fee7c --- /dev/null +++ b/chasm/lib/callback/frontend_validation_test.go @@ -0,0 +1,42 @@ +package callback + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRequiredFields(t *testing.T) { + // Positive tests + positiveTests := requiredFields{ + {"Field1", "exists"}, + {"Field2", " "}, // Whitespace, but still non-empty. + } + require.NoError(t, positiveTests.Validate()) + for _, positiveTest := range positiveTests { + require.NoError(t, positiveTest.Validate()) + } + + // Negative tests + negativeTests := requiredFields{ + {"Field1", ""}, + {"Field2", ""}, + } + require.ErrorContains(t, negativeTests.Validate(), "Field1 is not set on request.") + for _, negativeTest := range negativeTests { + wantErr := fmt.Sprintf("%s is not set on request.", negativeTest.FieldName) + require.ErrorContains(t, negativeTest.Validate(), wantErr) + } + + // Special case: Confirm the validation error is stable, in that it + // is always the first invalid field in the slice. + mixedTests := requiredFields{ + {"Mixed Field1", "ok"}, + {"Mixed Field2", ""}, + {"Mixed Field3", ""}, + {"Mixed Field4", "ok"}, + {"Mixed Field5", ""}, + } + require.ErrorContains(t, mixedTests.Validate(), "Mixed Field2 is not set on request.") +} diff --git a/chasm/lib/callback/library.go b/chasm/lib/callback/library.go index e993ceccda4..19a672aa4dc 100644 --- a/chasm/lib/callback/library.go +++ b/chasm/lib/callback/library.go @@ -33,7 +33,6 @@ func (l *componentOnlyLibrary) Components() []*chasm.RegistrableComponent { type library struct { componentOnlyLibrary - config *Config InvocationTaskHandler *invocationTaskHandler BackoffTaskHandler *backoffTaskHandler CompletionScheduleToCloseTimeoutTaskHandler *CompletionScheduleToCloseTimeoutTaskHandler diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index bf9b7a39226..5c5f1ee1835 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -126,6 +126,7 @@ var TransitionSucceeded = chasm.NewTransition( func(cb *Callback, ctx chasm.MutableContext, event EventSucceeded) error { now := ctx.Now(cb) cb.recordAttempt(now) + cb.CloseTime = timestamppb.New(now) cb.LastAttemptFailure = nil cb.TerminalFailure = chasm.NewDataField[*failurepb.Failure](ctx, nil) return nil diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index 6b7a4f9d40f..447a0d0de5f 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -50,6 +50,7 @@ func TestValidTransitions(t *testing.T) { require.Less(t, dt, time.Millisecond*200) // Because of the retry policy, the first failure isn't terminal. + require.Nil(t, callback.CloseTime) _, hasTermFailure := callback.TerminalFailure.TryGet(mctx) require.False(t, hasTermFailure) @@ -71,6 +72,7 @@ func TestValidTransitions(t *testing.T) { require.Equal(t, int32(1), callback.Attempt) require.Equal(t, "error message", callback.LastAttemptFailure.Message) require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) + require.Nil(t, callback.CloseTime) require.Nil(t, callback.NextAttemptScheduleTime) _, hasTermFailure := callback.TerminalFailure.TryGet(mctx) require.False(t, hasTermFailure) @@ -100,6 +102,7 @@ func TestValidTransitions(t *testing.T) { require.Nil(t, callback.LastAttemptFailure) require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) require.Nil(t, callback.NextAttemptScheduleTime) + require.Equal(t, currentTime, callback.CloseTime.AsTime()) // TerminalFailure may explicitly be set to nil. termFailureValue, hasTermFailure := callback.TerminalFailure.TryGet(mctx) @@ -128,6 +131,7 @@ func TestValidTransitions(t *testing.T) { require.True(t, callback.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) require.Nil(t, callback.NextAttemptScheduleTime) + require.Equal(t, currentTime, callback.CloseTime.AsTime()) // Check the TerminalFailure field is set. require.NotNil(t, callback.TerminalFailure, "TerminalFailure not set") diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index 7c7bf9e2e57..ddbbf6a94be 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -861,7 +861,10 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { }) require.NoError(t, err) require.Len(t, listResp.GetExecutions(), 1) - require.Equal(t, callbackIDs[0], listResp.GetExecutions()[0].GetCallbackId()) + gotCb := listResp.GetExecutions()[0] + require.Equal(t, callbackIDs[0], gotCb.GetCallbackId()) + require.NotNil(t, gotCb.CreateTime) + require.Nil(t, gotCb.CloseTime) // Not in terminal state. }, waitUpTo, checkInterval, "Didn't find Running callback") // Terminate one callback to test filtering by terminal status. @@ -883,7 +886,13 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { }) require.NoError(t, err) require.Len(t, listResp.GetExecutions(), 1) - require.Equal(t, callbackIDs[1], listResp.GetExecutions()[0].GetCallbackId()) + gotCb := listResp.GetExecutions()[0] + require.Equal(t, callbackIDs[1], gotCb.GetCallbackId()) + require.NotNil(t, gotCb.CreateTime) + require.NotNil(t, gotCb.CloseTime) + // If Created and CloseTime are within 1ns, either the system is crazy-fast + // or there is a bug where we are not fetching the current time twice. + require.Greater(t, gotCb.CloseTime.AsTime(), gotCb.CreateTime.AsTime()) }, waitUpTo, checkInterval, "Didn't find Terminated callbacks") // Count callback executions. From e462fae9984c346c86cb347d59a711f0fe432163 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 9 May 2026 11:24:01 -0700 Subject: [PATCH 09/52] Address PR feedback --- chasm/lib/callback/fx.go | 4 ++-- chasm/lib/callback/gen/callbackpb/v1/message.pb.go | 2 +- chasm/lib/callback/library.go | 4 ++-- chasm/lib/callback/proto/v1/message.proto | 2 +- chasm/lib/callback/tasks.go | 12 ++++++------ 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/chasm/lib/callback/fx.go b/chasm/lib/callback/fx.go index 3d4401db7f3..a781ad25899 100644 --- a/chasm/lib/callback/fx.go +++ b/chasm/lib/callback/fx.go @@ -49,7 +49,7 @@ func httpCallerProviderProvider( // FrontendModule just contains the CHASM components, but not their implementation. var FrontendModule = fx.Module( - "callback-frontend", + "chasm.lib.callback.frontend", fx.Provide(callbackspb.NewCallbackServiceLayeredClient), fx.Provide(ConfigProvider), fx.Provide(NewFrontendHandler), @@ -67,7 +67,7 @@ var HistoryModule = fx.Module( fx.Provide(newInvocationTaskHandler), fx.Provide(newBackoffTaskHandler), fx.Provide(newCallbackHandler), - fx.Provide(NewCompletionScheduleToCloseTimeoutTaskHandler), + fx.Provide(newCompletionScheduleToCloseTimeoutTaskHandler), fx.Provide(newLibrary), fx.Invoke(func(registry *chasm.Registry, library *library) error { diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index 6ae4eed6fd5..71e7e9e2411 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -139,7 +139,7 @@ type CallbackState struct { // The time when the callback reached a terminal state. CloseTime *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=close_time,json=closeTime,proto3" json:"close_time,omitempty"` // (standalone only) User-supplied business ID set when StartCallbackExecution() is - // called. Used to identify the callback for operations like Describe- or Terminate-. + // called. Used to identify the callback for operations like Describe- or Terminate-. CallbackId string `protobuf:"bytes,12,opt,name=callback_id,json=callbackId,proto3" json:"callback_id,omitempty"` // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() // is called to when the result gets delivered. diff --git a/chasm/lib/callback/library.go b/chasm/lib/callback/library.go index 19a672aa4dc..1c0e3545ea8 100644 --- a/chasm/lib/callback/library.go +++ b/chasm/lib/callback/library.go @@ -35,14 +35,14 @@ type library struct { InvocationTaskHandler *invocationTaskHandler BackoffTaskHandler *backoffTaskHandler - CompletionScheduleToCloseTimeoutTaskHandler *CompletionScheduleToCloseTimeoutTaskHandler + CompletionScheduleToCloseTimeoutTaskHandler *completionScheduleToCloseTimeoutTaskHandler callbackSvcHandler *callbackHandler } func newLibrary( InvocationTaskHandler *invocationTaskHandler, BackoffTaskHandler *backoffTaskHandler, - CompletionScheduleToCloseTimeoutTaskHandler *CompletionScheduleToCloseTimeoutTaskHandler, + CompletionScheduleToCloseTimeoutTaskHandler *completionScheduleToCloseTimeoutTaskHandler, callbackSvcHandler *callbackHandler, ) *library { return &library{ diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index b6d78a6c764..15433f31460 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -41,7 +41,7 @@ message CallbackState { google.protobuf.Timestamp close_time = 11; // (standalone only) User-supplied business ID set when StartCallbackExecution() is - // called. Used to identify the callback for operations like Describe- or Terminate-. + // called. Used to identify the callback for operations like Describe- or Terminate-. string callback_id = 12; // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() diff --git a/chasm/lib/callback/tasks.go b/chasm/lib/callback/tasks.go index 406b2468045..684810a284c 100644 --- a/chasm/lib/callback/tasks.go +++ b/chasm/lib/callback/tasks.go @@ -181,16 +181,16 @@ func (h *backoffTaskHandler) Validate( return callback.Status == callbackspb.CALLBACK_STATUS_BACKING_OFF && callback.Attempt == task.Attempt, nil } -// CompletionScheduleToCloseTimeoutTaskHandler handles schedule-to-close timeout for standalone callback executions. -type CompletionScheduleToCloseTimeoutTaskHandler struct { +// completionScheduleToCloseTimeoutTaskHandler handles schedule-to-close timeout for standalone callback executions. +type completionScheduleToCloseTimeoutTaskHandler struct { chasm.PureTaskHandlerBase } -func NewCompletionScheduleToCloseTimeoutTaskHandler() *CompletionScheduleToCloseTimeoutTaskHandler { - return &CompletionScheduleToCloseTimeoutTaskHandler{} +func newCompletionScheduleToCloseTimeoutTaskHandler() *completionScheduleToCloseTimeoutTaskHandler { + return &completionScheduleToCloseTimeoutTaskHandler{} } -func (h *CompletionScheduleToCloseTimeoutTaskHandler) Validate( +func (h *completionScheduleToCloseTimeoutTaskHandler) Validate( _ chasm.Context, callback *Callback, _ chasm.TaskAttributes, @@ -199,7 +199,7 @@ func (h *CompletionScheduleToCloseTimeoutTaskHandler) Validate( return TransitionTimedOut.Possible(callback), nil } -func (h *CompletionScheduleToCloseTimeoutTaskHandler) Execute( +func (h *completionScheduleToCloseTimeoutTaskHandler) Execute( ctx chasm.MutableContext, callback *Callback, _ chasm.TaskAttributes, From 684eae84ab856e2d1ffb52657835fb5114bd7cc2 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 9 May 2026 13:31:56 -0700 Subject: [PATCH 10/52] Address PR feedback --- chasm/lib/callback/component.go | 3 ++- chasm/lib/callback/frontend.go | 4 ---- chasm/lib/callback/frontend_validation.go | 16 +++++++------- .../lib/callback/frontend_validation_test.go | 6 ++--- tests/standalone_callbacks_test.go | 22 +++++++++---------- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 02a04e8f18e..ffa0f85b8d2 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -50,6 +50,7 @@ type Callback struct { // Persisted internal state *callbackspb.CallbackState + // Failure from an external termination (timeout or terminate), stored separately because // of its potential size, and to not overload CallbackState::LastAttemptFailure. TerminalFailure chasm.Field[*failurepb.Failure] @@ -245,7 +246,7 @@ func (c *Callback) saveResult( // If the callback was terminated while the invocation was in-flight, // the result is no longer relevant. We'll just drop it silently. // - // This shouldn't happen outside of tests, since the Nexus machinary + // This shouldn't happen outside of tests, since the Nexus machinery // would prevent an invalid transition anyways. (e.g. terminating // an already terminated Callback.) if c.LifecycleState(ctx).IsClosed() { diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go index 40d5aef5fa4..a475f32d065 100644 --- a/chasm/lib/callback/frontend.go +++ b/chasm/lib/callback/frontend.go @@ -19,9 +19,6 @@ import ( var ErrStandaloneCallbacksDisabled = serviceerror.NewUnimplemented("standalone callback executions are not enabled") -// FrontendHandler defines the frontend interface for standalone callback execution RPCs, -// in which the Frontend microservice receives requests from the Temporal SDK and proxies -// them to the implementation of the CHASM component running in the History service. type FrontendHandler interface { StartCallbackExecution(context.Context, *workflowservice.StartCallbackExecutionRequest) (*workflowservice.StartCallbackExecutionResponse, error) DescribeCallbackExecution(context.Context, *workflowservice.DescribeCallbackExecutionRequest) (*workflowservice.DescribeCallbackExecutionResponse, error) @@ -77,7 +74,6 @@ func (h *frontendHandler) getTargetNamespace(requestProto Namespacer) (namespace return namespaceID, nil } -// Checks if standalone callback executions are supported in the target namespace. func (h *frontendHandler) checkFeatureEnabled(requestProto Namespacer) error { // Confirm CHASM is enabled. targetNamespaceName := requestProto.GetNamespace() diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index e9c69489326..9d2d392a0e7 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -17,28 +17,28 @@ import ( // Returns a serviceerror.InvalidArgument error for a missing required field. func missingRequiredFieldError(fieldName string) error { - msg := fmt.Sprintf("%s is not set on request.", fieldName) + msg := fmt.Sprintf("%s is required", fieldName) return serviceerror.NewInvalidArgument(msg) } -type RequestIDer interface { +type requestIder interface { GetRequestId() string } -func verifyRequestIDLength(reqProto RequestIDer, config *Config) error { +func verifyRequestIDLength(reqProto requestIder, config *Config) error { l := len(reqProto.GetRequestId()) maxLen := config.MaxIDLength() if l > maxLen { - return serviceerror.NewInvalidArgumentf("callback ID exceeds length limit. Length=%d Limit=%d", l, maxLen) + return serviceerror.NewInvalidArgumentf("request ID exceeds length limit. Length=%d Limit=%d", l, maxLen) } return nil } -type CallbackIDer interface { +type callbackIder interface { GetCallbackId() string } -func verifyCallbackIDLength(reqProto CallbackIDer, config *Config) error { +func verifyCallbackIDLength(reqProto callbackIder, config *Config) error { l := len(reqProto.GetCallbackId()) maxLen := config.MaxIDLength() if l > maxLen { @@ -73,9 +73,9 @@ func (fields requiredFields) Validate() error { return nil } -// frontendRequestValidator bundles the configuration data for validating an incomming request. +// frontendRequestValidator bundles the configuration data for validating an incoming request. // -// IMPORTANT: Validation methods MAY mutate the incomming request, in order to ensure they all have +// IMPORTANT: Validation methods MAY mutate the incoming request, in order to ensure they all have // a valid RunID (if one was not specified already). type frontendRequestValidator struct { config *Config diff --git a/chasm/lib/callback/frontend_validation_test.go b/chasm/lib/callback/frontend_validation_test.go index d4f9d7fee7c..d8b8fb18973 100644 --- a/chasm/lib/callback/frontend_validation_test.go +++ b/chasm/lib/callback/frontend_validation_test.go @@ -23,9 +23,9 @@ func TestRequiredFields(t *testing.T) { {"Field1", ""}, {"Field2", ""}, } - require.ErrorContains(t, negativeTests.Validate(), "Field1 is not set on request.") + require.ErrorContains(t, negativeTests.Validate(), "Field1 is required") for _, negativeTest := range negativeTests { - wantErr := fmt.Sprintf("%s is not set on request.", negativeTest.FieldName) + wantErr := fmt.Sprintf("%s is required", negativeTest.FieldName) require.ErrorContains(t, negativeTest.Validate(), wantErr) } @@ -38,5 +38,5 @@ func TestRequiredFields(t *testing.T) { {"Mixed Field4", "ok"}, {"Mixed Field5", ""}, } - require.ErrorContains(t, mixedTests.Validate(), "Mixed Field2 is not set on request.") + require.ErrorContains(t, mixedTests.Validate(), "Mixed Field2 is required") } diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index ddbbf6a94be..2928eaed728 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -66,8 +66,8 @@ type externalRequestResult struct { // service will then notify Temporal the work is done (via StartCallbackExecution), and // then put metadata into the `requestResults` channel. type fakeExternalService struct { - incommingRequests chan<- externalRequestInfo - requestResults <-chan externalRequestResult + incomingRequests chan<- externalRequestInfo + requestResults <-chan externalRequestResult } // startFakeExternalService starts a new fake service Goroutine, using the supplied client. @@ -85,27 +85,27 @@ func startFakeExternalService(ctx context.Context, c workflowservice.WorkflowSer for { select { - case incommingRequest := <-input: + case incomingRequest := <-input: // Uniquely identify the callback execution. callbackID := "faux-svc-callback-" + uuid.NewString() targetCallback := &commonpb.Callback{ Variant: &commonpb.Callback_Nexus_{ Nexus: &commonpb.Callback_Nexus{ - Url: incommingRequest.URL, - Token: incommingRequest.Token, + Url: incomingRequest.URL, + Token: incomingRequest.Token, }, }, } resp, err := c.StartCallbackExecution(ctx, &workflowservice.StartCallbackExecutionRequest{ - Namespace: incommingRequest.Namespace, + Namespace: incomingRequest.Namespace, Identity: "faux-external-service", RequestId: uuid.NewString(), CallbackId: callbackID, Callback: targetCallback, Input: &workflowservice.StartCallbackExecutionRequest_Completion{ - Completion: incommingRequest.Result, + Completion: incomingRequest.Result, }, ScheduleToCloseTimeout: durationpb.New(10 * time.Second), }) @@ -123,8 +123,8 @@ func startFakeExternalService(ctx context.Context, c workflowservice.WorkflowSer }() return &fakeExternalService{ - incommingRequests: input, - requestResults: output, + incomingRequests: input, + requestResults: output, } } @@ -270,7 +270,7 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { s.Equal(nexusSvcOp, operation) // Send the request to the external service to do the work. - fakeSvc.incommingRequests <- externalRequestInfo{ + fakeSvc.incomingRequests <- externalRequestInfo{ Namespace: env.Namespace().String(), Token: options.CallbackHeader.Get(commonnexus.CallbackTokenHeader), URL: options.CallbackURL, @@ -674,7 +674,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() mutate: func(req *workflowservice.StartCallbackExecutionRequest) { req.CallbackId = "" }, - errMsg: "CallbackId is not set", + errMsg: "CallbackId is required", }, { name: "missing callback", From e6d586bba54ed8b1e442cf2f601a54719fca2992 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 9 May 2026 13:57:22 -0700 Subject: [PATCH 11/52] Move snippet to loadInvocationArgs --- chasm/lib/callback/component.go | 11 +++++++++++ chasm/lib/callback/invocable_outbound.go | 10 ---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index ffa0f85b8d2..f2710ab2af3 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/nexus-rpc/sdk-go/nexus" callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" failurepb "go.temporal.io/api/failure/v1" @@ -11,6 +12,7 @@ import ( "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/backoff" + commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" queueserrors "go.temporal.io/server/service/history/queues/errors" "google.golang.org/protobuf/proto" @@ -217,6 +219,15 @@ func (c *Callback) loadInvocationArgs( ) } + // Setup the completion's headers. + completion.Header = callback.Header + if callback.GetToken() != "" { + if completion.Header == nil { + completion.Header = nexus.Header{} + } + completion.Header.Set(commonnexus.CallbackTokenHeader, callback.GetToken()) + } + if callback.Url == chasm.NexusCompletionHandlerURL { return invocableInternal{ callback: callback, diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index ccac789e159..971d9f268aa 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -82,16 +82,6 @@ func (n invocableOutbound) Invoke( }) // Make the call and record metrics. startTime := time.Now() - n.completion.Header = n.callback.Header - - // If the outbound call is to a standalone callback, then supply the Nexus - // operation's token in the request. - if n.callback.GetToken() != "" { - if n.completion.Header == nil { - n.completion.Header = nexus.Header{} - } - n.completion.Header.Set(commonnexus.CallbackTokenHeader, n.callback.GetToken()) - } err := client.CompleteOperation(ctx, n.callback.Url, n.completion) From 32bd8e06c819cf746bb8d880d5d1ec6e5f6f3179 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 9 May 2026 14:08:22 -0700 Subject: [PATCH 12/52] Address PR feedback --- chasm/lib/callback/config.go | 4 ++++ chasm/lib/callback/validator.go | 6 +----- chasm/lib/callback/validator_test.go | 12 ++++++++++++ tests/standalone_callbacks_test.go | 4 ++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 63a1ffa58ee..65e58861ca5 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -117,6 +117,10 @@ func (a AddressMatchRules) Validate(rawURL string) error { if rawURL == nexus.SystemCallbackURL || rawURL == chasm.NexusCompletionHandlerURL { return nil } + // To avoid a confusing error, where url.Parse would fail becase of a missing scheme. + if rawURL == "" { + return status.Errorf(codes.InvalidArgument, "invalid callback url: not set") + } u, err := url.Parse(rawURL) if err != nil { return status.Errorf(codes.InvalidArgument, "invalid callback url: %v", err) diff --git a/chasm/lib/callback/validator.go b/chasm/lib/callback/validator.go index d9b872b98ce..8328a8650a3 100644 --- a/chasm/lib/callback/validator.go +++ b/chasm/lib/callback/validator.go @@ -48,16 +48,12 @@ func (v *validator) Validate(_ context.Context, namespaceName string, cbs []*com for _, cb := range cbs { if cb == nil { - return serviceerror.NewInvalidArgument("Callback is not set") + return serviceerror.NewInvalidArgument("invalid callback: not set") } switch variant := cb.GetVariant().(type) { case *commonpb.Callback_Nexus_: rawURL := variant.Nexus.GetUrl() - if rawURL == "" { - return serviceerror.NewInvalidArgument("Callback URL is not set") - } - if len(rawURL) > v.urlMaxLength(namespaceName) { return serviceerror.NewInvalidArgumentf( "invalid url: url length longer than max length allowed of %d", v.urlMaxLength(namespaceName), diff --git a/chasm/lib/callback/validator_test.go b/chasm/lib/callback/validator_test.go index 63d95d87d96..664ea331e30 100644 --- a/chasm/lib/callback/validator_test.go +++ b/chasm/lib/callback/validator_test.go @@ -36,6 +36,18 @@ func TestValidateCallbacks(t *testing.T) { require.NoError(t, err) }) + t.Run("NoURL", func(t *testing.T) { + cbs := []*commonpb.Callback{ + {Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "", + }, + }}, + } + err := v.Validate("ns", cbs) + require.ErrorContains(t, err, "invalid callback url: not set") + }) + t.Run("TooManyCallbacks", func(t *testing.T) { v := NewValidator( func(string) int { return 1 }, diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index 2928eaed728..e62f3edb52f 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -681,7 +681,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() mutate: func(req *workflowservice.StartCallbackExecutionRequest) { req.Callback = nil }, - errMsg: "Callback is not set", + errMsg: "invalid callback: not set", }, { name: "missing callback URL", @@ -692,7 +692,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() }, } }, - errMsg: "Callback URL is not set", + errMsg: "invalid callback url: not set", }, { name: "invalid callback URL scheme", From 1076a5961b9c02abee98b744f8de0f02772bca9e Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 11 May 2026 10:40:52 -0700 Subject: [PATCH 13/52] Wire Identity field to TerminateFailureInfo --- chasm/lib/callback/statemachine.go | 20 +++++++++++++------- chasm/lib/callback/statemachine_test.go | 13 +++++++++++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 5c5f1ee1835..bf1330afb86 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -1,6 +1,7 @@ package callback import ( + "cmp" "fmt" "net/url" "time" @@ -135,7 +136,8 @@ var TransitionSucceeded = chasm.NewTransition( // EventTerminated is triggered when the callback is forcefully terminated. type EventTerminated struct { - Reason string + Identity string + Reason string } var TransitionTerminated = chasm.NewTransition( @@ -149,14 +151,18 @@ var TransitionTerminated = chasm.NewTransition( now := ctx.Now(cb) cb.CloseTime = timestamppb.New(now) - reason := event.Reason - if reason == "" { - reason = "callback execution terminated" + reason := cmp.Or(event.Reason, "callback execution terminated") + var failureInfo *failurepb.TerminatedFailureInfo + if event.Identity != "" { + failureInfo = &failurepb.TerminatedFailureInfo{ + Identity: event.Identity, + } } - failure := &failurepb.Failure{ - Message: reason, - FailureInfo: &failurepb.Failure_TerminatedFailureInfo{}, + Message: reason, + FailureInfo: &failurepb.Failure_TerminatedFailureInfo{ + TerminatedFailureInfo: failureInfo, + }, } cb.TerminalFailure = chasm.NewDataField(ctx, failure) diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index 447a0d0de5f..170c93d5065 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -174,7 +174,10 @@ func TestTerminatedTransition(t *testing.T) { } cb.SetStateMachineState(src) - err := TransitionTerminated.Apply(cb, mctx, EventTerminated{}) + err := TransitionTerminated.Apply(cb, mctx, EventTerminated{ + Identity: "user-supplied identity", + Reason: "user-supplied reason", + }) require.NoError(t, err) // Confirm expected state changes. @@ -187,7 +190,13 @@ func TestTerminatedTransition(t *testing.T) { // Confirm the Callback's terminal failure reason is set. termFailure := cb.TerminalFailure.Get(mctx) - require.Equal(t, "callback execution terminated", termFailure.Message) + require.Equal(t, "user-supplied reason", termFailure.Message) + + // If the Identity is supplied to the request, we should see it + // in the response. + gotTermFailureInfo := termFailure.GetTerminatedFailureInfo() + require.NotNil(t, gotTermFailureInfo) + require.Equal(t, "user-supplied identity", gotTermFailureInfo.Identity) // No new tasks were generated. require.Empty(t, mctx.Tasks) From e1175780d3e4594651c60a1f2950736f9863112d Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 11 May 2026 11:30:20 -0700 Subject: [PATCH 14/52] Refactor to table-driven test --- chasm/lib/callback/statemachine.go | 3 +- chasm/lib/callback/statemachine_test.go | 131 +++++++++++++++--------- 2 files changed, 87 insertions(+), 47 deletions(-) diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index bf1330afb86..ce2e71b3aec 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -101,6 +101,8 @@ var TransitionFailed = chasm.NewTransition( cb.recordAttempt(now) cb.CloseTime = timestamppb.New(now) + // Set the TerminalFailure, but we intentionally leave LastAttemptFailure + // as-is for debugability. failure := &failurepb.Failure{ Message: event.Err.Error(), FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ @@ -109,7 +111,6 @@ var TransitionFailed = chasm.NewTransition( }, }, } - cb.LastAttemptFailure = failure cb.TerminalFailure = chasm.NewDataField(ctx, failure) return nil diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index 170c93d5065..722de19930c 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -121,23 +121,26 @@ func TestValidTransitions(t *testing.T) { mctx := &chasm.MockMutableContext{} mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } - err := TransitionFailed.Apply(callback, mctx, EventFailed{Time: currentTime, Err: errors.New("failed")}) + err := TransitionFailed.Apply(callback, mctx, EventFailed{Time: currentTime, Err: errors.New("new failure msg")}) require.NoError(t, err) // Assert info object is updated. require.Equal(t, callbackspb.CALLBACK_STATUS_FAILED, callback.StateMachineState()) require.Equal(t, int32(2), callback.Attempt) - require.Equal(t, "failed", callback.LastAttemptFailure.Message) - require.True(t, callback.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) require.Nil(t, callback.NextAttemptScheduleTime) require.Equal(t, currentTime, callback.CloseTime.AsTime()) - // Check the TerminalFailure field is set. + // Assert LastAttemptFailure is unchanged. + lastFailure := callback.LastAttemptFailure + require.Equal(t, "error message", lastFailure.Message) + require.False(t, lastFailure.GetApplicationFailureInfo().NonRetryable) + + // Assert the TerminalFailure field is set to the final error. require.NotNil(t, callback.TerminalFailure, "TerminalFailure not set") - got := callback.TerminalFailure.Get(mctx) - want := callback.LastAttemptFailure - require.True(t, proto.Equal(want, got), "TerminalFailure not as expected. Got %v, want %v.", got, want) + termFailure := callback.TerminalFailure.Get(mctx) + require.Equal(t, "new failure msg", termFailure.Message) + require.True(t, termFailure.GetApplicationFailureInfo().NonRetryable) // Assert no tasks generated. In terminal state. require.Empty(t, mctx.Tasks) @@ -159,25 +162,88 @@ func TestTerminatedTransition(t *testing.T) { CallbackState: initialCallbackState, } - for _, src := range []callbackspb.CallbackStatus{ - callbackspb.CALLBACK_STATUS_STANDBY, - callbackspb.CALLBACK_STATUS_SCHEDULED, - callbackspb.CALLBACK_STATUS_BACKING_OFF, - } { - t.Run("from_"+src.String(), func(t *testing.T) { + emptyTerminateEvent := EventTerminated{} + assertEmptyEventResults := func(t *testing.T, mctx *chasm.MockMutableContext, cb *Callback) { + // Confirm default reason, but no additional metadata. + termFailure := cb.TerminalFailure.Get(mctx) + require.Equal(t, "callback execution terminated", termFailure.Message) + require.Nil(t, termFailure.GetTerminatedFailureInfo()) + } + + populatedTerminateEvent := EventTerminated{ + Identity: "user-supplied identity", + Reason: "user-supplied reason", + } + assertPopulatedEventResults := func(t *testing.T, mctx *chasm.MockMutableContext, cb *Callback) { + // Confirm user-supplied reason and identity are available. + termFailure := cb.TerminalFailure.Get(mctx) + require.Equal(t, "user-supplied reason", termFailure.Message) + gotTermFailureInfo := termFailure.GetTerminatedFailureInfo() + require.NotNil(t, gotTermFailureInfo) + require.Equal(t, "user-supplied identity", gotTermFailureInfo.Identity) + } + + tests := []struct { + Name string + FromStatus callbackspb.CallbackStatus + Event EventTerminated + Prepare func(*Callback) + Assert func(*testing.T, *chasm.MockMutableContext, *Callback) + }{ + // Transitions with no Reason/Identity supplied for termination. + { + Name: "in standby", + FromStatus: callbackspb.CALLBACK_STATUS_STANDBY, + Event: emptyTerminateEvent, + Assert: assertEmptyEventResults, + }, + { + Name: "in scheduled, with identity supplied", + FromStatus: callbackspb.CALLBACK_STATUS_SCHEDULED, + Event: emptyTerminateEvent, + Assert: assertEmptyEventResults, + }, + { + Name: "in backing off, with identity supplied", + FromStatus: callbackspb.CALLBACK_STATUS_BACKING_OFF, + Event: emptyTerminateEvent, + Assert: assertEmptyEventResults, + }, + + // With the Reason/Identity supplied. + { + Name: "in standby, with identity supplied", + FromStatus: callbackspb.CALLBACK_STATUS_STANDBY, + Event: populatedTerminateEvent, + Assert: assertPopulatedEventResults, + }, + { + Name: "in scheduled, with identity supplied", + FromStatus: callbackspb.CALLBACK_STATUS_SCHEDULED, + Event: populatedTerminateEvent, + Assert: assertPopulatedEventResults, + }, + { + Name: "in backing off, with identity supplied", + FromStatus: callbackspb.CALLBACK_STATUS_BACKING_OFF, + Event: populatedTerminateEvent, + Assert: assertPopulatedEventResults, + }, + } + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { currentTime := time.Now().UTC() mctx := &chasm.MockMutableContext{} mctx.HandleNow = func(chasm.Component) time.Time { return currentTime } + // Configure cb := &Callback{ CallbackState: proto.Clone(initialCallbackState).(*callbackspb.CallbackState), } - cb.SetStateMachineState(src) + cb.SetStateMachineState(test.FromStatus) - err := TransitionTerminated.Apply(cb, mctx, EventTerminated{ - Identity: "user-supplied identity", - Reason: "user-supplied reason", - }) + // Call + err := TransitionTerminated.Apply(cb, mctx, test.Event) require.NoError(t, err) // Confirm expected state changes. @@ -188,37 +254,10 @@ func TestTerminatedTransition(t *testing.T) { require.True(t, proto.Equal(initialCallbackState.RegistrationTime, cb.RegistrationTime)) require.Equal(t, initialCallback.LastAttemptFailure, cb.LastAttemptFailure) - // Confirm the Callback's terminal failure reason is set. - termFailure := cb.TerminalFailure.Get(mctx) - require.Equal(t, "user-supplied reason", termFailure.Message) - - // If the Identity is supplied to the request, we should see it - // in the response. - gotTermFailureInfo := termFailure.GetTerminatedFailureInfo() - require.NotNil(t, gotTermFailureInfo) - require.Equal(t, "user-supplied identity", gotTermFailureInfo.Identity) - // No new tasks were generated. require.Empty(t, mctx.Tasks) - }) - } - - // Terminal states. Confirm the request should be rejected by CHASM. - for _, src := range []callbackspb.CallbackStatus{ - callbackspb.CALLBACK_STATUS_SUCCEEDED, - callbackspb.CALLBACK_STATUS_FAILED, - callbackspb.CALLBACK_STATUS_TERMINATED, - } { - t.Run("from_"+src.String(), func(t *testing.T) { - mctx := &chasm.MockMutableContext{} - - cb := &Callback{ - CallbackState: proto.Clone(initialCallbackState).(*callbackspb.CallbackState), - } - cb.SetStateMachineState(src) - err := TransitionTerminated.Apply(cb, mctx, EventTerminated{}) - require.ErrorContains(t, err, "invalid transition from") + test.Assert(t, mctx, cb) }) } } From 62ce87265314bd026e05c1356a40aee1d1190d42 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 11 May 2026 13:28:43 -0700 Subject: [PATCH 15/52] Address PR feedback --- chasm/lib/callback/component.go | 156 ++++++++++++++++++- chasm/lib/callback/component_properties.go | 165 --------------------- chasm/lib/callback/config.go | 5 - chasm/lib/callback/frontend.go | 14 +- chasm/lib/callback/frontend_validation.go | 159 ++++++++++++++------ chasm/lib/callback/handler.go | 11 +- tests/standalone_callbacks_test.go | 12 +- 7 files changed, 287 insertions(+), 235 deletions(-) delete mode 100644 chasm/lib/callback/component_properties.go diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index f2710ab2af3..589997ebd5f 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -7,6 +7,7 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" @@ -58,7 +59,7 @@ type Callback struct { TerminalFailure chasm.Field[*failurepb.Failure] // For most callbacks, the completion result is obtained from the parent component. - // e.g. the Workflow result to be delivered. However, for "standalone" callbacks, there + // e.g. the Workflow result to be delivered. However, for standalone callbacks, there // is no parent and the user-supplied SuppliedCompletion will be used instead. ParentCompletionSource chasm.ParentPtr[CompletionSource] SuppliedCompletion chasm.Field[*callbackpb.CallbackExecutionCompletion] @@ -206,7 +207,7 @@ func (c *Callback) loadInvocationArgs( _ chasm.NoValue, ) (invocable, error) { // Get the completion result to be delivered. - completionSource := c.CompletionSource(ctx) + completionSource := c.completionSource(ctx) completion, err := completionSource.GetNexusCompletion(ctx, c.RequestId) if err != nil { return nil, err @@ -326,3 +327,154 @@ func ScheduleStandbyCallbacks(ctx chasm.MutableContext, callbacks chasm.Map[stri } return nil } + +func callbackCompletionToNexusCompleteOperationOpts( + cb *Callback, + completion *callbackpb.CallbackExecutionCompletion) (nexusrpc.CompleteOperationOptions, error) { + + nexusCompletion := nexusrpc.CompleteOperationOptions{ + StartTime: cb.GetRegistrationTime().AsTime(), + CloseTime: cb.CloseTime.AsTime(), + } + + switch completion.Result.(type) { + case *callbackpb.CallbackExecutionCompletion_Success: + nexusCompletion.Result = completion.GetSuccess() + return nexusCompletion, nil + + case *callbackpb.CallbackExecutionCompletion_Failure: + f, err := commonnexus.TemporalFailureToNexusFailure(completion.GetFailure()) + if err != nil { + wrappedErr := fmt.Errorf("failed to convert failure: %w", err) + return nexusrpc.CompleteOperationOptions{}, wrappedErr + } + opErr := &nexus.OperationError{ + State: nexus.OperationStateFailed, + Message: "operation failed", + Cause: &nexus.FailureError{Failure: f}, + } + if err := nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr); err != nil { + wrappedErr := fmt.Errorf("failed to mark wrapper error: %w", err) + return nexusrpc.CompleteOperationOptions{}, wrappedErr + } + nexusCompletion.Error = opErr + return nexusCompletion, nil + + default: + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInvalidArgument("no completion result provided") + } +} + +// completionSource returns the completionSource from the callback, which depends on whether it +// is embedded or is running in standalone mode. +func (c *Callback) completionSource(ctx chasm.Context) CompletionSource { + // Embedded callbacks use their parent component as a CompletionSource. + source, ok := c.ParentCompletionSource.TryGet(ctx) + if ok { + return source + } + + // For standalone completions, get the user-supplied value and convert it + // into the Nexus API type. + suppliedCompletion, ok := c.SuppliedCompletion.TryGet(ctx) + if !ok { + return CompletionSourceFn(func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInternal("no completion available") + }) + } + + convertOutcomeProtoFn := func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { + return callbackCompletionToNexusCompleteOperationOpts(c, suppliedCompletion) + } + return CompletionSourceFn(convertOutcomeProtoFn) +} + +// describe returns the CallbackExecutionInfo for the describe RPC. Only applies to standalone callbacks. +func (c *Callback) describe(ctx chasm.Context) (*callbackpb.CallbackExecutionInfo, error) { + apiCb, err := c.ToAPICallback() + if err != nil { + return nil, err + } + + exInfo := ctx.ExecutionInfo() + info := &callbackpb.CallbackExecutionInfo{ + CallbackId: c.CallbackId, + RunId: ctx.ExecutionKey().RunID, + Callback: apiCb, + Status: callbackStatusToAPIExecutionStatus(c.Status), + State: callbackStatusToAPIState(c.Status), + Attempt: c.Attempt, + CreateTime: c.RegistrationTime, + LastAttemptCompleteTime: c.LastAttemptCompleteTime, + LastAttemptFailure: c.LastAttemptFailure, + NextAttemptScheduleTime: c.NextAttemptScheduleTime, + CloseTime: c.CloseTime, + ScheduleToCloseTimeout: c.CompletionScheduleToCloseTimeout, + StateTransitionCount: exInfo.StateTransitionCount, + } + return info, nil +} + +// outcome returns the callback execution outcome if the execution is in a terminal state. (Otherwise, nil.) +// +// IMPORTANT: This is specific to the callback delivery, and not the actual completion. The outcome will be +// a success even if it was to deliver a failed completion result. +func (c *Callback) outcome(ctx chasm.Context) *callbackpb.CallbackExecutionOutcome { + switch c.Status { + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + val := &callbackpb.CallbackExecutionOutcome_Success{} + return &callbackpb.CallbackExecutionOutcome{ + Value: val, + } + + case callbackspb.CALLBACK_STATUS_FAILED, + callbackspb.CALLBACK_STATUS_TERMINATED: + val := &callbackpb.CallbackExecutionOutcome_Failure{ + Failure: c.TerminalFailure.Get(ctx), + } + return &callbackpb.CallbackExecutionOutcome{ + Value: val, + } + + default: + return nil + } +} + +// callbackStatusToAPIExecutionStatus maps internal CallbackStatus to public API CallbackExecutionStatus. +func callbackStatusToAPIExecutionStatus(status callbackspb.CallbackStatus) enumspb.CallbackExecutionStatus { + switch status { + case callbackspb.CALLBACK_STATUS_STANDBY, + callbackspb.CALLBACK_STATUS_SCHEDULED, + callbackspb.CALLBACK_STATUS_BACKING_OFF: + return enumspb.CALLBACK_EXECUTION_STATUS_RUNNING + case callbackspb.CALLBACK_STATUS_FAILED: + return enumspb.CALLBACK_EXECUTION_STATUS_FAILED + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + return enumspb.CALLBACK_EXECUTION_STATUS_SUCCEEDED + case callbackspb.CALLBACK_STATUS_TERMINATED: + return enumspb.CALLBACK_EXECUTION_STATUS_TERMINATED + default: + return enumspb.CALLBACK_EXECUTION_STATUS_UNSPECIFIED + } +} + +// callbackStatusToAPIState maps internal CallbackStatus to public API CallbackState. +func callbackStatusToAPIState(status callbackspb.CallbackStatus) enumspb.CallbackState { + switch status { + case callbackspb.CALLBACK_STATUS_STANDBY: + return enumspb.CALLBACK_STATE_STANDBY + case callbackspb.CALLBACK_STATUS_SCHEDULED: + return enumspb.CALLBACK_STATE_SCHEDULED + case callbackspb.CALLBACK_STATUS_BACKING_OFF: + return enumspb.CALLBACK_STATE_BACKING_OFF + case callbackspb.CALLBACK_STATUS_FAILED: + return enumspb.CALLBACK_STATE_FAILED + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + return enumspb.CALLBACK_STATE_SUCCEEDED + case callbackspb.CALLBACK_STATUS_TERMINATED: + return enumspb.CALLBACK_STATE_TERMINATED + default: + return enumspb.CALLBACK_STATE_UNSPECIFIED + } +} diff --git a/chasm/lib/callback/component_properties.go b/chasm/lib/callback/component_properties.go deleted file mode 100644 index 497e15f876f..00000000000 --- a/chasm/lib/callback/component_properties.go +++ /dev/null @@ -1,165 +0,0 @@ -package callback - -import ( - "fmt" - - "github.com/nexus-rpc/sdk-go/nexus" - callbackpb "go.temporal.io/api/callback/v1" - enumspb "go.temporal.io/api/enums/v1" - "go.temporal.io/api/serviceerror" - "go.temporal.io/server/chasm" - callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" - commonnexus "go.temporal.io/server/common/nexus" - "go.temporal.io/server/common/nexus/nexusrpc" -) - -func callbackCompletionToNexusCompleteOperationOpts( - cb *Callback, - completion *callbackpb.CallbackExecutionCompletion) (nexusrpc.CompleteOperationOptions, error) { - - nexusCompletion := nexusrpc.CompleteOperationOptions{ - StartTime: cb.GetRegistrationTime().AsTime(), - CloseTime: cb.CloseTime.AsTime(), - } - - switch completion.Result.(type) { - case *callbackpb.CallbackExecutionCompletion_Success: - nexusCompletion.Result = completion.GetSuccess() - return nexusCompletion, nil - - case *callbackpb.CallbackExecutionCompletion_Failure: - f, err := commonnexus.TemporalFailureToNexusFailure(completion.GetFailure()) - if err != nil { - wrappedErr := fmt.Errorf("failed to convert failure: %w", err) - return nexusrpc.CompleteOperationOptions{}, wrappedErr - } - opErr := &nexus.OperationError{ - State: nexus.OperationStateFailed, - Message: "operation failed", - Cause: &nexus.FailureError{Failure: f}, - } - if err := nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr); err != nil { - wrappedErr := fmt.Errorf("failed to mark wrapper error: %w", err) - return nexusrpc.CompleteOperationOptions{}, wrappedErr - } - nexusCompletion.Error = opErr - return nexusCompletion, nil - - default: - return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInvalidArgument("no completion result provided") - } -} - -// CompletionSource returns the CompletionSource from the callback, which depends on whether it -// is embedded or is running in standalone mode. -func (c *Callback) CompletionSource(ctx chasm.Context) CompletionSource { - // Embedded callbacks use their parent component as a CompletionSource. - source, ok := c.ParentCompletionSource.TryGet(ctx) - if ok { - return source - } - - // For standalone completions, get the user-supplied value and convert it - // into the Nexus API type. - suppliedCompletion, ok := c.SuppliedCompletion.TryGet(ctx) - if !ok { - return CompletionSourceFn(func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { - return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInternal("no completion available") - }) - } - - convertOutcomeProtoFn := func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { - return callbackCompletionToNexusCompleteOperationOpts(c, suppliedCompletion) - } - return CompletionSourceFn(convertOutcomeProtoFn) -} - -// callbackStatusToAPIExecutionStatus maps internal CallbackStatus to public API CallbackExecutionStatus. -func callbackStatusToAPIExecutionStatus(status callbackspb.CallbackStatus) enumspb.CallbackExecutionStatus { - switch status { - case callbackspb.CALLBACK_STATUS_STANDBY, - callbackspb.CALLBACK_STATUS_SCHEDULED, - callbackspb.CALLBACK_STATUS_BACKING_OFF: - return enumspb.CALLBACK_EXECUTION_STATUS_RUNNING - case callbackspb.CALLBACK_STATUS_FAILED: - return enumspb.CALLBACK_EXECUTION_STATUS_FAILED - case callbackspb.CALLBACK_STATUS_SUCCEEDED: - return enumspb.CALLBACK_EXECUTION_STATUS_SUCCEEDED - case callbackspb.CALLBACK_STATUS_TERMINATED: - return enumspb.CALLBACK_EXECUTION_STATUS_TERMINATED - default: - return enumspb.CALLBACK_EXECUTION_STATUS_UNSPECIFIED - } -} - -// callbackStatusToAPIState maps internal CallbackStatus to public API CallbackState. -func callbackStatusToAPIState(status callbackspb.CallbackStatus) enumspb.CallbackState { - switch status { - case callbackspb.CALLBACK_STATUS_STANDBY: - return enumspb.CALLBACK_STATE_STANDBY - case callbackspb.CALLBACK_STATUS_SCHEDULED: - return enumspb.CALLBACK_STATE_SCHEDULED - case callbackspb.CALLBACK_STATUS_BACKING_OFF: - return enumspb.CALLBACK_STATE_BACKING_OFF - case callbackspb.CALLBACK_STATUS_FAILED: - return enumspb.CALLBACK_STATE_FAILED - case callbackspb.CALLBACK_STATUS_SUCCEEDED: - return enumspb.CALLBACK_STATE_SUCCEEDED - case callbackspb.CALLBACK_STATUS_TERMINATED: - return enumspb.CALLBACK_STATE_TERMINATED - default: - return enumspb.CALLBACK_STATE_UNSPECIFIED - } -} - -// Describe returns the CallbackExecutionInfo for the describe RPC. Only applies to standalone callbacks. -func (c *Callback) Describe(ctx chasm.Context) (*callbackpb.CallbackExecutionInfo, error) { - apiCb, err := c.ToAPICallback() - if err != nil { - return nil, err - } - - exInfo := ctx.ExecutionInfo() - info := &callbackpb.CallbackExecutionInfo{ - CallbackId: c.CallbackId, - RunId: ctx.ExecutionKey().RunID, - Callback: apiCb, - Status: callbackStatusToAPIExecutionStatus(c.Status), - State: callbackStatusToAPIState(c.Status), - Attempt: c.Attempt, - CreateTime: c.RegistrationTime, - LastAttemptCompleteTime: c.LastAttemptCompleteTime, - LastAttemptFailure: c.LastAttemptFailure, - NextAttemptScheduleTime: c.NextAttemptScheduleTime, - CloseTime: c.CloseTime, - ScheduleToCloseTimeout: c.CompletionScheduleToCloseTimeout, - StateTransitionCount: exInfo.StateTransitionCount, - } - return info, nil -} - -// Outcome returns the callback execution outcome if the execution is in a terminal state. (Otherwise, nil.) -// -// IMPORTANT: This is specific to the callback delivery, and not the actual completion. The outcome will be -// a success even if it was to deliver a failed completion result. -func (c *Callback) Outcome(ctx chasm.Context) *callbackpb.CallbackExecutionOutcome { - switch c.Status { - case callbackspb.CALLBACK_STATUS_SUCCEEDED: - val := &callbackpb.CallbackExecutionOutcome_Success{} - return &callbackpb.CallbackExecutionOutcome{ - Value: val, - } - - case callbackspb.CALLBACK_STATUS_FAILED, - callbackspb.CALLBACK_STATUS_TERMINATED: - val := &callbackpb.CallbackExecutionOutcome_Failure{ - Failure: c.TerminalFailure.Get(ctx), - } - return &callbackpb.CallbackExecutionOutcome{ - Value: val, - } - - default: - return nil - } -} diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 65e58861ca5..52986e0f292 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -76,11 +76,6 @@ type Config struct { // NOTE: The configuration setting defining the allowlist of supported callback // addresses is defined in components/callbacks/config.go, via AllowedAddresses. - // - // Similarly, MaxPerExecution is missing. It is used by `Validator` and is loaded there. - // Once HSM callbacks (components/callbacks) are removed, the callbackValidatorProvider in - // frontend/fx.go can be moved into this package. And at that time, we can simply have the - // callback.Validator inject callback.Config. (And have a single location for all config.) } func ConfigProvider(dc *dynamicconfig.Collection) *Config { diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go index a475f32d065..58028cfdccd 100644 --- a/chasm/lib/callback/frontend.go +++ b/chasm/lib/callback/frontend.go @@ -96,7 +96,7 @@ func (h *frontendHandler) StartCallbackExecution( if err := h.checkFeatureEnabled(request); err != nil { return nil, err } - if err := h.reqValidator.ValidateStartCallbackExecution(request); err != nil { + if err := h.reqValidator.ValidateAndNormalizeStartCallbackExecution(request); err != nil { return nil, err } @@ -126,15 +126,17 @@ func (h *frontendHandler) DescribeCallbackExecution( if err := h.checkFeatureEnabled(request); err != nil { return nil, err } - if err := h.reqValidator.ValidateDescribeCallbackExecution(request); err != nil { - return nil, err - } - // Execute + // Get the Namespace ID to confirm the optional long-poll token matches. namespaceID, err := h.getTargetNamespace(request) if err != nil { return nil, err } + if err := h.reqValidator.ValidateDescribeCallbackExecution(request, namespaceID); err != nil { + return nil, err + } + + // Execute resp, err := h.client.DescribeCallbackExecution(ctx, &callbackspb.DescribeCallbackExecutionRequest{ NamespaceId: namespaceID.String(), FrontendRequest: request, @@ -183,7 +185,7 @@ func (h *frontendHandler) TerminateCallbackExecution( if err := h.checkFeatureEnabled(request); err != nil { return nil, err } - if err := h.reqValidator.ValidateTerminateCallbackExecution(request); err != nil { + if err := h.reqValidator.ValidateAndNormalizeTerminateCallbackExecution(request); err != nil { return nil, err } diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index 9d2d392a0e7..41775021355 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -8,9 +8,11 @@ import ( commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" "go.temporal.io/server/common" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/searchattribute" "google.golang.org/protobuf/proto" ) @@ -21,6 +23,32 @@ func missingRequiredFieldError(fieldName string) error { return serviceerror.NewInvalidArgument(msg) } +type callbackIder interface { + GetCallbackId() string +} + +func verifyCallbackIDLength(reqProto callbackIder, config *Config) error { + l := len(reqProto.GetCallbackId()) + maxLen := config.MaxIDLength() + if l > maxLen { + return serviceerror.NewInvalidArgumentf("callback_id exceeds length limit. Length=%d Limit=%d", l, maxLen) + } + return nil +} + +type identityer interface { + GetIdentity() string +} + +func verifyIdentityLength(reqProto identityer, config *Config) error { + l := len(reqProto.GetIdentity()) + maxLen := config.MaxIDLength() + if l > maxLen { + return serviceerror.NewInvalidArgumentf("identity exceeds length limit. Length=%d Limit=%d", l, maxLen) + } + return nil +} + type requestIder interface { GetRequestId() string } @@ -29,20 +57,18 @@ func verifyRequestIDLength(reqProto requestIder, config *Config) error { l := len(reqProto.GetRequestId()) maxLen := config.MaxIDLength() if l > maxLen { - return serviceerror.NewInvalidArgumentf("request ID exceeds length limit. Length=%d Limit=%d", l, maxLen) + return serviceerror.NewInvalidArgumentf("request_id exceeds length limit. Length=%d Limit=%d", l, maxLen) } return nil } -type callbackIder interface { - GetCallbackId() string +type runIder interface { + GetRunId() string } -func verifyCallbackIDLength(reqProto callbackIder, config *Config) error { - l := len(reqProto.GetCallbackId()) - maxLen := config.MaxIDLength() - if l > maxLen { - return serviceerror.NewInvalidArgumentf("callback ID exceeds length limit. Length=%d Limit=%d", l, maxLen) +func verifyRunIDIsUUID(reqProto runIder) error { + if err := uuid.Validate(reqProto.GetRunId()); err != nil { + return serviceerror.NewInvalidArgument("invalid run_id: must be a valid UUID") } return nil } @@ -75,8 +101,8 @@ func (fields requiredFields) Validate() error { // frontendRequestValidator bundles the configuration data for validating an incoming request. // -// IMPORTANT: Validation methods MAY mutate the incoming request, in order to ensure they all have -// a valid RunID (if one was not specified already). +// In addition some operations may *mutate* the request object too, e.g. to ensure a valid RunID +// is avaliable if one wasn't supplied by the caller. type frontendRequestValidator struct { config *Config cbValidator Validator @@ -85,7 +111,7 @@ type frontendRequestValidator struct { saValidator *searchattribute.Validator } -func (rv *frontendRequestValidator) ValidateStartCallbackExecution(req *workflowservice.StartCallbackExecutionRequest) error { +func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution(req *workflowservice.StartCallbackExecutionRequest) error { // Set RequestID if missing. if req.GetRequestId() == "" { req.RequestId = uuid.NewString() @@ -93,20 +119,22 @@ func (rv *frontendRequestValidator) ValidateStartCallbackExecution(req *workflow // Check required fields each have a value. requiredFields := requiredFields{ - {"Namespace", req.GetNamespace()}, - {"Identity", req.GetIdentity()}, - {"RequestId", req.GetRequestId()}, - {"CallbackId", req.GetCallbackId()}, + {"namespace", req.GetNamespace()}, + {"identity", req.GetIdentity()}, + {"callback_id", req.GetCallbackId()}, } if err := requiredFields.Validate(); err != nil { return err } // Field lengths - if err := verifyRequestIDLength(req, rv.config); err != nil { + if err := verifyCallbackIDLength(req, rv.config); err != nil { return err } - if err := verifyCallbackIDLength(req, rv.config); err != nil { + if err := verifyIdentityLength(req, rv.config); err != nil { + return err + } + if err := verifyRequestIDLength(req, rv.config); err != nil { return err } @@ -116,22 +144,21 @@ func (rv *frontendRequestValidator) ValidateStartCallbackExecution(req *workflow } // ScheduleToCloseTimeout - if req.GetScheduleToCloseTimeout() == nil || req.GetScheduleToCloseTimeout().AsDuration() <= 0 { - return serviceerror.NewInvalidArgument("ScheduleToCloseTimeout must be set and positive.") + if req.GetScheduleToCloseTimeout() != nil && req.GetScheduleToCloseTimeout().AsDuration() <= 0 { + return serviceerror.NewInvalidArgument("schedule_to_close_timeout must be positive.") } // Validate the input data to deliver to the callback URL, currently only one kind is supported (Completion). completion := req.GetCompletion() if completion == nil { - return serviceerror.NewInvalidArgument("Completion is not set on request.") + return serviceerror.NewInvalidArgument("completion is not set on request.") } if completion.GetSuccess() == nil && completion.GetFailure() == nil { - return serviceerror.NewInvalidArgument("Completion must have either success or failure set.") + return serviceerror.NewInvalidArgument("completion must have either success or failure set.") } if completion.GetSuccess() != nil && completion.GetFailure() != nil { - return serviceerror.NewInvalidArgument("Completion must have exactly one of success or failure set, not both.") + return serviceerror.NewInvalidArgument("completion must have exactly one of success or failure set, not both.") } - // Validate the size of the completion is reasonable. if err := rv.validateCompletionSize(req, completion); err != nil { return err } @@ -147,15 +174,15 @@ func (rv *frontendRequestValidator) ValidateStartCallbackExecution(req *workflow } func (rv *frontendRequestValidator) validateCompletionSize(req Namespacer, completion *callbackpb.CallbackExecutionCompletion) error { - namespace := req.GetNamespace() + ns := req.GetNamespace() - sizeWarnLimit := rv.config.BlobSizeLimitWarn(namespace) - sizeErrorLimit := rv.config.BlobSizeLimitError(namespace) + sizeWarnLimit := rv.config.BlobSizeLimitWarn(ns) + sizeErrorLimit := rv.config.BlobSizeLimitError(ns) blobSize := proto.Size(completion) if blobSize > sizeWarnLimit { - rv.logger.Warn("Completion blob size exceeds the warning limit.", - tag.WorkflowNamespace(namespace), + rv.logger.Warn("completion blob size exceeds the warning limit.", + tag.WorkflowNamespace(ns), tag.BlobSize(int64(blobSize))) } @@ -185,24 +212,41 @@ func (rv *frontendRequestValidator) validateSearchAttributes(req Namespacer, saT return rv.saValidator.ValidateSize(saToValidate, namespaceName) } -func (rv *frontendRequestValidator) ValidateDescribeCallbackExecution(req *workflowservice.DescribeCallbackExecutionRequest) error { +func (rv *frontendRequestValidator) ValidateDescribeCallbackExecution(req *workflowservice.DescribeCallbackExecutionRequest, targetNamespaceID namespace.ID) error { // Check required fields each have a value. requiredFields := requiredFields{ - {"Namespace", req.GetNamespace()}, - {"CallbackId", req.GetCallbackId()}, + {"namespace", req.GetNamespace()}, + {"callback_id", req.GetCallbackId()}, } if err := requiredFields.Validate(); err != nil { return err } + // RunID (if set) + if req.GetRunId() != "" { + if err := verifyRunIDIsUUID(req); err != nil { + return err + } + } + // Field lengths if err := verifyCallbackIDLength(req, rv.config); err != nil { return err } - // A long-poll token requires the RunID be set. - if len(req.GetLongPollToken()) > 0 && req.GetRunId() == "" { - return serviceerror.NewInvalidArgument("RunID is required when LongPollToken is provided") + // Long-poll Token + if len(req.GetLongPollToken()) > 0 { + if req.GetRunId() == "" { + return serviceerror.NewInvalidArgument("run_id is required when long_poll_token is provided") + } + + ref, err := chasm.DeserializeComponentRef(req.GetLongPollToken()) + if err != nil { + return serviceerror.NewInvalidArgument("invalid long poll token") + } + if ref.NamespaceID != targetNamespaceID.String() { + return serviceerror.NewInvalidArgument("long poll token does not match execution") + } } return nil @@ -211,18 +255,25 @@ func (rv *frontendRequestValidator) ValidateDescribeCallbackExecution(req *workf func (rv *frontendRequestValidator) ValidatePollCallbackExecution(req *workflowservice.PollCallbackExecutionRequest) error { // Check required fields each have a value. requiredFields := requiredFields{ - {"Namespace", req.GetNamespace()}, - {"CallbackId", req.GetCallbackId()}, + {"namespace", req.GetNamespace()}, + {"callback_id", req.GetCallbackId()}, } if err := requiredFields.Validate(); err != nil { return err } + // RunID (if set) + if req.GetRunId() != "" { + if err := verifyRunIDIsUUID(req); err != nil { + return err + } + } + // Field lengths return verifyCallbackIDLength(req, rv.config) } -func (rv *frontendRequestValidator) ValidateTerminateCallbackExecution(req *workflowservice.TerminateCallbackExecutionRequest) error { +func (rv *frontendRequestValidator) ValidateAndNormalizeTerminateCallbackExecution(req *workflowservice.TerminateCallbackExecutionRequest) error { // Set RequestID if missing. if req.GetRequestId() == "" { req.RequestId = uuid.NewString() @@ -232,26 +283,44 @@ func (rv *frontendRequestValidator) ValidateTerminateCallbackExecution(req *work // NOTE: We don't require the Identity or Reason fields to be set, // and just set reasonable defaults. requiredFields := requiredFields{ - {"RequestId", req.GetRequestId()}, - {"Namespace", req.GetNamespace()}, - {"CallbackId", req.GetCallbackId()}, + {"namespace", req.GetNamespace()}, + {"callback_id", req.GetCallbackId()}, } if err := requiredFields.Validate(); err != nil { return err } + // RunID (if set) + if req.GetRunId() != "" { + if err := verifyRunIDIsUUID(req); err != nil { + return err + } + } + // Field lengths + if err := verifyCallbackIDLength(req, rv.config); err != nil { + return err + } + if err := verifyIdentityLength(req, rv.config); err != nil { + return err + } if err := verifyRequestIDLength(req, rv.config); err != nil { return err } - return verifyCallbackIDLength(req, rv.config) + + // Capping reason to "MaxIDLength", despite it not really being an ID. + reasonLen := len(req.GetReason()) + if reasonLen > rv.config.MaxIDLength() { + return serviceerror.NewInvalidArgumentf("reason exceeds length limit. Length=%d Limit=%d", reasonLen, rv.config.MaxIDLength()) + } + return nil } func (rv *frontendRequestValidator) ValidateDeleteCallbackExecution(req *workflowservice.DeleteCallbackExecutionRequest) error { // Check required fields each have a value. requiredFields := requiredFields{ - {"Namespace", req.GetNamespace()}, - {"CallbackId", req.GetCallbackId()}, + {"namespace", req.GetNamespace()}, + {"callback_id", req.GetCallbackId()}, } if err := requiredFields.Validate(); err != nil { return err @@ -263,14 +332,14 @@ func (rv *frontendRequestValidator) ValidateDeleteCallbackExecution(req *workflo func (rv *frontendRequestValidator) ValidateListCallbackExecutions(req *workflowservice.ListCallbackExecutionsRequest) error { if req.GetNamespace() == "" { - return missingRequiredFieldError("Namespace") + return missingRequiredFieldError("namespace") } return nil } func (rv *frontendRequestValidator) ValidateCountCallbackExecutions(req *workflowservice.CountCallbackExecutionsRequest) error { if req.GetNamespace() == "" { - return missingRequiredFieldError("Namespace") + return missingRequiredFieldError("namespace") } return nil } diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go index b70824a19df..5ec058a2259 100644 --- a/chasm/lib/callback/handler.go +++ b/chasm/lib/callback/handler.go @@ -59,8 +59,7 @@ func (h *callbackHandler) StartCallbackExecution( } } - // Create the CHASM Callback in so-called "standalone" mode, where it will be the root - // of the CHASM execution. + // Create the Callback in standalone mode, where it will be the root of the CHASM execution. result, err := chasm.StartExecution( ctx, chasm.ExecutionKey{ @@ -110,7 +109,7 @@ func (h *callbackHandler) DescribeCallbackExecution( ctx chasm.Context, c *Callback, ) (*callbackspb.DescribeCallbackExecutionResponse, error) { - info, err := c.Describe(ctx) + info, err := c.describe(ctx) if err != nil { return nil, err } @@ -122,7 +121,7 @@ func (h *callbackHandler) DescribeCallbackExecution( resp.Input = c.SuppliedCompletion.Get(ctx) } if req.FrontendRequest.GetIncludeOutcome() { - resp.Outcome = c.Outcome(ctx) + resp.Outcome = c.outcome(ctx) } return &callbackspb.DescribeCallbackExecutionResponse{ @@ -230,7 +229,7 @@ func (h *callbackHandler) PollCallbackExecution( return &callbackspb.PollCallbackExecutionResponse{ FrontendResponse: &workflowservice.PollCallbackExecutionResponse{ RunId: ctx.ExecutionKey().RunID, - Outcome: c.Outcome(ctx), + Outcome: c.outcome(ctx), }, }, true, nil }, req) @@ -310,7 +309,7 @@ type createStandaloneCallbackInput struct { SearchAttributes map[string]*commonpb.Payload } -// createStandaloneCallback constructs a new Callback component in "standalone" mode. +// createStandaloneCallback constructs a new Callback component in standalone mode. // The Callback is immediately transitioned to SCHEDULED state to begin invocation. func createStandaloneCallback( ctx chasm.MutableContext, diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index e62f3edb52f..5756f6028c0 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -674,7 +674,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() mutate: func(req *workflowservice.StartCallbackExecutionRequest) { req.CallbackId = "" }, - errMsg: "CallbackId is required", + errMsg: "callback_id is required", }, { name: "missing callback", @@ -721,21 +721,21 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() mutate: func(req *workflowservice.StartCallbackExecutionRequest) { req.Input = nil }, - errMsg: "Completion is not set", + errMsg: "completion is not set", }, { name: "empty completion", mutate: func(req *workflowservice.StartCallbackExecutionRequest) { req.Input = &workflowservice.StartCallbackExecutionRequest_Completion{Completion: &callbackpb.CallbackExecutionCompletion{}} }, - errMsg: "Completion must have either success or failure set", + errMsg: "completion must have either success or failure set", }, { - name: "missing schedule_to_close_timeout", + name: "negative schedule_to_close_timeout", mutate: func(req *workflowservice.StartCallbackExecutionRequest) { - req.ScheduleToCloseTimeout = nil + req.ScheduleToCloseTimeout = durationpb.New(-time.Second) }, - errMsg: "ScheduleToCloseTimeout must be set", + errMsg: "schedule_to_close_timeout must be positive", }, } From b800cddcc06db47611512b852dce4a97cb68b63b Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 11 May 2026 14:42:45 -0700 Subject: [PATCH 16/52] PR feedback from /review bot --- chasm/lib/callback/config.go | 29 ------- chasm/lib/callback/frontend.go | 8 +- chasm/lib/callback/frontend_validation.go | 98 +++++++---------------- chasm/lib/callback/handler.go | 9 +-- service/frontend/configs/quotas.go | 2 +- tests/standalone_callbacks_test.go | 27 +++---- 6 files changed, 50 insertions(+), 123 deletions(-) diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 52986e0f292..10c094ce238 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -163,35 +163,6 @@ func (a AddressMatchRule) Allow(u *url.URL) (bool, error) { return true, nil } -func allowedAddressConverter(val any) (AddressMatchRules, error) { - type entry struct { - Pattern string - AllowInsecure bool - } - intermediate, err := dynamicconfig.ConvertStructure[[]entry](nil)(val) - if err != nil { - return AddressMatchRules{}, err - } - - configs := []AddressMatchRule{} - for _, e := range intermediate { - if e.Pattern == "" { - // Skip configs with missing / unparsable Pattern - continue - } - re, err := regexp.Compile(addressPatternToRegexp(e.Pattern)) - if err != nil { - // Skip configs with malformed Pattern - continue - } - configs = append(configs, AddressMatchRule{ - Regexp: re, - AllowInsecure: e.AllowInsecure, - }) - } - return AddressMatchRules{Rules: configs}, nil -} - func addressPatternToRegexp(pattern string) string { var result strings.Builder result.WriteString("^") diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go index 58028cfdccd..67f88d6c213 100644 --- a/chasm/lib/callback/frontend.go +++ b/chasm/lib/callback/frontend.go @@ -62,10 +62,10 @@ func NewFrontendHandler( } } -type Namespacer interface{ GetNamespace() string } +type namespacer interface{ GetNamespace() string } // Looks up the namespace ID from the user-supplied namespace name in the request proto. -func (h *frontendHandler) getTargetNamespace(requestProto Namespacer) (namespace.ID, error) { +func (h *frontendHandler) getTargetNamespace(requestProto namespacer) (namespace.ID, error) { targetNamespaceName := namespace.Name(requestProto.GetNamespace()) namespaceID, err := h.namespaceRegistry.GetNamespaceID(targetNamespaceName) if err != nil { @@ -74,7 +74,7 @@ func (h *frontendHandler) getTargetNamespace(requestProto Namespacer) (namespace return namespaceID, nil } -func (h *frontendHandler) checkFeatureEnabled(requestProto Namespacer) error { +func (h *frontendHandler) checkFeatureEnabled(requestProto namespacer) error { // Confirm CHASM is enabled. targetNamespaceName := requestProto.GetNamespace() if !h.config.CHASMEnabled(targetNamespaceName) || !h.config.CHASMCallbacksEnabled(targetNamespaceName) { @@ -176,7 +176,7 @@ func (h *frontendHandler) PollCallbackExecution( } // TerminateCallbackExecution forcefully stops a running callback execution. -// No-op if already in a terminal state. +// Fails if already in a terminate state and on a different Request ID. func (h *frontendHandler) TerminateCallbackExecution( ctx context.Context, request *workflowservice.TerminateCallbackExecutionRequest, diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index 41775021355..73ad3f82919 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -23,52 +23,16 @@ func missingRequiredFieldError(fieldName string) error { return serviceerror.NewInvalidArgument(msg) } -type callbackIder interface { - GetCallbackId() string -} - -func verifyCallbackIDLength(reqProto callbackIder, config *Config) error { - l := len(reqProto.GetCallbackId()) - maxLen := config.MaxIDLength() - if l > maxLen { - return serviceerror.NewInvalidArgumentf("callback_id exceeds length limit. Length=%d Limit=%d", l, maxLen) - } - return nil -} - -type identityer interface { - GetIdentity() string -} - -func verifyIdentityLength(reqProto identityer, config *Config) error { - l := len(reqProto.GetIdentity()) - maxLen := config.MaxIDLength() - if l > maxLen { - return serviceerror.NewInvalidArgumentf("identity exceeds length limit. Length=%d Limit=%d", l, maxLen) +func verifyFieldLength(fieldName, fieldValue string, maxLen int) error { + if l := len(fieldValue); l > maxLen { + return serviceerror.NewInvalidArgumentf("%s exceeds length limit. Length=%d Limit=%d", fieldName, l, maxLen) } return nil } -type requestIder interface { - GetRequestId() string -} - -func verifyRequestIDLength(reqProto requestIder, config *Config) error { - l := len(reqProto.GetRequestId()) - maxLen := config.MaxIDLength() - if l > maxLen { - return serviceerror.NewInvalidArgumentf("request_id exceeds length limit. Length=%d Limit=%d", l, maxLen) - } - return nil -} - -type runIder interface { - GetRunId() string -} - -func verifyRunIDIsUUID(reqProto runIder) error { - if err := uuid.Validate(reqProto.GetRunId()); err != nil { - return serviceerror.NewInvalidArgument("invalid run_id: must be a valid UUID") +func verifyIsUUID(fieldName, fieldValue string) error { + if err := uuid.Validate(fieldValue); err != nil { + return serviceerror.NewInvalidArgumentf("invalid %s: must be a valid UUID", fieldName) } return nil } @@ -102,7 +66,7 @@ func (fields requiredFields) Validate() error { // frontendRequestValidator bundles the configuration data for validating an incoming request. // // In addition some operations may *mutate* the request object too, e.g. to ensure a valid RunID -// is avaliable if one wasn't supplied by the caller. +// is available if one wasn't supplied by the caller. type frontendRequestValidator struct { config *Config cbValidator Validator @@ -128,13 +92,14 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution(r } // Field lengths - if err := verifyCallbackIDLength(req, rv.config); err != nil { + maxLen := rv.config.MaxIDLength() + if err := verifyFieldLength("callback_id", req.GetCallbackId(), maxLen); err != nil { return err } - if err := verifyIdentityLength(req, rv.config); err != nil { + if err := verifyFieldLength("identity", req.GetIdentity(), maxLen); err != nil { return err } - if err := verifyRequestIDLength(req, rv.config); err != nil { + if err := verifyFieldLength("request_id", req.GetRequestId(), maxLen); err != nil { return err } @@ -145,19 +110,19 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution(r // ScheduleToCloseTimeout if req.GetScheduleToCloseTimeout() != nil && req.GetScheduleToCloseTimeout().AsDuration() <= 0 { - return serviceerror.NewInvalidArgument("schedule_to_close_timeout must be positive.") + return serviceerror.NewInvalidArgument("schedule_to_close_timeout must be positive") } // Validate the input data to deliver to the callback URL, currently only one kind is supported (Completion). completion := req.GetCompletion() if completion == nil { - return serviceerror.NewInvalidArgument("completion is not set on request.") + return serviceerror.NewInvalidArgument("completion is not set on request") } if completion.GetSuccess() == nil && completion.GetFailure() == nil { - return serviceerror.NewInvalidArgument("completion must have either success or failure set.") + return serviceerror.NewInvalidArgument("completion must have either success or failure set") } if completion.GetSuccess() != nil && completion.GetFailure() != nil { - return serviceerror.NewInvalidArgument("completion must have exactly one of success or failure set, not both.") + return serviceerror.NewInvalidArgument("completion must have exactly one of success or failure set, not both") } if err := rv.validateCompletionSize(req, completion); err != nil { return err @@ -173,7 +138,7 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution(r return nil } -func (rv *frontendRequestValidator) validateCompletionSize(req Namespacer, completion *callbackpb.CallbackExecutionCompletion) error { +func (rv *frontendRequestValidator) validateCompletionSize(req namespacer, completion *callbackpb.CallbackExecutionCompletion) error { ns := req.GetNamespace() sizeWarnLimit := rv.config.BlobSizeLimitWarn(ns) @@ -181,7 +146,7 @@ func (rv *frontendRequestValidator) validateCompletionSize(req Namespacer, compl blobSize := proto.Size(completion) if blobSize > sizeWarnLimit { - rv.logger.Warn("completion blob size exceeds the warning limit.", + rv.logger.Warn("completion blob size exceeds the warning limit", tag.WorkflowNamespace(ns), tag.BlobSize(int64(blobSize))) } @@ -193,7 +158,7 @@ func (rv *frontendRequestValidator) validateCompletionSize(req Namespacer, compl return nil } -func (rv *frontendRequestValidator) validateSearchAttributes(req Namespacer, saToValidate *commonpb.SearchAttributes) error { +func (rv *frontendRequestValidator) validateSearchAttributes(req namespacer, saToValidate *commonpb.SearchAttributes) error { namespaceName := req.GetNamespace() // Unalias search attributes for validation. @@ -224,13 +189,13 @@ func (rv *frontendRequestValidator) ValidateDescribeCallbackExecution(req *workf // RunID (if set) if req.GetRunId() != "" { - if err := verifyRunIDIsUUID(req); err != nil { + if err := verifyIsUUID("run_id", req.GetRunId()); err != nil { return err } } // Field lengths - if err := verifyCallbackIDLength(req, rv.config); err != nil { + if err := verifyFieldLength("callback_id", req.GetCallbackId(), rv.config.MaxIDLength()); err != nil { return err } @@ -264,13 +229,13 @@ func (rv *frontendRequestValidator) ValidatePollCallbackExecution(req *workflows // RunID (if set) if req.GetRunId() != "" { - if err := verifyRunIDIsUUID(req); err != nil { + if err := verifyIsUUID("run_id", req.GetRunId()); err != nil { return err } } // Field lengths - return verifyCallbackIDLength(req, rv.config) + return verifyFieldLength("callback_id", req.GetCallbackId(), rv.config.MaxIDLength()) } func (rv *frontendRequestValidator) ValidateAndNormalizeTerminateCallbackExecution(req *workflowservice.TerminateCallbackExecutionRequest) error { @@ -292,28 +257,25 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeTerminateCallbackExecuti // RunID (if set) if req.GetRunId() != "" { - if err := verifyRunIDIsUUID(req); err != nil { + if err := verifyIsUUID("run_id", req.GetRunId()); err != nil { return err } } - // Field lengths - if err := verifyCallbackIDLength(req, rv.config); err != nil { + // Field lengths. + maxLen := rv.config.MaxIDLength() + if err := verifyFieldLength("callback_id", req.GetCallbackId(), maxLen); err != nil { return err } - if err := verifyIdentityLength(req, rv.config); err != nil { + if err := verifyFieldLength("identity", req.GetIdentity(), maxLen); err != nil { return err } - if err := verifyRequestIDLength(req, rv.config); err != nil { + if err := verifyFieldLength("request_id", req.GetRequestId(), maxLen); err != nil { return err } // Capping reason to "MaxIDLength", despite it not really being an ID. - reasonLen := len(req.GetReason()) - if reasonLen > rv.config.MaxIDLength() { - return serviceerror.NewInvalidArgumentf("reason exceeds length limit. Length=%d Limit=%d", reasonLen, rv.config.MaxIDLength()) - } - return nil + return verifyFieldLength("reason", req.GetReason(), maxLen) } func (rv *frontendRequestValidator) ValidateDeleteCallbackExecution(req *workflowservice.DeleteCallbackExecutionRequest) error { @@ -327,7 +289,7 @@ func (rv *frontendRequestValidator) ValidateDeleteCallbackExecution(req *workflo } // Field lengths - return verifyCallbackIDLength(req, rv.config) + return verifyFieldLength("callback_id", req.GetCallbackId(), rv.config.MaxIDLength()) } func (rv *frontendRequestValidator) ValidateListCallbackExecutions(req *workflowservice.ListCallbackExecutionsRequest) error { diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go index 5ec058a2259..702df8258d9 100644 --- a/chasm/lib/callback/handler.go +++ b/chasm/lib/callback/handler.go @@ -12,7 +12,6 @@ import ( "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/contextutil" - "go.temporal.io/server/common/log" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -21,13 +20,11 @@ type callbackHandler struct { callbackspb.UnimplementedCallbackServiceServer config *Config - logger log.Logger } -func newCallbackHandler(config *Config, logger log.Logger) *callbackHandler { +func newCallbackHandler(config *Config) *callbackHandler { return &callbackHandler{ config: config, - logger: logger, } } @@ -168,7 +165,7 @@ func (h *callbackHandler) DescribeCallbackExecution( longpollReadFn := func( c *Callback, ctx chasm.Context, - req *callbackspb.DescribeCallbackExecutionRequest) (*callbackspb.DescribeCallbackExecutionResponse, bool, error) { + _ *callbackspb.DescribeCallbackExecutionRequest) (*callbackspb.DescribeCallbackExecutionResponse, bool, error) { changed, err := chasm.ExecutionStateChanged(c, ctx, token) if err != nil { if errors.Is(err, chasm.ErrMalformedComponentRef) { @@ -234,7 +231,7 @@ func (h *callbackHandler) PollCallbackExecution( }, true, nil }, req) - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + if err != nil && ctx.Err() != nil { // Send an empty non-error response as an invitation to resubmit the long-poll. return &callbackspb.PollCallbackExecutionResponse{ FrontendResponse: &workflowservice.PollCallbackExecutionResponse{}, diff --git a/service/frontend/configs/quotas.go b/service/frontend/configs/quotas.go index 555bb94fb16..33279617549 100644 --- a/service/frontend/configs/quotas.go +++ b/service/frontend/configs/quotas.go @@ -47,7 +47,7 @@ var ( "/temporal.api.workflowservice.v1.WorkflowService/PollNexusTaskQueue": 1, "/temporal.api.workflowservice.v1.WorkflowService/PollNexusOperationExecution": 1, - // Long-running if the outcome is the resources isn't in a terminal state. + // Long-running if the resources isn't in a terminal state. "/temporal.api.workflowservice.v1.WorkflowService/PollActivityExecution": 1, "/temporal.api.workflowservice.v1.WorkflowService/PollCallbackExecution": 1, diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index 5756f6028c0..a8156c8f5d7 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -212,7 +212,7 @@ func (s *StandaloneCallbackSuite) callStartCallbackExecution( return resp } -// TestBasicaOperation tests that a Nexus operation started by a workflow can be completed using the +// TestBasicOperation tests that a Nexus operation started by a workflow can be completed using the // StartCallbackExecution API to deliver the Nexus completion to the operation's callback URL. // // Flow: @@ -225,7 +225,7 @@ func (s *StandaloneCallbackSuite) callStartCallbackExecution( // 5. The caller workflow receives the operation's result and completes. // // The test is ran in two variants. First, where the out-of-band service reports a successful -// compoetion result. The second reports a failure. Causing the calling workflow to fail. +// completion result. The second reports a failure, causing the calling workflow to fail. func (s *StandaloneCallbackSuite) TestBasicOperation() { // Implementation of the test scenario, standing up the workflow, Nexus operation, @@ -254,10 +254,6 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // and sends the callback URL and token to the fake service. The Nexus // handler terminates, while the fake service reports results out-of-band. nexusEndpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) - const ( - nexusSvcName = "nexus-service" - nexusSvcOp = "nexus-operation" - ) nexusHandler := nexustest.Handler{ OnStartOperation: func( @@ -266,9 +262,6 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { input *nexus.LazyValue, options nexus.StartOperationOptions, ) (nexus.HandlerStartOperationResult[any], error) { - s.Equal(nexusSvcName, service) - s.Equal(nexusSvcOp, operation) - // Send the request to the external service to do the work. fakeSvc.incomingRequests <- externalRequestInfo{ Namespace: env.Namespace().String(), @@ -311,8 +304,8 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // workflow will then block until the Nexus operation completes, which will // not be until the fake external service has reported the callback's result. callerWf := func(ctx workflow.Context) (string, error) { - c := workflow.NewNexusClient(nexusEndpointName, nexusSvcName) - fut := c.ExecuteOperation(ctx, nexusSvcOp, "input", workflow.NexusOperationOptions{}) + c := workflow.NewNexusClient(nexusEndpointName, "nexus-service") + fut := c.ExecuteOperation(ctx, "nexus-op", "input", workflow.NexusOperationOptions{}) var nexusOpResult string err := fut.Get(ctx, &nexusOpResult) @@ -449,9 +442,9 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { // of a callback execution. It returns an empty response when the poll times out, and // the CallbackExecutionOutcome when the callback reaches a terminal state. func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { - env := s.newEnv() s.Run("returns_empty_for_non_terminal", func(s *StandaloneCallbackSuite) { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -492,6 +485,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { }) s.Run("blocks_until_complete", func(s *StandaloneCallbackSuite) { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -517,6 +511,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { case <-resultCh: s.Fail("expected poll to block, but it returned before terminate") case <-time.After(500 * time.Millisecond): + // Expected to fire, since resultCh is still waiting for Poll- to complete. } // Terminate the CallbackExecution. Confirm that the poll result (from Goroutine) has completed. @@ -536,6 +531,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { }) s.Run("returns_run_id", func(s *StandaloneCallbackSuite) { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -565,6 +561,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { }) s.Run("poll_after_timeout", func(s *StandaloneCallbackSuite) { + env := s.newEnv() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -756,7 +753,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() tc.mutate(req) _, err := env.FrontendClient().StartCallbackExecution(ctx, req) s.Error(err) - s.Contains(err.Error(), tc.errMsg) + s.ErrorContains(err, tc.errMsg) }) } } @@ -808,7 +805,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { req.RequestId = uuid.NewString() _, err = env.FrontendClient().StartCallbackExecution(ctx, req) s.Error(err) - s.Contains(err.Error(), "callback execution already started") + s.ErrorContains(err, "callback execution already started") } // TestListAndCountCallbackExecutions tests that standalone callback executions @@ -1056,7 +1053,7 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { Reason: "different request", }) s.Error(err) - s.Contains(err.Error(), "already terminated with request ID") + s.ErrorContains(err, "already terminated with request ID") } // TestCallbackExecutionFailedOutcome tests that when a callback fails with a non-retryable error From 4c23e41f52717fa217b211b3303ba543db55c3ce Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 14:32:26 -0700 Subject: [PATCH 17/52] Bring in latest API bits --- chasm/lib/callback/handler.go | 1 - go.mod | 3 ++- go.sum | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go index 702df8258d9..016fc41b491 100644 --- a/chasm/lib/callback/handler.go +++ b/chasm/lib/callback/handler.go @@ -81,7 +81,6 @@ func (h *callbackHandler) StartCallbackExecution( "callback execution already started", alreadyStartedErr.CurrentRequestID, alreadyStartedErr.CurrentRunID, - frontendReq.GetCallbackId(), ) return nil, svcErr } diff --git a/go.mod b/go.mod index 6e6dbdca09f..b2613a9bc7b 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 - go.temporal.io/api v1.62.12-0.20260506203937-27ab43932052 // DO NOT SUBMIT -- Branch chrsmith/standalone-callbacks + go.temporal.io/api v1.62.12-0.20260513212731-4fa5ab4b3909 // DO NOT SUBMIT -- Branch chrsmith/standalone-callbacks go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 go.temporal.io/sdk v1.41.1 go.uber.org/fx v1.24.0 @@ -99,6 +99,7 @@ require ( github.com/go-openapi/swag/typeutils v0.26.0 // indirect github.com/go-openapi/swag/yamlutils v0.26.0 // indirect github.com/hashicorp/go-version v1.9.0 // indirect + github.com/nexus-rpc/nexus-proto-annotations v0.1.0 // indirect go.opentelemetry.io/collector/featuregate v1.56.0 // indirect ) diff --git a/go.sum b/go.sum index 9d452e6683c..e5f8d388383 100644 --- a/go.sum +++ b/go.sum @@ -319,6 +319,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/nexus-rpc/nexus-proto-annotations v0.1.0 h1:2fELd+9sqUtNu6Fg//pw8YFsxOvp8vZ8hfP0nHhNI80= +github.com/nexus-rpc/nexus-proto-annotations v0.1.0/go.mod h1:n3UjF1bPCW8llR8tHvbxJ+27yPWrhpo8w/Yg1IOuY0Y= github.com/nexus-rpc/sdk-go v0.6.0 h1:QRgnP2zTbxEbiyWG/aXH8uSC5LV/Mg1fqb19jb4DBlo= github.com/nexus-rpc/sdk-go v0.6.0/go.mod h1:FHdPfVQwRuJFZFTF0Y2GOAxCrbIBNrcPna9slkGKPYk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -469,8 +471,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.62.12-0.20260506203937-27ab43932052 h1:zTB6uMLzdBsLksMH073JTuLfcS0S53+Bm5Kxestwnz4= -go.temporal.io/api v1.62.12-0.20260506203937-27ab43932052/go.mod h1:iaxoP/9OXMJcQkETTECfwYq4cw/bj4nwov8b3ZLVnXM= +go.temporal.io/api v1.62.12-0.20260513212731-4fa5ab4b3909 h1:1sHlmcA+G9aeneEr20SiG9bKpbWejwoKNfpNioiVx+g= +go.temporal.io/api v1.62.12-0.20260513212731-4fa5ab4b3909/go.mod h1:iqSd4FzEdRg8o0TIkhKIc5wIafoP/iix8q+zl5yN8oo= go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 h1:1hKeH3GyR6YD6LKMHGCZ76t6h1Sgha0hXVQBxWi3dlQ= go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2/go.mod h1:T8dnzVPeO+gaUTj9eDgm/lT2lZH4+JXNvrGaQGyVi50= go.temporal.io/sdk v1.41.1 h1:yOpvsHyDD1lNuwlGBv/SUodCPhjv9nDeC9lLHW/fJUA= From 0ecbc51c1726006403bb983b875cdc6e4782bbef Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 14:43:13 -0700 Subject: [PATCH 18/52] Rename task to ScheduleToClose timeout --- chasm/lib/callback/component.go | 12 ++++---- chasm/lib/callback/frontend.go | 7 ----- chasm/lib/callback/fx.go | 2 +- .../callback/gen/callbackpb/v1/message.pb.go | 18 ++++++------ .../gen/callbackpb/v1/tasks.go-helpers.pb.go | 20 ++++++------- .../callback/gen/callbackpb/v1/tasks.pb.go | 28 +++++++++---------- chasm/lib/callback/handler.go | 10 +++---- chasm/lib/callback/library.go | 20 ++++++------- chasm/lib/callback/proto/v1/message.proto | 2 +- chasm/lib/callback/proto/v1/tasks.proto | 4 +-- chasm/lib/callback/tasks.go | 16 +++++------ 11 files changed, 66 insertions(+), 73 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 589997ebd5f..67fa531f108 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -92,10 +92,10 @@ type newStandaloneCallbackOpts struct { RegistrationTime *timestamppb.Timestamp Callback *callbackspb.Callback - CallbackID string - CompletionScheduleToCloseTimeout *durationpb.Duration - Completion *callbackpb.CallbackExecutionCompletion - SearchAttributes map[string]*commonpb.Payload + CallbackID string + ScheduleToCloseTimeout *durationpb.Duration + Completion *callbackpb.CallbackExecutionCompletion + SearchAttributes map[string]*commonpb.Payload } // newStandaloneCallback returns a new Callback component which will deliver the supplied @@ -108,7 +108,7 @@ func newStandaloneCallback( // Add standalone-specific fields. cb.CallbackId = opts.CallbackID - cb.CompletionScheduleToCloseTimeout = opts.CompletionScheduleToCloseTimeout + cb.ScheduleToCloseTimeout = opts.ScheduleToCloseTimeout cb.SuppliedCompletion = chasm.NewDataField(ctx, opts.Completion) visibility := chasm.NewVisibilityWithData(ctx, opts.SearchAttributes, nil) @@ -409,7 +409,7 @@ func (c *Callback) describe(ctx chasm.Context) (*callbackpb.CallbackExecutionInf LastAttemptFailure: c.LastAttemptFailure, NextAttemptScheduleTime: c.NextAttemptScheduleTime, CloseTime: c.CloseTime, - ScheduleToCloseTimeout: c.CompletionScheduleToCloseTimeout, + ScheduleToCloseTimeout: c.ScheduleToCloseTimeout, StateTransitionCount: exInfo.StateTransitionCount, } return info, nil diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go index 67f88d6c213..242c9237593 100644 --- a/chasm/lib/callback/frontend.go +++ b/chasm/lib/callback/frontend.go @@ -248,10 +248,6 @@ func (h *frontendHandler) ListCallbackExecutions( // Lookup the namespace by its name, to confirm it actually exists. namespaceName := namespace.Name(request.GetNamespace()) - if _, err := h.namespaceRegistry.GetNamespaceID(namespaceName); err != nil { - return nil, err - } - resp, err := chasm.ListExecutions[*Callback, *callbackpb.CallbackExecutionListInfo]( ctx, &chasm.ListExecutionsRequest{ @@ -313,9 +309,6 @@ func (h *frontendHandler) CountCallbackExecutions( // Lookup the namespace by its name, to confirm it actually exists. namespaceName := namespace.Name(request.GetNamespace()) - if _, err := h.namespaceRegistry.GetNamespaceID(namespaceName); err != nil { - return nil, err - } resp, err := chasm.CountExecutions[*Callback]( ctx, &chasm.CountExecutionsRequest{ diff --git a/chasm/lib/callback/fx.go b/chasm/lib/callback/fx.go index a781ad25899..94410bfe83d 100644 --- a/chasm/lib/callback/fx.go +++ b/chasm/lib/callback/fx.go @@ -67,7 +67,7 @@ var HistoryModule = fx.Module( fx.Provide(newInvocationTaskHandler), fx.Provide(newBackoffTaskHandler), fx.Provide(newCallbackHandler), - fx.Provide(newCompletionScheduleToCloseTimeoutTaskHandler), + fx.Provide(newScheduleToCloseTimeoutTaskHandler), fx.Provide(newLibrary), fx.Invoke(func(registry *chasm.Registry, library *library) error { diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index 71e7e9e2411..02b4a3fc3a8 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -143,9 +143,9 @@ type CallbackState struct { CallbackId string `protobuf:"bytes,12,opt,name=callback_id,json=callbackId,proto3" json:"callback_id,omitempty"` // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() // is called to when the result gets delivered. - CompletionScheduleToCloseTimeout *durationpb.Duration `protobuf:"bytes,13,opt,name=completion_schedule_to_close_timeout,json=completionScheduleToCloseTimeout,proto3" json:"completion_schedule_to_close_timeout,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + ScheduleToCloseTimeout *durationpb.Duration `protobuf:"bytes,13,opt,name=schedule_to_close_timeout,json=scheduleToCloseTimeout,proto3" json:"schedule_to_close_timeout,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CallbackState) Reset() { @@ -255,9 +255,9 @@ func (x *CallbackState) GetCallbackId() string { return "" } -func (x *CallbackState) GetCompletionScheduleToCloseTimeout() *durationpb.Duration { +func (x *CallbackState) GetScheduleToCloseTimeout() *durationpb.Duration { if x != nil { - return x.CompletionScheduleToCloseTimeout + return x.ScheduleToCloseTimeout } return nil } @@ -443,7 +443,7 @@ var File_temporal_server_chasm_lib_callback_proto_v1_message_proto protoreflect. const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = "" + "\n" + - "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\xcc\x06\n" + + "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\xb7\x06\n" + "\rCallbackState\x12R\n" + "\bcallback\x18\x01 \x01(\v26.temporal.server.chasm.lib.callbacks.proto.v1.CallbackR\bcallback\x12G\n" + "\x11registration_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x10registrationTime\x12T\n" + @@ -459,8 +459,8 @@ const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = " "\n" + "close_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcloseTime\x12\x1f\n" + "\vcallback_id\x18\f \x01(\tR\n" + - "callbackId\x12i\n" + - "$completion_schedule_to_close_timeout\x18\r \x01(\v2\x19.google.protobuf.DurationR completionScheduleToCloseTimeout\x1a\x10\n" + + "callbackId\x12T\n" + + "\x19schedule_to_close_timeout\x18\r \x01(\v2\x19.google.protobuf.DurationR\x16scheduleToCloseTimeout\x1a\x10\n" + "\x0eWorkflowClosed\"\xf4\x02\n" + "\bCallback\x12T\n" + "\x05nexus\x18\x02 \x01(\v2<.temporal.server.chasm.lib.callbacks.proto.v1.Callback.NexusH\x00R\x05nexus\x122\n" + @@ -516,7 +516,7 @@ var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_depIdxs = []i 7, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure 6, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp 6, // 6: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.close_time:type_name -> google.protobuf.Timestamp - 8, // 7: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.completion_schedule_to_close_timeout:type_name -> google.protobuf.Duration + 8, // 7: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.schedule_to_close_timeout:type_name -> google.protobuf.Duration 4, // 8: temporal.server.chasm.lib.callbacks.proto.v1.Callback.nexus:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus 9, // 9: temporal.server.chasm.lib.callbacks.proto.v1.Callback.links:type_name -> temporal.api.common.v1.Link 5, // 10: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.header:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry diff --git a/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go b/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go index 7c0fcca665b..0ea4db28c6c 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/tasks.go-helpers.pb.go @@ -79,35 +79,35 @@ func (this *BackoffTask) Equal(that interface{}) bool { return proto.Equal(this, that1) } -// Marshal an object of type CompletionScheduleToCloseTimeoutTask to the protobuf v3 wire format -func (val *CompletionScheduleToCloseTimeoutTask) Marshal() ([]byte, error) { +// Marshal an object of type ScheduleToCloseTimeoutTask to the protobuf v3 wire format +func (val *ScheduleToCloseTimeoutTask) Marshal() ([]byte, error) { return proto.Marshal(val) } -// Unmarshal an object of type CompletionScheduleToCloseTimeoutTask from the protobuf v3 wire format -func (val *CompletionScheduleToCloseTimeoutTask) Unmarshal(buf []byte) error { +// Unmarshal an object of type ScheduleToCloseTimeoutTask from the protobuf v3 wire format +func (val *ScheduleToCloseTimeoutTask) Unmarshal(buf []byte) error { return proto.Unmarshal(buf, val) } // Size returns the size of the object, in bytes, once serialized -func (val *CompletionScheduleToCloseTimeoutTask) Size() int { +func (val *ScheduleToCloseTimeoutTask) Size() int { return proto.Size(val) } -// Equal returns whether two CompletionScheduleToCloseTimeoutTask values are equivalent by recursively +// Equal returns whether two ScheduleToCloseTimeoutTask values are equivalent by recursively // comparing the message's fields. // For more information see the documentation for // https://pkg.go.dev/google.golang.org/protobuf/proto#Equal -func (this *CompletionScheduleToCloseTimeoutTask) Equal(that interface{}) bool { +func (this *ScheduleToCloseTimeoutTask) Equal(that interface{}) bool { if that == nil { return this == nil } - var that1 *CompletionScheduleToCloseTimeoutTask + var that1 *ScheduleToCloseTimeoutTask switch t := that.(type) { - case *CompletionScheduleToCloseTimeoutTask: + case *ScheduleToCloseTimeoutTask: that1 = t - case CompletionScheduleToCloseTimeoutTask: + case ScheduleToCloseTimeoutTask: that1 = &t default: return false diff --git a/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go b/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go index 33e29da618d..22e76c46975 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/tasks.pb.go @@ -112,27 +112,27 @@ func (x *BackoffTask) GetAttempt() int32 { return 0 } -// Fired when the callback completion's schedule-to-close timeout expires. -type CompletionScheduleToCloseTimeoutTask struct { +// Fired when the callback's schedule-to-close timeout expires. +type ScheduleToCloseTimeoutTask struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *CompletionScheduleToCloseTimeoutTask) Reset() { - *x = CompletionScheduleToCloseTimeoutTask{} +func (x *ScheduleToCloseTimeoutTask) Reset() { + *x = ScheduleToCloseTimeoutTask{} mi := &file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *CompletionScheduleToCloseTimeoutTask) String() string { +func (x *ScheduleToCloseTimeoutTask) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CompletionScheduleToCloseTimeoutTask) ProtoMessage() {} +func (*ScheduleToCloseTimeoutTask) ProtoMessage() {} -func (x *CompletionScheduleToCloseTimeoutTask) ProtoReflect() protoreflect.Message { +func (x *ScheduleToCloseTimeoutTask) ProtoReflect() protoreflect.Message { mi := &file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -144,8 +144,8 @@ func (x *CompletionScheduleToCloseTimeoutTask) ProtoReflect() protoreflect.Messa return mi.MessageOf(x) } -// Deprecated: Use CompletionScheduleToCloseTimeoutTask.ProtoReflect.Descriptor instead. -func (*CompletionScheduleToCloseTimeoutTask) Descriptor() ([]byte, []int) { +// Deprecated: Use ScheduleToCloseTimeoutTask.ProtoReflect.Descriptor instead. +func (*ScheduleToCloseTimeoutTask) Descriptor() ([]byte, []int) { return file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDescGZIP(), []int{2} } @@ -157,8 +157,8 @@ const file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDesc = "" "\x0eInvocationTask\x12\x18\n" + "\aattempt\x18\x01 \x01(\x05R\aattempt\"'\n" + "\vBackoffTask\x12\x18\n" + - "\aattempt\x18\x01 \x01(\x05R\aattempt\"&\n" + - "$CompletionScheduleToCloseTimeoutTaskBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" + "\aattempt\x18\x01 \x01(\x05R\aattempt\"\x1c\n" + + "\x1aScheduleToCloseTimeoutTaskBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" var ( file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDescOnce sync.Once @@ -174,9 +174,9 @@ func file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_rawDescGZIP() var file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_goTypes = []any{ - (*InvocationTask)(nil), // 0: temporal.server.chasm.lib.callbacks.proto.v1.InvocationTask - (*BackoffTask)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.BackoffTask - (*CompletionScheduleToCloseTimeoutTask)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.CompletionScheduleToCloseTimeoutTask + (*InvocationTask)(nil), // 0: temporal.server.chasm.lib.callbacks.proto.v1.InvocationTask + (*BackoffTask)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.BackoffTask + (*ScheduleToCloseTimeoutTask)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.ScheduleToCloseTimeoutTask } var file_temporal_server_chasm_lib_callback_proto_v1_tasks_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go index 016fc41b491..af996a1541a 100644 --- a/chasm/lib/callback/handler.go +++ b/chasm/lib/callback/handler.go @@ -319,10 +319,10 @@ func createStandaloneCallback( RegistrationTime: now, Callback: input.Callback, - CallbackID: input.CallbackID, - CompletionScheduleToCloseTimeout: input.CompletionScheduleToCloseTimeout, - Completion: input.Completion, - SearchAttributes: input.SearchAttributes, + CallbackID: input.CallbackID, + ScheduleToCloseTimeout: input.CompletionScheduleToCloseTimeout, + Completion: input.Completion, + SearchAttributes: input.SearchAttributes, } cb := newStandaloneCallback(ctx, opts) @@ -338,7 +338,7 @@ func createStandaloneCallback( ctx.AddTask( cb, chasm.TaskAttributes{ScheduledTime: timeoutTime}, - &callbackspb.CompletionScheduleToCloseTimeoutTask{}, + &callbackspb.ScheduleToCloseTimeoutTask{}, ) } } diff --git a/chasm/lib/callback/library.go b/chasm/lib/callback/library.go index 1c0e3545ea8..ac822e3dc22 100644 --- a/chasm/lib/callback/library.go +++ b/chasm/lib/callback/library.go @@ -33,23 +33,23 @@ func (l *componentOnlyLibrary) Components() []*chasm.RegistrableComponent { type library struct { componentOnlyLibrary - InvocationTaskHandler *invocationTaskHandler - BackoffTaskHandler *backoffTaskHandler - CompletionScheduleToCloseTimeoutTaskHandler *completionScheduleToCloseTimeoutTaskHandler - callbackSvcHandler *callbackHandler + InvocationTaskHandler *invocationTaskHandler + BackoffTaskHandler *backoffTaskHandler + ScheduleToCloseTimeoutTaskHandler *scheduleToCloseTimeoutTaskHandler + callbackSvcHandler *callbackHandler } func newLibrary( InvocationTaskHandler *invocationTaskHandler, BackoffTaskHandler *backoffTaskHandler, - CompletionScheduleToCloseTimeoutTaskHandler *completionScheduleToCloseTimeoutTaskHandler, + ScheduleToCloseTimeoutTaskHandler *scheduleToCloseTimeoutTaskHandler, callbackSvcHandler *callbackHandler, ) *library { return &library{ - InvocationTaskHandler: InvocationTaskHandler, - BackoffTaskHandler: BackoffTaskHandler, - CompletionScheduleToCloseTimeoutTaskHandler: CompletionScheduleToCloseTimeoutTaskHandler, - callbackSvcHandler: callbackSvcHandler, + InvocationTaskHandler: InvocationTaskHandler, + BackoffTaskHandler: BackoffTaskHandler, + ScheduleToCloseTimeoutTaskHandler: ScheduleToCloseTimeoutTaskHandler, + callbackSvcHandler: callbackSvcHandler, } } @@ -65,7 +65,7 @@ func (l *library) Tasks() []*chasm.RegistrableTask { ), chasm.NewRegistrablePureTask( "completionScheduleToCloseTimer", - l.CompletionScheduleToCloseTimeoutTaskHandler, + l.ScheduleToCloseTimeoutTaskHandler, ), } } diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index 15433f31460..e7b7f29b41c 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -46,7 +46,7 @@ message CallbackState { // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() // is called to when the result gets delivered. - google.protobuf.Duration completion_schedule_to_close_timeout = 13; + google.protobuf.Duration schedule_to_close_timeout = 13; } // Status of a callback. diff --git a/chasm/lib/callback/proto/v1/tasks.proto b/chasm/lib/callback/proto/v1/tasks.proto index dde93a730f9..253f62940c8 100644 --- a/chasm/lib/callback/proto/v1/tasks.proto +++ b/chasm/lib/callback/proto/v1/tasks.proto @@ -14,5 +14,5 @@ message BackoffTask { int32 attempt = 1; } -// Fired when the callback completion's schedule-to-close timeout expires. -message CompletionScheduleToCloseTimeoutTask {} +// Fired when the callback's schedule-to-close timeout expires. +message ScheduleToCloseTimeoutTask {} diff --git a/chasm/lib/callback/tasks.go b/chasm/lib/callback/tasks.go index 684810a284c..d59737c6860 100644 --- a/chasm/lib/callback/tasks.go +++ b/chasm/lib/callback/tasks.go @@ -181,29 +181,29 @@ func (h *backoffTaskHandler) Validate( return callback.Status == callbackspb.CALLBACK_STATUS_BACKING_OFF && callback.Attempt == task.Attempt, nil } -// completionScheduleToCloseTimeoutTaskHandler handles schedule-to-close timeout for standalone callback executions. -type completionScheduleToCloseTimeoutTaskHandler struct { +// scheduleToCloseTimeoutTaskHandler handles schedule-to-close timeouts for standalone callback executions. +type scheduleToCloseTimeoutTaskHandler struct { chasm.PureTaskHandlerBase } -func newCompletionScheduleToCloseTimeoutTaskHandler() *completionScheduleToCloseTimeoutTaskHandler { - return &completionScheduleToCloseTimeoutTaskHandler{} +func newScheduleToCloseTimeoutTaskHandler() *scheduleToCloseTimeoutTaskHandler { + return &scheduleToCloseTimeoutTaskHandler{} } -func (h *completionScheduleToCloseTimeoutTaskHandler) Validate( +func (h *scheduleToCloseTimeoutTaskHandler) Validate( _ chasm.Context, callback *Callback, _ chasm.TaskAttributes, - _ *callbackspb.CompletionScheduleToCloseTimeoutTask, + _ *callbackspb.ScheduleToCloseTimeoutTask, ) (bool, error) { return TransitionTimedOut.Possible(callback), nil } -func (h *completionScheduleToCloseTimeoutTaskHandler) Execute( +func (h *scheduleToCloseTimeoutTaskHandler) Execute( ctx chasm.MutableContext, callback *Callback, _ chasm.TaskAttributes, - _ *callbackspb.CompletionScheduleToCloseTimeoutTask, + _ *callbackspb.ScheduleToCloseTimeoutTask, ) error { return TransitionTimedOut.Apply(callback, ctx, EventTimedOut{}) } From 26539f14a1f4cb52f21f8d576c47a4a598fe36c7 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 15:04:39 -0700 Subject: [PATCH 19/52] Add MaxScheduledToCloseTimeout --- chasm/lib/callback/config.go | 27 +++++++++++++++-------- chasm/lib/callback/frontend_validation.go | 14 ++++++++++-- chasm/lib/callback/invocable_internal.go | 3 +++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 10c094ce238..39e54eecfd1 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -45,6 +45,13 @@ var RequestTimeout = dynamicconfig.NewDestinationDurationSetting( `RequestTimeout is the timeout for executing a single callback request.`, ) +var MaxCallbackScheduleToCloseTimeout = dynamicconfig.NewNamespaceDurationSetting( + "callback.limit.scheduleToCloseTimeout", + 0, + `Maximum allowed duration of a callback execution. Commands that specify no schedule-to-close timeout +or a longer timeout than permitted will have their schedule-to-close timeout capped to this value. 0 implies no limit.`, +) + var RetryPolicyInitialInterval = dynamicconfig.NewGlobalDurationSetting( "callback.retryPolicy.initialInterval", time.Second, @@ -59,11 +66,12 @@ var RetryPolicyMaximumInterval = dynamicconfig.NewGlobalDurationSetting( type Config struct { // callback.* settings. - EnableStandaloneExecutions dynamicconfig.BoolPropertyFnWithNamespaceFilter - LongPollBuffer dynamicconfig.DurationPropertyFnWithNamespaceFilter - LongPollTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter - RequestTimeout dynamicconfig.DurationPropertyFnWithDestinationFilter - RetryPolicy func() backoff.RetryPolicy + EnableStandaloneExecutions dynamicconfig.BoolPropertyFnWithNamespaceFilter + LongPollBuffer dynamicconfig.DurationPropertyFnWithNamespaceFilter + LongPollTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter + MaxCallbackScheduleToCloseTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter + RequestTimeout dynamicconfig.DurationPropertyFnWithDestinationFilter + RetryPolicy func() backoff.RetryPolicy // Settings defined elsewhere. CHASMEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -80,10 +88,11 @@ type Config struct { func ConfigProvider(dc *dynamicconfig.Collection) *Config { return &Config{ - EnableStandaloneExecutions: EnableStandaloneExecutions.Get(dc), - LongPollBuffer: LongPollBuffer.Get(dc), - LongPollTimeout: LongPollTimeout.Get(dc), - RequestTimeout: RequestTimeout.Get(dc), + EnableStandaloneExecutions: EnableStandaloneExecutions.Get(dc), + LongPollBuffer: LongPollBuffer.Get(dc), + LongPollTimeout: LongPollTimeout.Get(dc), + MaxCallbackScheduleToCloseTimeout: MaxCallbackScheduleToCloseTimeout.Get(dc), + RequestTimeout: RequestTimeout.Get(dc), RetryPolicy: func() backoff.RetryPolicy { return backoff.NewExponentialRetryPolicy( RetryPolicyInitialInterval.Get(dc)(), diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index 73ad3f82919..371a37c169a 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -15,6 +15,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/searchattribute" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" ) // Returns a serviceerror.InvalidArgument error for a missing required field. @@ -109,8 +110,17 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution(r } // ScheduleToCloseTimeout - if req.GetScheduleToCloseTimeout() != nil && req.GetScheduleToCloseTimeout().AsDuration() <= 0 { - return serviceerror.NewInvalidArgument("schedule_to_close_timeout must be positive") + if schedToCloseTimeout := req.GetScheduleToCloseTimeout(); schedToCloseTimeout != nil { + if schedToCloseTimeout.AsDuration() <= 0 { + return serviceerror.NewInvalidArgument("schedule_to_close_timeout must be positive") + } + + // Clamp the ScheduleToCloseTimeout to the maximum allowed if set. + maxAllowed := rv.config.MaxCallbackScheduleToCloseTimeout(req.Namespace) + if maxAllowed > 0 { + clamped := min(schedToCloseTimeout.AsDuration(), maxAllowed) + req.ScheduleToCloseTimeout = durationpb.New(clamped) + } } // Validate the input data to deliver to the callback URL, currently only one kind is supported (Completion). diff --git a/chasm/lib/callback/invocable_internal.go b/chasm/lib/callback/invocable_internal.go index 75f465236ae..bfc1350d05b 100644 --- a/chasm/lib/callback/invocable_internal.go +++ b/chasm/lib/callback/invocable_internal.go @@ -58,6 +58,9 @@ func (c invocableInternal) Invoke( task *callbackspb.InvocationTask, taskAttr chasm.TaskAttributes, ) invocationResult { + // TODO(chrsmith): Both here and in invocable_outbound.go. + // > we should validate that temporal:// URLs have a token either via the old header format or the new structured Token field. + // Get the token from the dedicated Token field, falling back to the header for backwards compat. encodedRef := c.callback.GetToken() if encodedRef == "" { From 9157af1b987256dab19b4e76d7b8b9472fe58c28 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 15:07:09 -0700 Subject: [PATCH 20/52] Remove dead code --- .../callback/gen/callbackpb/v1/message.pb.go | 87 +++++-------------- chasm/lib/callback/proto/v1/message.proto | 3 - 2 files changed, 24 insertions(+), 66 deletions(-) diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index 02b4a3fc3a8..b116c832cbc 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -336,43 +336,6 @@ type Callback_Nexus_ struct { func (*Callback_Nexus_) isCallback_Variant() {} -// Trigger for when the workflow is closed. -type CallbackState_WorkflowClosed struct { - state protoimpl.MessageState `protogen:"open.v1"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *CallbackState_WorkflowClosed) Reset() { - *x = CallbackState_WorkflowClosed{} - mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *CallbackState_WorkflowClosed) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*CallbackState_WorkflowClosed) ProtoMessage() {} - -func (x *CallbackState_WorkflowClosed) ProtoReflect() protoreflect.Message { - mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[2] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use CallbackState_WorkflowClosed.ProtoReflect.Descriptor instead. -func (*CallbackState_WorkflowClosed) Descriptor() ([]byte, []int) { - return file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDescGZIP(), []int{0, 0} -} - type Callback_Nexus struct { state protoimpl.MessageState `protogen:"open.v1"` // Callback URL. @@ -390,7 +353,7 @@ type Callback_Nexus struct { func (x *Callback_Nexus) Reset() { *x = Callback_Nexus{} - mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[3] + mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -402,7 +365,7 @@ func (x *Callback_Nexus) String() string { func (*Callback_Nexus) ProtoMessage() {} func (x *Callback_Nexus) ProtoReflect() protoreflect.Message { - mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[3] + mi := &file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -443,7 +406,7 @@ var File_temporal_server_chasm_lib_callback_proto_v1_message_proto protoreflect. const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = "" + "\n" + - "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\xb7\x06\n" + + "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\xa5\x06\n" + "\rCallbackState\x12R\n" + "\bcallback\x18\x01 \x01(\v26.temporal.server.chasm.lib.callbacks.proto.v1.CallbackR\bcallback\x12G\n" + "\x11registration_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x10registrationTime\x12T\n" + @@ -460,8 +423,7 @@ const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = " "close_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcloseTime\x12\x1f\n" + "\vcallback_id\x18\f \x01(\tR\n" + "callbackId\x12T\n" + - "\x19schedule_to_close_timeout\x18\r \x01(\v2\x19.google.protobuf.DurationR\x16scheduleToCloseTimeout\x1a\x10\n" + - "\x0eWorkflowClosed\"\xf4\x02\n" + + "\x19schedule_to_close_timeout\x18\r \x01(\v2\x19.google.protobuf.DurationR\x16scheduleToCloseTimeout\"\xf4\x02\n" + "\bCallback\x12T\n" + "\x05nexus\x18\x02 \x01(\v2<.temporal.server.chasm.lib.callbacks.proto.v1.Callback.NexusH\x00R\x05nexus\x122\n" + "\x05links\x18d \x03(\v2\x1c.temporal.api.common.v1.LinkR\x05links\x1a\xcc\x01\n" + @@ -495,31 +457,30 @@ func file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDescGZIP( } var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_goTypes = []any{ - (CallbackStatus)(0), // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus - (*CallbackState)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState - (*Callback)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.Callback - (*CallbackState_WorkflowClosed)(nil), // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.WorkflowClosed - (*Callback_Nexus)(nil), // 4: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus - nil, // 5: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry - (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp - (*v1.Failure)(nil), // 7: temporal.api.failure.v1.Failure - (*durationpb.Duration)(nil), // 8: google.protobuf.Duration - (*v11.Link)(nil), // 9: temporal.api.common.v1.Link + (CallbackStatus)(0), // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus + (*CallbackState)(nil), // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState + (*Callback)(nil), // 2: temporal.server.chasm.lib.callbacks.proto.v1.Callback + (*Callback_Nexus)(nil), // 3: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus + nil, // 4: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry + (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp + (*v1.Failure)(nil), // 6: temporal.api.failure.v1.Failure + (*durationpb.Duration)(nil), // 7: google.protobuf.Duration + (*v11.Link)(nil), // 8: temporal.api.common.v1.Link } var file_temporal_server_chasm_lib_callback_proto_v1_message_proto_depIdxs = []int32{ 2, // 0: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.callback:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback - 6, // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.registration_time:type_name -> google.protobuf.Timestamp + 5, // 1: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.registration_time:type_name -> google.protobuf.Timestamp 0, // 2: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.status:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.CallbackStatus - 6, // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 7, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure - 6, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp - 6, // 6: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.close_time:type_name -> google.protobuf.Timestamp - 8, // 7: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.schedule_to_close_timeout:type_name -> google.protobuf.Duration - 4, // 8: temporal.server.chasm.lib.callbacks.proto.v1.Callback.nexus:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus - 9, // 9: temporal.server.chasm.lib.callbacks.proto.v1.Callback.links:type_name -> temporal.api.common.v1.Link - 5, // 10: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.header:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry + 5, // 3: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 6, // 4: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure + 5, // 5: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp + 5, // 6: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.close_time:type_name -> google.protobuf.Timestamp + 7, // 7: temporal.server.chasm.lib.callbacks.proto.v1.CallbackState.schedule_to_close_timeout:type_name -> google.protobuf.Duration + 3, // 8: temporal.server.chasm.lib.callbacks.proto.v1.Callback.nexus:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus + 8, // 9: temporal.server.chasm.lib.callbacks.proto.v1.Callback.links:type_name -> temporal.api.common.v1.Link + 4, // 10: temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.header:type_name -> temporal.server.chasm.lib.callbacks.proto.v1.Callback.Nexus.HeaderEntry 11, // [11:11] is the sub-list for method output_type 11, // [11:11] is the sub-list for method input_type 11, // [11:11] is the sub-list for extension type_name @@ -541,7 +502,7 @@ func file_temporal_server_chasm_lib_callback_proto_v1_message_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc), len(file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc)), NumEnums: 1, - NumMessages: 5, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index e7b7f29b41c..16e9970166d 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -10,9 +10,6 @@ import "temporal/api/failure/v1/message.proto"; option go_package = "go.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspb"; message CallbackState { - // Trigger for when the workflow is closed. - message WorkflowClosed {} - // Information on how this callback should be invoked (e.g. its URL and type). Callback callback = 1; // The time when the callback was registered. From cd002c2a85a0093ee63f6f783a5d6c3576b8e482 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 15:11:14 -0700 Subject: [PATCH 21/52] Rely on ChasmCtx.BusinessID --- chasm/lib/callback/component.go | 9 ++++----- .../callback/gen/callbackpb/v1/message.pb.go | 20 ++++--------------- chasm/lib/callback/proto/v1/message.proto | 6 +----- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 67fa531f108..9b691c3de57 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -107,7 +107,6 @@ func newStandaloneCallback( cb := NewEmbeddedCallback(ctx, opts.RequestID, opts.RegistrationTime, opts.Callback) // Add standalone-specific fields. - cb.CallbackId = opts.CallbackID cb.ScheduleToCloseTimeout = opts.ScheduleToCloseTimeout cb.SuppliedCompletion = chasm.NewDataField(ctx, opts.Completion) @@ -140,11 +139,11 @@ func (c *Callback) SetStateMachineState(status callbackspb.CallbackStatus) { c.Status = status } -func (c *Callback) ContextMetadata(_ chasm.Context) map[string]string { +func (c *Callback) ContextMetadata(ctx chasm.Context) map[string]string { return map[string]string{ "RequestID": c.RequestId, // Only set for standalone callbacks. - "CallbackID": c.CallbackId, + "CallbackID": ctx.ExecutionKey().BusinessID, } } @@ -160,7 +159,7 @@ func (c *Callback) SearchAttributes(ctx chasm.Context) []chasm.SearchAttributeKe // as the memo for visibility queries. func (c *Callback) Memo(ctx chasm.Context) proto.Message { return &callbackpb.CallbackExecutionListInfo{ - CallbackId: c.CallbackId, + CallbackId: ctx.ExecutionKey().BusinessID, Status: callbackStatusToAPIExecutionStatus(c.Status), CreateTime: c.RegistrationTime, CloseTime: c.CloseTime, @@ -398,7 +397,7 @@ func (c *Callback) describe(ctx chasm.Context) (*callbackpb.CallbackExecutionInf exInfo := ctx.ExecutionInfo() info := &callbackpb.CallbackExecutionInfo{ - CallbackId: c.CallbackId, + CallbackId: ctx.ExecutionKey().BusinessID, RunId: ctx.ExecutionKey().RunID, Callback: apiCb, Status: callbackStatusToAPIExecutionStatus(c.Status), diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index b116c832cbc..30ad75c801f 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -138,12 +138,9 @@ type CallbackState struct { TerminateRequestId string `protobuf:"bytes,10,opt,name=terminate_request_id,json=terminateRequestId,proto3" json:"terminate_request_id,omitempty"` // The time when the callback reached a terminal state. CloseTime *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=close_time,json=closeTime,proto3" json:"close_time,omitempty"` - // (standalone only) User-supplied business ID set when StartCallbackExecution() is - // called. Used to identify the callback for operations like Describe- or Terminate-. - CallbackId string `protobuf:"bytes,12,opt,name=callback_id,json=callbackId,proto3" json:"callback_id,omitempty"` // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() // is called to when the result gets delivered. - ScheduleToCloseTimeout *durationpb.Duration `protobuf:"bytes,13,opt,name=schedule_to_close_timeout,json=scheduleToCloseTimeout,proto3" json:"schedule_to_close_timeout,omitempty"` + ScheduleToCloseTimeout *durationpb.Duration `protobuf:"bytes,12,opt,name=schedule_to_close_timeout,json=scheduleToCloseTimeout,proto3" json:"schedule_to_close_timeout,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -248,13 +245,6 @@ func (x *CallbackState) GetCloseTime() *timestamppb.Timestamp { return nil } -func (x *CallbackState) GetCallbackId() string { - if x != nil { - return x.CallbackId - } - return "" -} - func (x *CallbackState) GetScheduleToCloseTimeout() *durationpb.Duration { if x != nil { return x.ScheduleToCloseTimeout @@ -406,7 +396,7 @@ var File_temporal_server_chasm_lib_callback_proto_v1_message_proto protoreflect. const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = "" + "\n" + - "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\xa5\x06\n" + + "9temporal/server/chasm/lib/callback/proto/v1/message.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\"\x84\x06\n" + "\rCallbackState\x12R\n" + "\bcallback\x18\x01 \x01(\v26.temporal.server.chasm.lib.callbacks.proto.v1.CallbackR\bcallback\x12G\n" + "\x11registration_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x10registrationTime\x12T\n" + @@ -420,10 +410,8 @@ const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = " "\x14terminate_request_id\x18\n" + " \x01(\tR\x12terminateRequestId\x129\n" + "\n" + - "close_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcloseTime\x12\x1f\n" + - "\vcallback_id\x18\f \x01(\tR\n" + - "callbackId\x12T\n" + - "\x19schedule_to_close_timeout\x18\r \x01(\v2\x19.google.protobuf.DurationR\x16scheduleToCloseTimeout\"\xf4\x02\n" + + "close_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcloseTime\x12T\n" + + "\x19schedule_to_close_timeout\x18\f \x01(\v2\x19.google.protobuf.DurationR\x16scheduleToCloseTimeout\"\xf4\x02\n" + "\bCallback\x12T\n" + "\x05nexus\x18\x02 \x01(\v2<.temporal.server.chasm.lib.callbacks.proto.v1.Callback.NexusH\x00R\x05nexus\x122\n" + "\x05links\x18d \x03(\v2\x1c.temporal.api.common.v1.LinkR\x05links\x1a\xcc\x01\n" + diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index 16e9970166d..e3c3933145e 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -37,13 +37,9 @@ message CallbackState { // The time when the callback reached a terminal state. google.protobuf.Timestamp close_time = 11; - // (standalone only) User-supplied business ID set when StartCallbackExecution() is - // called. Used to identify the callback for operations like Describe- or Terminate-. - string callback_id = 12; - // (standalone only) Schedule-to-close timeout from when StartCallbackExecution() // is called to when the result gets delivered. - google.protobuf.Duration schedule_to_close_timeout = 13; + google.protobuf.Duration schedule_to_close_timeout = 12; } // Status of a callback. From 173fb588cf952196b330a884024f9dff6b1d81a3 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 15:41:46 -0700 Subject: [PATCH 22/52] Address PR feedback --- chasm/lib/callback/component.go | 37 ++++----------------- chasm/lib/callback/handler.go | 39 ++++++++++------------- chasm/lib/callback/proto/v1/message.proto | 5 ++- chasm/lib/callback/statemachine.go | 1 - 4 files changed, 27 insertions(+), 55 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 9b691c3de57..8489f488de7 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -33,13 +33,10 @@ func (csFunc CompletionSourceFn) GetNexusCompletion(ctx chasm.Context, requestID } var ( - _ chasm.Component = (*Callback)(nil) + _ chasm.RootComponent = (*Callback)(nil) _ chasm.StateMachine[callbackspb.CallbackStatus] = (*Callback)(nil) - - // Capabilities only supported/used for standalone callbacks. - _ chasm.RootComponent = (*Callback)(nil) - _ chasm.VisibilityMemoProvider = (*Callback)(nil) - _ chasm.VisibilitySearchAttributesProvider = (*Callback)(nil) + _ chasm.VisibilityMemoProvider = (*Callback)(nil) + _ chasm.VisibilitySearchAttributesProvider = (*Callback)(nil) ) var executionStatusSearchAttribute = chasm.NewSearchAttributeKeyword( @@ -58,11 +55,11 @@ type Callback struct { // of its potential size, and to not overload CallbackState::LastAttemptFailure. TerminalFailure chasm.Field[*failurepb.Failure] - // For most callbacks, the completion result is obtained from the parent component. - // e.g. the Workflow result to be delivered. However, for standalone callbacks, there - // is no parent and the user-supplied SuppliedCompletion will be used instead. + // For embedded callbacks, the completion result is obtained from the parent component. + // e.g. the Workflow result to be delivered. ParentCompletionSource chasm.ParentPtr[CompletionSource] - SuppliedCompletion chasm.Field[*callbackpb.CallbackExecutionCompletion] + // The user-supplied completion for standalone callbacks. + SuppliedCompletion chasm.Field[*callbackpb.CallbackExecutionCompletion] // Visibility sub-component for search attributes and memo indexing. Visibility chasm.Field[*chasm.Visibility] @@ -83,7 +80,6 @@ func NewEmbeddedCallback( Callback: cb, Status: callbackspb.CALLBACK_STATUS_STANDBY, }, - TerminalFailure: chasm.NewDataField[*failurepb.Failure](ctx, nil), } } @@ -92,30 +88,11 @@ type newStandaloneCallbackOpts struct { RegistrationTime *timestamppb.Timestamp Callback *callbackspb.Callback - CallbackID string ScheduleToCloseTimeout *durationpb.Duration Completion *callbackpb.CallbackExecutionCompletion SearchAttributes map[string]*commonpb.Payload } -// newStandaloneCallback returns a new Callback component which will deliver the supplied -// completion result. -func newStandaloneCallback( - ctx chasm.MutableContext, - opts newStandaloneCallbackOpts, -) *Callback { - cb := NewEmbeddedCallback(ctx, opts.RequestID, opts.RegistrationTime, opts.Callback) - - // Add standalone-specific fields. - cb.ScheduleToCloseTimeout = opts.ScheduleToCloseTimeout - cb.SuppliedCompletion = chasm.NewDataField(ctx, opts.Completion) - - visibility := chasm.NewVisibilityWithData(ctx, opts.SearchAttributes, nil) - cb.Visibility = chasm.NewComponentField(ctx, visibility) - - return cb -} - func (c *Callback) LifecycleState(_ chasm.Context) chasm.LifecycleState { switch c.Status { case callbackspb.CALLBACK_STATUS_SUCCEEDED: diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go index af996a1541a..9e995351e39 100644 --- a/chasm/lib/callback/handler.go +++ b/chasm/lib/callback/handler.go @@ -36,11 +36,10 @@ func (h *callbackHandler) StartCallbackExecution( // Gather all the data necessary to create the Callback component. input := &createStandaloneCallbackInput{ - CallbackID: frontendReq.GetCallbackId(), - RequestID: frontendReq.GetRequestId(), - CompletionScheduleToCloseTimeout: frontendReq.GetScheduleToCloseTimeout(), - Completion: frontendReq.GetCompletion(), - SearchAttributes: frontendReq.GetSearchAttributes().GetIndexedFields(), + RequestID: frontendReq.GetRequestId(), + ScheduleToCloseTimeout: frontendReq.GetScheduleToCloseTimeout(), + Completion: frontendReq.GetCompletion(), + SearchAttributes: frontendReq.GetSearchAttributes().GetIndexedFields(), } // Convert the API Callback to internal Callback proto. @@ -297,12 +296,11 @@ func (h *callbackHandler) DeleteCallbackExecution( // createStandaloneCallbackInput is the bundle of inputs to the CHASM execution. type createStandaloneCallbackInput struct { - RequestID string - Callback *callbackspb.Callback - CallbackID string - CompletionScheduleToCloseTimeout *durationpb.Duration - Completion *callbackpb.CallbackExecutionCompletion - SearchAttributes map[string]*commonpb.Payload + RequestID string + Callback *callbackspb.Callback + ScheduleToCloseTimeout *durationpb.Duration + Completion *callbackpb.CallbackExecutionCompletion + SearchAttributes map[string]*commonpb.Payload } // createStandaloneCallback constructs a new Callback component in standalone mode. @@ -313,18 +311,13 @@ func createStandaloneCallback( ) (*Callback, error) { now := timestamppb.Now() - // Create child Callback component. - opts := newStandaloneCallbackOpts{ - RequestID: input.RequestID, - RegistrationTime: now, - Callback: input.Callback, + // Create Callback component. + cb := NewEmbeddedCallback(ctx, input.RequestID, now, input.Callback) + cb.ScheduleToCloseTimeout = input.ScheduleToCloseTimeout + cb.SuppliedCompletion = chasm.NewDataField(ctx, input.Completion) - CallbackID: input.CallbackID, - ScheduleToCloseTimeout: input.CompletionScheduleToCloseTimeout, - Completion: input.Completion, - SearchAttributes: input.SearchAttributes, - } - cb := newStandaloneCallback(ctx, opts) + visibility := chasm.NewVisibilityWithData(ctx, input.SearchAttributes, nil) + cb.Visibility = chasm.NewComponentField(ctx, visibility) // Immediately schedule the callback for invocation. if err := TransitionScheduled.Apply(cb, ctx, EventScheduled{}); err != nil { @@ -332,7 +325,7 @@ func createStandaloneCallback( } // Schedule the timeout as applicable. - if durationProto := input.CompletionScheduleToCloseTimeout; durationProto != nil { + if durationProto := input.ScheduleToCloseTimeout; durationProto != nil { if duration := durationProto.AsDuration(); duration > 0 { timeoutTime := now.AsTime().Add(duration) ctx.AddTask( diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index e3c3933145e..b309edbc3ca 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -56,8 +56,11 @@ enum CallbackStatus { CALLBACK_STATUS_FAILED = 4; // Callback has succeeded. CALLBACK_STATUS_SUCCEEDED = 5; - // Callback was terminated by request. + // Callback was terminated by request. Only relevant for standalone callbacks. CALLBACK_STATUS_TERMINATED = 6; + // Callback exceeded the schedule-to-close timeout. + CALLBACK_STATUS_TIMED_OUT = 7; + // TODO(chrsmith): Wire this new state in. } message Callback { diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index ce2e71b3aec..66545063d16 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -130,7 +130,6 @@ var TransitionSucceeded = chasm.NewTransition( cb.recordAttempt(now) cb.CloseTime = timestamppb.New(now) cb.LastAttemptFailure = nil - cb.TerminalFailure = chasm.NewDataField[*failurepb.Failure](ctx, nil) return nil }, ) From 47213157e757b4f8f2cd42443f1603b9ad2e389b Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 15:51:11 -0700 Subject: [PATCH 23/52] Address PR feedback --- chasm/lib/callback/component.go | 11 +++++------ .../gen/callbackpb/v1/message.go-helpers.pb.go | 1 + chasm/lib/callback/gen/callbackpb/v1/message.pb.go | 13 ++++++++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 8489f488de7..6f521944049 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -98,10 +98,9 @@ func (c *Callback) LifecycleState(_ chasm.Context) chasm.LifecycleState { case callbackspb.CALLBACK_STATUS_SUCCEEDED: return chasm.LifecycleStateCompleted case callbackspb.CALLBACK_STATUS_FAILED, - callbackspb.CALLBACK_STATUS_TERMINATED: - // TODO: Use chasm.LifecycleStateTerminated when it's available (currently commented out - // in chasm/component.go:70). For now, LifecycleStateFailed is functionally correct - // as IsClosed() returns true for all states >= LifecycleStateCompleted. + callbackspb.CALLBACK_STATUS_TERMINATED, + callbackspb.CALLBACK_STATUS_TIMED_OUT: + // TODO(chrsmith): Confirm the response from #crew-chasm return chasm.LifecycleStateFailed default: return chasm.LifecycleStateRunning @@ -118,9 +117,9 @@ func (c *Callback) SetStateMachineState(status callbackspb.CallbackStatus) { func (c *Callback) ContextMetadata(ctx chasm.Context) map[string]string { return map[string]string{ - "RequestID": c.RequestId, + "request-id": c.RequestId, // Only set for standalone callbacks. - "CallbackID": ctx.ExecutionKey().BusinessID, + "callback-id": ctx.ExecutionKey().BusinessID, } } diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go index c9900bb64b0..476b38dc3df 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.go-helpers.pb.go @@ -90,6 +90,7 @@ var ( "Failed": 4, "Succeeded": 5, "Terminated": 6, + "TimedOut": 7, } ) diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index 30ad75c801f..542cfba8cdb 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -43,8 +43,10 @@ const ( CALLBACK_STATUS_FAILED CallbackStatus = 4 // Callback has succeeded. CALLBACK_STATUS_SUCCEEDED CallbackStatus = 5 - // Callback was terminated by request. + // Callback was terminated by request. Only relevant for standalone callbacks. CALLBACK_STATUS_TERMINATED CallbackStatus = 6 + // Callback exceeded the schedule-to-close timeout. + CALLBACK_STATUS_TIMED_OUT CallbackStatus = 7 // TODO(chrsmith): Wire this new state in. ) // Enum value maps for CallbackStatus. @@ -57,6 +59,7 @@ var ( 4: "CALLBACK_STATUS_FAILED", 5: "CALLBACK_STATUS_SUCCEEDED", 6: "CALLBACK_STATUS_TERMINATED", + 7: "CALLBACK_STATUS_TIMED_OUT", } CallbackStatus_value = map[string]int32{ "CALLBACK_STATUS_UNSPECIFIED": 0, @@ -66,6 +69,7 @@ var ( "CALLBACK_STATUS_FAILED": 4, "CALLBACK_STATUS_SUCCEEDED": 5, "CALLBACK_STATUS_TERMINATED": 6, + "CALLBACK_STATUS_TIMED_OUT": 7, } ) @@ -91,6 +95,8 @@ func (x CallbackStatus) String() string { return "Succeeded" case CALLBACK_STATUS_TERMINATED: return "Terminated" + case CALLBACK_STATUS_TIMED_OUT: + return "TimedOut" default: return strconv.Itoa(int(x)) } @@ -422,7 +428,7 @@ const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = " "\vHeaderEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\t\n" + - "\avariantJ\x04\b\x01\x10\x02*\xe9\x01\n" + + "\avariantJ\x04\b\x01\x10\x02*\x88\x02\n" + "\x0eCallbackStatus\x12\x1f\n" + "\x1bCALLBACK_STATUS_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17CALLBACK_STATUS_STANDBY\x10\x01\x12\x1d\n" + @@ -430,7 +436,8 @@ const file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDesc = " "\x1bCALLBACK_STATUS_BACKING_OFF\x10\x03\x12\x1a\n" + "\x16CALLBACK_STATUS_FAILED\x10\x04\x12\x1d\n" + "\x19CALLBACK_STATUS_SUCCEEDED\x10\x05\x12\x1e\n" + - "\x1aCALLBACK_STATUS_TERMINATED\x10\x06BGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" + "\x1aCALLBACK_STATUS_TERMINATED\x10\x06\x12\x1d\n" + + "\x19CALLBACK_STATUS_TIMED_OUT\x10\aBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" var ( file_temporal_server_chasm_lib_callback_proto_v1_message_proto_rawDescOnce sync.Once From 9bd7dcb92ae15542c3ccf9f81f5187592dc876e3 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 16:03:50 -0700 Subject: [PATCH 24/52] Address PR feedback --- chasm/lib/callback/component.go | 84 ++++++++----------------- chasm/lib/callback/statemachine_test.go | 17 ----- 2 files changed, 25 insertions(+), 76 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 6f521944049..3f3dd9b810a 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -16,7 +16,6 @@ import ( commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" queueserrors "go.temporal.io/server/service/history/queues/errors" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -25,18 +24,14 @@ type CompletionSource interface { GetNexusCompletion(ctx chasm.Context, requestID string) (nexusrpc.CompleteOperationOptions, error) } -// CompletionSourceFn allows a function value to be used as a CompletionSource instance. -type CompletionSourceFn func(chasm.Context, string) (nexusrpc.CompleteOperationOptions, error) - -func (csFunc CompletionSourceFn) GetNexusCompletion(ctx chasm.Context, requestID string) (nexusrpc.CompleteOperationOptions, error) { - return csFunc(ctx, requestID) -} - var ( _ chasm.RootComponent = (*Callback)(nil) _ chasm.StateMachine[callbackspb.CallbackStatus] = (*Callback)(nil) - _ chasm.VisibilityMemoProvider = (*Callback)(nil) _ chasm.VisibilitySearchAttributesProvider = (*Callback)(nil) + + // The Callback itself is a completion source. Either delegating to its parent + // or returning the user-supplied completion. + _ CompletionSource = (*Callback)(nil) ) var executionStatusSearchAttribute = chasm.NewSearchAttributeKeyword( @@ -125,23 +120,12 @@ func (c *Callback) ContextMetadata(ctx chasm.Context) map[string]string { // SearchAttributes implements chasm.VisibilitySearchAttributesProvider. func (c *Callback) SearchAttributes(ctx chasm.Context) []chasm.SearchAttributeKeyValue { - apiStatus := callbackStatusToAPIExecutionStatus(c.Status) + apiStatus := c.statusAsAPIExecutionStatus() return []chasm.SearchAttributeKeyValue{ executionStatusSearchAttribute.Value(apiStatus.String()), } } -// Memo implements chasm.VisibilityMemoProvider. Returns the CallbackExecutionListInfo -// as the memo for visibility queries. -func (c *Callback) Memo(ctx chasm.Context) proto.Message { - return &callbackpb.CallbackExecutionListInfo{ - CallbackId: ctx.ExecutionKey().BusinessID, - Status: callbackStatusToAPIExecutionStatus(c.Status), - CreateTime: c.RegistrationTime, - CloseTime: c.CloseTime, - } -} - // Terminate forcefully terminates the callback execution. // // If already terminated with the same request ID, this is a no-op. @@ -182,8 +166,7 @@ func (c *Callback) loadInvocationArgs( _ chasm.NoValue, ) (invocable, error) { // Get the completion result to be delivered. - completionSource := c.completionSource(ctx) - completion, err := completionSource.GetNexusCompletion(ctx, c.RequestId) + completion, err := c.GetNexusCompletion(ctx, c.RequestId) if err != nil { return nil, err } @@ -197,6 +180,9 @@ func (c *Callback) loadInvocationArgs( // Setup the completion's headers. completion.Header = callback.Header + // TODO(chrsmith): + // > Is this a behavior change for workflow callbacks? + // Yes? And I'm not sure if that's a good thing or not... if callback.GetToken() != "" { if completion.Header == nil { completion.Header = nexus.Header{} @@ -230,16 +216,6 @@ func (c *Callback) saveResult( ctx chasm.MutableContext, input saveResultInput, ) (chasm.NoValue, error) { - // If the callback was terminated while the invocation was in-flight, - // the result is no longer relevant. We'll just drop it silently. - // - // This shouldn't happen outside of tests, since the Nexus machinery - // would prevent an invalid transition anyways. (e.g. terminating - // an already terminated Callback.) - if c.LifecycleState(ctx).IsClosed() { - return nil, nil - } - switch r := input.result.(type) { case invocationResultOK: err := TransitionSucceeded.Apply(c, ctx, EventSucceeded{Time: ctx.Now(c)}) @@ -340,28 +316,19 @@ func callbackCompletionToNexusCompleteOperationOpts( } } -// completionSource returns the completionSource from the callback, which depends on whether it -// is embedded or is running in standalone mode. -func (c *Callback) completionSource(ctx chasm.Context) CompletionSource { +// GetNexusCompletion returns the Nexus completion to be delivered for the callback. +func (c *Callback) GetNexusCompletion(ctx chasm.Context, requestID string) (nexusrpc.CompleteOperationOptions, error) { // Embedded callbacks use their parent component as a CompletionSource. - source, ok := c.ParentCompletionSource.TryGet(ctx) - if ok { - return source + if source, ok := c.ParentCompletionSource.TryGet(ctx); ok { + return source.GetNexusCompletion(ctx, requestID) } - // For standalone completions, get the user-supplied value and convert it - // into the Nexus API type. + // For standalone completions, get the user-supplied value and convert it into the Nexus API type. suppliedCompletion, ok := c.SuppliedCompletion.TryGet(ctx) if !ok { - return CompletionSourceFn(func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { - return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInternal("no completion available") - }) + panic("no completion available") } - - convertOutcomeProtoFn := func(_ chasm.Context, _ string) (nexusrpc.CompleteOperationOptions, error) { - return callbackCompletionToNexusCompleteOperationOpts(c, suppliedCompletion) - } - return CompletionSourceFn(convertOutcomeProtoFn) + return callbackCompletionToNexusCompleteOperationOpts(c, suppliedCompletion) } // describe returns the CallbackExecutionInfo for the describe RPC. Only applies to standalone callbacks. @@ -372,12 +339,13 @@ func (c *Callback) describe(ctx chasm.Context) (*callbackpb.CallbackExecutionInf } exInfo := ctx.ExecutionInfo() + exKey := ctx.ExecutionKey() info := &callbackpb.CallbackExecutionInfo{ - CallbackId: ctx.ExecutionKey().BusinessID, - RunId: ctx.ExecutionKey().RunID, + CallbackId: exKey.BusinessID, + RunId: exKey.RunID, Callback: apiCb, - Status: callbackStatusToAPIExecutionStatus(c.Status), - State: callbackStatusToAPIState(c.Status), + Status: c.statusAsAPIExecutionStatus(), + State: c.statusAsAPIState(), Attempt: c.Attempt, CreateTime: c.RegistrationTime, LastAttemptCompleteTime: c.LastAttemptCompleteTime, @@ -416,9 +384,8 @@ func (c *Callback) outcome(ctx chasm.Context) *callbackpb.CallbackExecutionOutco } } -// callbackStatusToAPIExecutionStatus maps internal CallbackStatus to public API CallbackExecutionStatus. -func callbackStatusToAPIExecutionStatus(status callbackspb.CallbackStatus) enumspb.CallbackExecutionStatus { - switch status { +func (c *Callback) statusAsAPIExecutionStatus() enumspb.CallbackExecutionStatus { + switch c.Status { case callbackspb.CALLBACK_STATUS_STANDBY, callbackspb.CALLBACK_STATUS_SCHEDULED, callbackspb.CALLBACK_STATUS_BACKING_OFF: @@ -434,9 +401,8 @@ func callbackStatusToAPIExecutionStatus(status callbackspb.CallbackStatus) enums } } -// callbackStatusToAPIState maps internal CallbackStatus to public API CallbackState. -func callbackStatusToAPIState(status callbackspb.CallbackStatus) enumspb.CallbackState { - switch status { +func (c *Callback) statusAsAPIState() enumspb.CallbackState { + switch c.Status { case callbackspb.CALLBACK_STATUS_STANDBY: return enumspb.CALLBACK_STATE_STANDBY case callbackspb.CALLBACK_STATUS_SCHEDULED: diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index 722de19930c..45565fee882 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -261,20 +261,3 @@ func TestTerminatedTransition(t *testing.T) { }) } } - -func TestSaveResult_TerminatedWhileInFlight(t *testing.T) { - // If the callback was terminated while an invocation was in-flight, - // saveResult should drop the result silently. - cb := &Callback{ - CallbackState: &callbackspb.CallbackState{ - Status: callbackspb.CALLBACK_STATUS_TERMINATED, - }, - } - mctx := &chasm.MockMutableContext{} - _, err := cb.saveResult(mctx, saveResultInput{ - result: invocationResultOK{}, - retryPolicy: backoff.NewExponentialRetryPolicy(time.Second), - }) - require.NoError(t, err) - require.Equal(t, callbackspb.CALLBACK_STATUS_TERMINATED, cb.StateMachineState()) -} From 764d360b8c8cd409b87a362097fcfaf9a41620ed Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 19:07:20 -0700 Subject: [PATCH 25/52] Address PR feedback --- chasm/lib/callback/frontend.go | 14 +++--- chasm/lib/callback/frontend_validation.go | 8 +-- .../lib/callback/frontend_validation_test.go | 2 +- tests/standalone_callbacks_test.go | 49 +++++++------------ 4 files changed, 29 insertions(+), 44 deletions(-) diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go index 242c9237593..a1a316cc1de 100644 --- a/chasm/lib/callback/frontend.go +++ b/chasm/lib/callback/frontend.go @@ -65,7 +65,7 @@ func NewFrontendHandler( type namespacer interface{ GetNamespace() string } // Looks up the namespace ID from the user-supplied namespace name in the request proto. -func (h *frontendHandler) getTargetNamespace(requestProto namespacer) (namespace.ID, error) { +func (h *frontendHandler) targetNamespace(requestProto namespacer) (namespace.ID, error) { targetNamespaceName := namespace.Name(requestProto.GetNamespace()) namespaceID, err := h.namespaceRegistry.GetNamespaceID(targetNamespaceName) if err != nil { @@ -77,7 +77,7 @@ func (h *frontendHandler) getTargetNamespace(requestProto namespacer) (namespace func (h *frontendHandler) checkFeatureEnabled(requestProto namespacer) error { // Confirm CHASM is enabled. targetNamespaceName := requestProto.GetNamespace() - if !h.config.CHASMEnabled(targetNamespaceName) || !h.config.CHASMCallbacksEnabled(targetNamespaceName) { + if !h.config.CHASMEnabled(targetNamespaceName) { return ErrStandaloneCallbacksDisabled } if !h.config.EnableStandaloneExecutions(targetNamespaceName) { @@ -101,7 +101,7 @@ func (h *frontendHandler) StartCallbackExecution( } // Execute - namespaceID, err := h.getTargetNamespace(request) + namespaceID, err := h.targetNamespace(request) if err != nil { return nil, err } @@ -128,7 +128,7 @@ func (h *frontendHandler) DescribeCallbackExecution( } // Get the Namespace ID to confirm the optional long-poll token matches. - namespaceID, err := h.getTargetNamespace(request) + namespaceID, err := h.targetNamespace(request) if err != nil { return nil, err } @@ -161,7 +161,7 @@ func (h *frontendHandler) PollCallbackExecution( } // Execute - namespaceID, err := h.getTargetNamespace(request) + namespaceID, err := h.targetNamespace(request) if err != nil { return nil, err } @@ -190,7 +190,7 @@ func (h *frontendHandler) TerminateCallbackExecution( } // Execute - namespaceID, err := h.getTargetNamespace(request) + namespaceID, err := h.targetNamespace(request) if err != nil { return nil, err } @@ -218,7 +218,7 @@ func (h *frontendHandler) DeleteCallbackExecution( } // Execute - namespaceID, err := h.getTargetNamespace(request) + namespaceID, err := h.targetNamespace(request) if err != nil { return nil, err } diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index 371a37c169a..95e2853905d 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -38,22 +38,22 @@ func verifyIsUUID(fieldName, fieldValue string) error { return nil } -// requiredField is a tuple of a required field name and its value. +// requiredStringField is a tuple of a required field name and its value. // Used instead of a map[string]string to provide deterministic // errors if multiple fields aren't set. -type requiredField struct { +type requiredStringField struct { FieldName string Value string } -func (rf requiredField) Validate() error { +func (rf requiredStringField) Validate() error { if rf.Value == "" { return missingRequiredFieldError(rf.FieldName) } return nil } -type requiredFields []requiredField +type requiredFields []requiredStringField func (fields requiredFields) Validate() error { for _, rf := range fields { diff --git a/chasm/lib/callback/frontend_validation_test.go b/chasm/lib/callback/frontend_validation_test.go index d8b8fb18973..a896e878bf3 100644 --- a/chasm/lib/callback/frontend_validation_test.go +++ b/chasm/lib/callback/frontend_validation_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestRequiredFields(t *testing.T) { +func TestRequiredStringFields(t *testing.T) { // Positive tests positiveTests := requiredFields{ {"Field1", "exists"}, diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index a8156c8f5d7..88a3f8970e6 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -37,7 +37,7 @@ import ( ) // Test suite for the Nexus "Standalone Callbacks". Which are Nexus operations corresponding to -// aysynchronous actions that take place outside of Temporal. (e.g. waiting for a payment to +// asynchronous actions that take place outside of Temporal. (e.g. waiting for a payment to // be processed, or webhook to be delivered, etc.) // Minimal information that an external service would need to report the results of a callback. @@ -377,8 +377,7 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { s.Run("success", func(s *StandaloneCallbackSuite) { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := env.Context() wantPayloadStr := "successfully delivered payload via external svc" successCompletion := &callbackpb.CallbackExecutionCompletion{ @@ -398,8 +397,7 @@ func (s *StandaloneCallbackSuite) TestBasicOperation() { s.Run("failure", func(s *StandaloneCallbackSuite) { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := env.Context() failureCompletion := &callbackpb.CallbackExecutionCompletion{ Result: &callbackpb.CallbackExecutionCompletion_Failure{ @@ -445,8 +443,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.Run("returns_empty_for_non_terminal", func(s *StandaloneCallbackSuite) { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + ctx := env.Context() callbackID := "poll-test-" + uuid.NewString() @@ -486,8 +483,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.Run("blocks_until_complete", func(s *StandaloneCallbackSuite) { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + ctx := env.Context() callbackID := "poll-blocks-" + uuid.NewString() s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) @@ -532,8 +528,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.Run("returns_run_id", func(s *StandaloneCallbackSuite) { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + ctx := env.Context() callbackID := "poll-runid-" + uuid.NewString() startResp := s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) @@ -562,8 +557,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { s.Run("poll_after_timeout", func(s *StandaloneCallbackSuite) { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() + ctx := env.Context() callbackID := "poll-timeout-" + uuid.NewString() // Start with a very short schedule-to-close timeout so it times out quickly. @@ -593,8 +587,7 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { // Delete terminates the callback if it's still running, then marks it for cleanup. func (s *StandaloneCallbackSuite) TestDeleteCallbackExecution() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := env.Context() // Create a callback that points to a non-existent URL so it won't complete on its own. // The callback will be in SCHEDULED/BACKING_OFF state when we delete it. @@ -738,8 +731,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() for _, tc := range tests { s.Run(tc.name, func(s *StandaloneCallbackSuite) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx := env.Context() req := &workflowservice.StartCallbackExecutionRequest{ Namespace: env.Namespace().String(), @@ -763,8 +755,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() // and that the same request_id is idempotent (returns the existing run_id without error). func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() + ctx := env.Context() callbackID := "dup-test-" + uuid.NewString() requestID := uuid.NewString() @@ -812,8 +803,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_DuplicateID() { // can be listed and counted via the visibility APIs, and verifies the returned data. func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) - defer cancel() + ctx := env.Context() // Create two callback executions with known IDs. callbackIDs := make([]string, 2) @@ -916,8 +906,7 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { // are persisted and can be used to query callback executions via list filtering. func (s *StandaloneCallbackSuite) TestStartCallbackExecution_SearchAttributes() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() + ctx := env.Context() callbackID := "sa-test-" + uuid.NewString() saValue := "sa-test-value-" + uuid.NewString() @@ -964,8 +953,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_SearchAttributes() // TestTerminateCallbackExecution tests terminate, run_id validation, and request ID idempotency. func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) - defer cancel() + ctx := env.Context() callbackID := "terminate-test-" + uuid.NewString() startResp := s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, time.Minute) @@ -1060,8 +1048,7 @@ func (s *StandaloneCallbackSuite) TestTerminateCallbackExecution() { // (e.g., a 400 response from the target), the poll outcome contains the failure details. func (s *StandaloneCallbackSuite) TestCallbackExecutionFailedOutcome() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() + ctx := env.Context() // Start an HTTP server that always returns 400 Bad Request (non-retryable). srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1107,8 +1094,7 @@ func (s *StandaloneCallbackSuite) TestCallbackExecutionFailedOutcome() { // receives the result regardless of the start handler timing. func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandlerReturns() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) - defer cancel() + ctx := env.Context() taskQueue := testcore.RandomizeStr(s.T().Name()) endpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) @@ -1215,11 +1201,10 @@ func (s *StandaloneCallbackSuite) TestNexusOperationCompletionBeforeStartHandler // when its schedule-to-close timeout expires before the callback succeeds. func (s *StandaloneCallbackSuite) TestScheduleToCloseTimeout() { env := s.newEnv() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() + ctx := env.Context() // Short timeout so it fires quickly during the test. - callbackID := "timeout-test-" + uuid.NewString() + callbackID := "timeout-test" + uuid.NewString() s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackID, 2*time.Second) // Poll until the callback reaches a terminal state due to timeout. From b14fb0e06db42ddec1f0662d0edbd51f6c1393d4 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 13 May 2026 19:08:59 -0700 Subject: [PATCH 26/52] Wire in TIMED_OUT state --- chasm/lib/callback/proto/v1/message.proto | 1 - chasm/lib/callback/statemachine.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/chasm/lib/callback/proto/v1/message.proto b/chasm/lib/callback/proto/v1/message.proto index b309edbc3ca..efd2a4f2858 100644 --- a/chasm/lib/callback/proto/v1/message.proto +++ b/chasm/lib/callback/proto/v1/message.proto @@ -60,7 +60,6 @@ enum CallbackStatus { CALLBACK_STATUS_TERMINATED = 6; // Callback exceeded the schedule-to-close timeout. CALLBACK_STATUS_TIMED_OUT = 7; - // TODO(chrsmith): Wire this new state in. } message Callback { diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 66545063d16..4541c5181b1 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -179,7 +179,7 @@ var TransitionTimedOut = chasm.NewTransition( callbackspb.CALLBACK_STATUS_SCHEDULED, callbackspb.CALLBACK_STATUS_BACKING_OFF, }, - callbackspb.CALLBACK_STATUS_FAILED, + callbackspb.CALLBACK_STATUS_TIMED_OUT, func(cb *Callback, ctx chasm.MutableContext, event EventTimedOut) error { now := ctx.Now(cb) cb.CloseTime = timestamppb.New(now) From afd037048851edb1f6a8433513feb419ee5ca8ba Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 14 May 2026 12:10:28 -0700 Subject: [PATCH 27/52] Move Describe- endpoint to long-poll category --- chasm/lib/callback/proto/v1/service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chasm/lib/callback/proto/v1/service.proto b/chasm/lib/callback/proto/v1/service.proto index 1413d10ac4b..dfc22fff170 100644 --- a/chasm/lib/callback/proto/v1/service.proto +++ b/chasm/lib/callback/proto/v1/service.proto @@ -16,7 +16,7 @@ service CallbackService { rpc DescribeCallbackExecution(DescribeCallbackExecutionRequest) returns (DescribeCallbackExecutionResponse) { option (temporal.server.api.routing.v1.routing).business_id = "frontend_request.callback_id"; - option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_STANDARD; + option (temporal.server.api.common.v1.api_category).category = API_CATEGORY_LONG_POLL; } rpc PollCallbackExecution(PollCallbackExecutionRequest) returns (PollCallbackExecutionResponse) { From 6e9925dbef546dc40eb314373b00a1a36dc70823 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 14 May 2026 13:23:41 -0700 Subject: [PATCH 28/52] Update tests to account for STATE_TIMED_OUT --- chasm/lib/callback/component.go | 5 +++++ chasm/lib/callback/frontend.go | 2 +- chasm/lib/callback/frontend_validation.go | 6 ++++-- chasm/lib/callback/validator_test.go | 3 ++- tests/standalone_callbacks_test.go | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 3f3dd9b810a..7e445275bdd 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -371,6 +371,7 @@ func (c *Callback) outcome(ctx chasm.Context) *callbackpb.CallbackExecutionOutco } case callbackspb.CALLBACK_STATUS_FAILED, + callbackspb.CALLBACK_STATUS_TIMED_OUT, callbackspb.CALLBACK_STATUS_TERMINATED: val := &callbackpb.CallbackExecutionOutcome_Failure{ Failure: c.TerminalFailure.Get(ctx), @@ -394,6 +395,8 @@ func (c *Callback) statusAsAPIExecutionStatus() enumspb.CallbackExecutionStatus return enumspb.CALLBACK_EXECUTION_STATUS_FAILED case callbackspb.CALLBACK_STATUS_SUCCEEDED: return enumspb.CALLBACK_EXECUTION_STATUS_SUCCEEDED + case callbackspb.CALLBACK_STATUS_TIMED_OUT: + return enumspb.CALLBACK_EXECUTION_STATUS_FAILED case callbackspb.CALLBACK_STATUS_TERMINATED: return enumspb.CALLBACK_EXECUTION_STATUS_TERMINATED default: @@ -413,6 +416,8 @@ func (c *Callback) statusAsAPIState() enumspb.CallbackState { return enumspb.CALLBACK_STATE_FAILED case callbackspb.CALLBACK_STATUS_SUCCEEDED: return enumspb.CALLBACK_STATE_SUCCEEDED + case callbackspb.CALLBACK_STATUS_TIMED_OUT: + return enumspb.CALLBACK_STATE_TIMED_OUT case callbackspb.CALLBACK_STATUS_TERMINATED: return enumspb.CALLBACK_STATE_TERMINATED default: diff --git a/chasm/lib/callback/frontend.go b/chasm/lib/callback/frontend.go index a1a316cc1de..8af52ae2558 100644 --- a/chasm/lib/callback/frontend.go +++ b/chasm/lib/callback/frontend.go @@ -96,7 +96,7 @@ func (h *frontendHandler) StartCallbackExecution( if err := h.checkFeatureEnabled(request); err != nil { return nil, err } - if err := h.reqValidator.ValidateAndNormalizeStartCallbackExecution(request); err != nil { + if err := h.reqValidator.ValidateAndNormalizeStartCallbackExecution(ctx, request); err != nil { return nil, err } diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index 95e2853905d..ca913560514 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -1,6 +1,7 @@ package callback import ( + "context" "fmt" "github.com/google/uuid" @@ -76,7 +77,8 @@ type frontendRequestValidator struct { saValidator *searchattribute.Validator } -func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution(req *workflowservice.StartCallbackExecutionRequest) error { +func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution( + ctx context.Context, req *workflowservice.StartCallbackExecutionRequest) error { // Set RequestID if missing. if req.GetRequestId() == "" { req.RequestId = uuid.NewString() @@ -105,7 +107,7 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution(r } // Validate the callback to be invoked and its parameters. - if err := rv.cbValidator.Validate(req.GetNamespace(), []*commonpb.Callback{req.Callback}); err != nil { + if err := rv.cbValidator.Validate(ctx, req.GetNamespace(), []*commonpb.Callback{req.Callback}); err != nil { return err } diff --git a/chasm/lib/callback/validator_test.go b/chasm/lib/callback/validator_test.go index 664ea331e30..4abfc3b7472 100644 --- a/chasm/lib/callback/validator_test.go +++ b/chasm/lib/callback/validator_test.go @@ -37,6 +37,7 @@ func TestValidateCallbacks(t *testing.T) { }) t.Run("NoURL", func(t *testing.T) { + ctx := context.Background() cbs := []*commonpb.Callback{ {Variant: &commonpb.Callback_Nexus_{ Nexus: &commonpb.Callback_Nexus{ @@ -44,7 +45,7 @@ func TestValidateCallbacks(t *testing.T) { }, }}, } - err := v.Validate("ns", cbs) + err := v.Validate(ctx, "ns", cbs) require.ErrorContains(t, err, "invalid callback url: not set") }) diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index 88a3f8970e6..a6e42db563d 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -1227,7 +1227,7 @@ func (s *StandaloneCallbackSuite) TestScheduleToCloseTimeout() { }) s.NoError(err) s.Equal(enumspb.CALLBACK_EXECUTION_STATUS_FAILED, descResp.GetInfo().GetStatus()) - s.Equal(enumspb.CALLBACK_STATE_FAILED, descResp.GetInfo().GetState()) + s.Equal(enumspb.CALLBACK_STATE_TIMED_OUT, descResp.GetInfo().GetState()) s.NotNil(descResp.GetInfo().GetCloseTime()) s.NotNil(descResp.GetOutcome().GetFailure()) s.NotNil(descResp.GetOutcome().GetFailure().GetTimeoutFailureInfo()) From b078bb98a8f29f9a9d62cdff89847230f4d6dea2 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 14 May 2026 13:26:47 -0700 Subject: [PATCH 29/52] Back out commit 0f8bbaa --- chasm/lib/callback/component.go | 12 ------------ chasm/lib/callback/invocable_outbound.go | 10 ++++++++++ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 7e445275bdd..576db74ca9b 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -178,18 +178,6 @@ func (c *Callback) loadInvocationArgs( ) } - // Setup the completion's headers. - completion.Header = callback.Header - // TODO(chrsmith): - // > Is this a behavior change for workflow callbacks? - // Yes? And I'm not sure if that's a good thing or not... - if callback.GetToken() != "" { - if completion.Header == nil { - completion.Header = nexus.Header{} - } - completion.Header.Set(commonnexus.CallbackTokenHeader, callback.GetToken()) - } - if callback.Url == chasm.NexusCompletionHandlerURL { return invocableInternal{ callback: callback, diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 971d9f268aa..ccac789e159 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -82,6 +82,16 @@ func (n invocableOutbound) Invoke( }) // Make the call and record metrics. startTime := time.Now() + n.completion.Header = n.callback.Header + + // If the outbound call is to a standalone callback, then supply the Nexus + // operation's token in the request. + if n.callback.GetToken() != "" { + if n.completion.Header == nil { + n.completion.Header = nexus.Header{} + } + n.completion.Header.Set(commonnexus.CallbackTokenHeader, n.callback.GetToken()) + } err := client.CompleteOperation(ctx, n.callback.Url, n.completion) From 6c2c16a7625318b20fdd76554d8824199fab552c Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 14 May 2026 13:30:45 -0700 Subject: [PATCH 30/52] Clean up some TODOs no longer applicable --- chasm/lib/callback/component.go | 1 - chasm/lib/callback/invocable_internal.go | 3 --- 2 files changed, 4 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 576db74ca9b..97fc37af224 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -95,7 +95,6 @@ func (c *Callback) LifecycleState(_ chasm.Context) chasm.LifecycleState { case callbackspb.CALLBACK_STATUS_FAILED, callbackspb.CALLBACK_STATUS_TERMINATED, callbackspb.CALLBACK_STATUS_TIMED_OUT: - // TODO(chrsmith): Confirm the response from #crew-chasm return chasm.LifecycleStateFailed default: return chasm.LifecycleStateRunning diff --git a/chasm/lib/callback/invocable_internal.go b/chasm/lib/callback/invocable_internal.go index bfc1350d05b..75f465236ae 100644 --- a/chasm/lib/callback/invocable_internal.go +++ b/chasm/lib/callback/invocable_internal.go @@ -58,9 +58,6 @@ func (c invocableInternal) Invoke( task *callbackspb.InvocationTask, taskAttr chasm.TaskAttributes, ) invocationResult { - // TODO(chrsmith): Both here and in invocable_outbound.go. - // > we should validate that temporal:// URLs have a token either via the old header format or the new structured Token field. - // Get the token from the dedicated Token field, falling back to the header for backwards compat. encodedRef := c.callback.GetToken() if encodedRef == "" { From 2a7fce0fe024b4b6e274dc5df960a0ebc0d57a68 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 14 May 2026 13:38:59 -0700 Subject: [PATCH 31/52] Use standard way to set completion's Nexus token --- chasm/lib/callback/invocable_outbound.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index ccac789e159..ab8ea6e5c78 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -85,12 +85,9 @@ func (n invocableOutbound) Invoke( n.completion.Header = n.callback.Header // If the outbound call is to a standalone callback, then supply the Nexus - // operation's token in the request. + // operation's token in the request. (Will make its way to the outbound HTTP headers.) if n.callback.GetToken() != "" { - if n.completion.Header == nil { - n.completion.Header = nexus.Header{} - } - n.completion.Header.Set(commonnexus.CallbackTokenHeader, n.callback.GetToken()) + n.completion.OperationToken = n.callback.GetToken() } err := client.CompleteOperation(ctx, n.callback.Url, n.completion) From a17e9ad0aae039f7ca0eae6cbabfd66f5a0769c2 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 14 May 2026 14:01:37 -0700 Subject: [PATCH 32/52] Address lint warnings --- chasm/lib/callback/component.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 97fc37af224..6e74ec6be44 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -313,7 +313,7 @@ func (c *Callback) GetNexusCompletion(ctx chasm.Context, requestID string) (nexu // For standalone completions, get the user-supplied value and convert it into the Nexus API type. suppliedCompletion, ok := c.SuppliedCompletion.TryGet(ctx) if !ok { - panic("no completion available") + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInvalidArgument("no completion result provided") } return callbackCompletionToNexusCompleteOperationOpts(c, suppliedCompletion) } @@ -378,12 +378,11 @@ func (c *Callback) statusAsAPIExecutionStatus() enumspb.CallbackExecutionStatus callbackspb.CALLBACK_STATUS_SCHEDULED, callbackspb.CALLBACK_STATUS_BACKING_OFF: return enumspb.CALLBACK_EXECUTION_STATUS_RUNNING - case callbackspb.CALLBACK_STATUS_FAILED: + case callbackspb.CALLBACK_STATUS_FAILED, + callbackspb.CALLBACK_STATUS_TIMED_OUT: return enumspb.CALLBACK_EXECUTION_STATUS_FAILED case callbackspb.CALLBACK_STATUS_SUCCEEDED: return enumspb.CALLBACK_EXECUTION_STATUS_SUCCEEDED - case callbackspb.CALLBACK_STATUS_TIMED_OUT: - return enumspb.CALLBACK_EXECUTION_STATUS_FAILED case callbackspb.CALLBACK_STATUS_TERMINATED: return enumspb.CALLBACK_EXECUTION_STATUS_TERMINATED default: From 9a0ee41a2a96c00e649d2ca4f0a8420e5e9a444b Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 15 May 2026 09:46:13 -0700 Subject: [PATCH 33/52] Address PR feedback from /review --- chasm/lib/activity/activity.go | 2 +- chasm/lib/callback/component.go | 18 +++++------------- chasm/lib/callback/frontend_validation.go | 2 +- chasm/lib/callback/handler.go | 10 ++++------ chasm/lib/callback/invocable_outbound.go | 5 +++-- chasm/lib/workflow/workflow.go | 2 +- common/nexus/nexusrpc/completion.go | 5 +++-- service/frontend/configs/quotas.go | 2 +- 8 files changed, 19 insertions(+), 27 deletions(-) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index a8d08c282fc..0a77b016f4f 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -337,7 +337,7 @@ func (a *Activity) addCompletionCallbacks( // requestID (unique per API call) + idx (position within the request) ensures unique, idempotent callback IDs. id := fmt.Sprintf("%s-%d", requestID, idx) - callbackObj := callback.NewEmbeddedCallback(ctx, requestID, registrationTime, chasmCB) + callbackObj := callback.NewEmbeddedCallback(requestID, registrationTime, chasmCB) a.Callbacks[id] = chasm.NewComponentField(ctx, callbackObj) } return nil diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 6e74ec6be44..b47809e7a52 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -16,7 +16,6 @@ import ( commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" queueserrors "go.temporal.io/server/service/history/queues/errors" - "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -63,7 +62,6 @@ type Callback struct { // NewEmbeddedCallback returns a Callback component, which will deliver the completion from // its parent CHASM component. The parent must implement CompletionSource. func NewEmbeddedCallback( - ctx chasm.MutableContext, requestID string, registrationTime *timestamppb.Timestamp, cb *callbackspb.Callback, @@ -78,16 +76,6 @@ func NewEmbeddedCallback( } } -type newStandaloneCallbackOpts struct { - RequestID string - RegistrationTime *timestamppb.Timestamp - Callback *callbackspb.Callback - - ScheduleToCloseTimeout *durationpb.Duration - Completion *callbackpb.CallbackExecutionCompletion - SearchAttributes map[string]*commonpb.Payload -} - func (c *Callback) LifecycleState(_ chasm.Context) chasm.LifecycleState { switch c.Status { case callbackspb.CALLBACK_STATUS_SUCCEEDED: @@ -145,7 +133,11 @@ func (c *Callback) Terminate( } return chasm.TerminateComponentResponse{}, nil } - if err := TransitionTerminated.Apply(c, ctx, EventTerminated{Reason: req.Reason}); err != nil { + event := EventTerminated{ + Identity: req.Identity, + Reason: req.Reason, + } + if err := TransitionTerminated.Apply(c, ctx, event); err != nil { return chasm.TerminateComponentResponse{}, fmt.Errorf("failed to terminate callback: %w", err) } diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index ca913560514..4c8dbef52de 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -107,7 +107,7 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution( } // Validate the callback to be invoked and its parameters. - if err := rv.cbValidator.Validate(ctx, req.GetNamespace(), []*commonpb.Callback{req.Callback}); err != nil { + if err := rv.cbValidator.Validate(ctx, req.GetNamespace(), []*commonpb.Callback{req.GetCallback()}); err != nil { return err } diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go index 9e995351e39..af5c7e899ca 100644 --- a/chasm/lib/callback/handler.go +++ b/chasm/lib/callback/handler.go @@ -142,16 +142,14 @@ func (h *callbackHandler) DescribeCallbackExecution( func( c *Callback, ctx chasm.Context, - req *callbackspb.DescribeCallbackExecutionRequest) (*callbackspb.DescribeCallbackExecutionResponse, error) { + _ *callbackspb.DescribeCallbackExecutionRequest) (*callbackspb.DescribeCallbackExecutionResponse, error) { return buildDescriptionProto(ctx, c) }, req) } // Below, we send an empty non-error response on context deadline expiry. Here we compute a - // deadline that causes us to send that response before the caller's own deadline (see - // chasm.activity.longPollBuffer). We also cap the caller's deadline at - // chasm.activity.longPollTimeout. + // deadline that causes us to send that response before the caller's own deadline. targetNamespace := req.GetFrontendRequest().GetNamespace() ctx, cancel := contextutil.WithDeadlineBuffer( ctx, @@ -242,7 +240,6 @@ func (h *callbackHandler) TerminateCallbackExecution( ctx context.Context, req *callbackspb.TerminateCallbackExecutionRequest, ) (resp *callbackspb.TerminateCallbackExecutionResponse, err error) { - resp, _, err = chasm.UpdateComponent( ctx, chasm.NewComponentRef[*Callback]( @@ -254,6 +251,7 @@ func (h *callbackHandler) TerminateCallbackExecution( ), func(c *Callback, ctx chasm.MutableContext, _ *callbackspb.TerminateCallbackExecutionRequest) (*callbackspb.TerminateCallbackExecutionResponse, error) { if _, err := c.Terminate(ctx, chasm.TerminateComponentRequest{ + Identity: req.FrontendRequest.GetIdentity(), Reason: req.FrontendRequest.GetReason(), RequestID: req.FrontendRequest.GetRequestId(), }); err != nil { @@ -312,7 +310,7 @@ func createStandaloneCallback( now := timestamppb.Now() // Create Callback component. - cb := NewEmbeddedCallback(ctx, input.RequestID, now, input.Callback) + cb := NewEmbeddedCallback(input.RequestID, now, input.Callback) cb.ScheduleToCloseTimeout = input.ScheduleToCloseTimeout cb.SuppliedCompletion = chasm.NewDataField(ctx, input.Completion) diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index ab8ea6e5c78..42cb954a9e6 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -84,9 +84,10 @@ func (n invocableOutbound) Invoke( startTime := time.Now() n.completion.Header = n.callback.Header - // If the outbound call is to a standalone callback, then supply the Nexus - // operation's token in the request. (Will make its way to the outbound HTTP headers.) + // If the token is set on the callback, pass it along in completion's headers. if n.callback.GetToken() != "" { + // TODO(chrsmith): This is a bug. OperationToken is wrong, we need to add a new field + // and wire it to a _different_ header. My mistake. n.completion.OperationToken = n.callback.GetToken() } diff --git a/chasm/lib/workflow/workflow.go b/chasm/lib/workflow/workflow.go index b79d244f926..178b0843fb0 100644 --- a/chasm/lib/workflow/workflow.go +++ b/chasm/lib/workflow/workflow.go @@ -110,7 +110,7 @@ func (w *Workflow) AddCompletionCallbacks( id := fmt.Sprintf("%s-%d", requestID, idx) // Create and add callback - callbackObj := callback.NewEmbeddedCallback(ctx, requestID, eventTime, chasmCB) + callbackObj := callback.NewEmbeddedCallback(requestID, eventTime, chasmCB) w.Callbacks[id] = chasm.NewComponentField(ctx, callbackObj) } return nil diff --git a/common/nexus/nexusrpc/completion.go b/common/nexus/nexusrpc/completion.go index 1019283f147..72f93bfb71b 100644 --- a/common/nexus/nexusrpc/completion.go +++ b/common/nexus/nexusrpc/completion.go @@ -92,8 +92,9 @@ type CompleteOperationOptions struct { // Header to send in the completion request. // Note that this is a Nexus header, not an HTTP header. Header nexus.Header - // OperationToken is the unique token for this operation. Used when a completion callback is received before a - // started response. + // OperationToken is the unique token for this operation. This is returned by the Nexus handler from the call to + // StartOperation. It has no meaning outside the handler, and is used to identify the operation within the handler system. + // and is handler-specific.Used when a completion callback is received before a started response. OperationToken string // StartTime is the time the operation started. Used when a completion callback is received before a started response. StartTime time.Time diff --git a/service/frontend/configs/quotas.go b/service/frontend/configs/quotas.go index 33279617549..348245ee590 100644 --- a/service/frontend/configs/quotas.go +++ b/service/frontend/configs/quotas.go @@ -47,7 +47,7 @@ var ( "/temporal.api.workflowservice.v1.WorkflowService/PollNexusTaskQueue": 1, "/temporal.api.workflowservice.v1.WorkflowService/PollNexusOperationExecution": 1, - // Long-running if the resources isn't in a terminal state. + // Long-running if the resource isn't in a terminal state. "/temporal.api.workflowservice.v1.WorkflowService/PollActivityExecution": 1, "/temporal.api.workflowservice.v1.WorkflowService/PollCallbackExecution": 1, From 0dc272dd63dcbec9c57e98339457622c7c2105eb Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Fri, 15 May 2026 10:04:25 -0700 Subject: [PATCH 34/52] Fix bug in setting wrong headers for resolving Nexus op callbacks --- chasm/lib/callback/invocable_outbound.go | 6 ++---- common/nexus/nexusrpc/completion.go | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 42cb954a9e6..e175d96ce0b 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -84,11 +84,9 @@ func (n invocableOutbound) Invoke( startTime := time.Now() n.completion.Header = n.callback.Header - // If the token is set on the callback, pass it along in completion's headers. + // If the Nexus callback token is set on the callback, pass it along in completion's headers. if n.callback.GetToken() != "" { - // TODO(chrsmith): This is a bug. OperationToken is wrong, we need to add a new field - // and wire it to a _different_ header. My mistake. - n.completion.OperationToken = n.callback.GetToken() + n.completion.CallbackToken = n.callback.GetToken() } err := client.CompleteOperation(ctx, n.callback.Url, n.completion) diff --git a/common/nexus/nexusrpc/completion.go b/common/nexus/nexusrpc/completion.go index 72f93bfb71b..e7de8353873 100644 --- a/common/nexus/nexusrpc/completion.go +++ b/common/nexus/nexusrpc/completion.go @@ -12,6 +12,12 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" ) +const ( + // Copy of common/nexus.CallbackTokenHeader, to avoid import cycle. + // Header to identify the callback being resolved for callbacks to resolve Nexus operations. + commonnexusCallbackTokenHeader = "Temporal-Callback-Token" +) + // CompletionHTTPClient is a client for sending Nexus operation completion callbacks via HTTP. type CompletionHTTPClient struct { baseHTTPClient @@ -92,10 +98,14 @@ type CompleteOperationOptions struct { // Header to send in the completion request. // Note that this is a Nexus header, not an HTTP header. Header nexus.Header - // OperationToken is the unique token for this operation. This is returned by the Nexus handler from the call to - // StartOperation. It has no meaning outside the handler, and is used to identify the operation within the handler system. - // and is handler-specific.Used when a completion callback is received before a started response. + // OperationToken is the unique token for this operation. Used when a completion callback is received before a + // started response. (Otherwise, the operation token received from the handler's response to StartOperation will + // be used instead.) OperationToken string + // CallbackToken is the unique token for the Nexus operation being completed. It is passed to the handler via + // the CallbackTokenHeader, but is now an explicit field on the Callback proto. Should be set if the completion + // operation is a Nexus callback. (See nexuscommon.CallbackTokenHeader.) + CallbackToken string // StartTime is the time the operation started. Used when a completion callback is received before a started response. StartTime time.Time // CloseTime is the time the operation completed. This may be different from the time the completion callback is delivered. @@ -167,6 +177,9 @@ func (c CompleteOperationOptions) applyToHTTPRequest(cc *CompletionHTTPClient, r if c.Header.Get(nexus.HeaderOperationToken) == "" && c.OperationToken != "" { request.Header.Set(nexus.HeaderOperationToken, c.OperationToken) } + if c.Header.Get(commonnexusCallbackTokenHeader) == "" && c.CallbackToken != "" { + request.Header.Set(commonnexusCallbackTokenHeader, c.CallbackToken) + } if c.Header.Get(headerOperationStartTime) == "" && !c.StartTime.IsZero() { request.Header.Set(headerOperationStartTime, c.StartTime.Format(http.TimeFormat)) } From 6a8b8ce7b35cc372809225894685bb70bc1b4881 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 16 May 2026 10:13:43 -0700 Subject: [PATCH 35/52] Use new LongPoll* constants --- chasm/lib/callback/config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 39e54eecfd1..17b1f0ef64f 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -7,6 +7,7 @@ import ( "time" "go.temporal.io/server/chasm" + "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/nexus" @@ -22,14 +23,14 @@ var EnableStandaloneExecutions = dynamicconfig.NewNamespaceBoolSetting( var LongPollBuffer = dynamicconfig.NewNamespaceDurationSetting( "callback.longPollBuffer", - time.Second, + common.DefaultLongPollBuffer, `A buffer used to adjust the callback execution long-poll timeouts. The long-poll response is sent before the caller's deadline by this amount of time.`, ) var LongPollTimeout = dynamicconfig.NewNamespaceDurationSetting( "callback.longPollTimeout", - 20*time.Second, + common.DefaultLongPollTimeout, `Timeout for callback execution long-poll requests.`, ) From 8e73d57a3f5418ac9339670333ae6b58a4c52fea Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 16 May 2026 10:33:56 -0700 Subject: [PATCH 36/52] Address minor PR feedback --- chasm/lib/callback/component.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index b47809e7a52..a578bd8a32a 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -97,10 +97,11 @@ func (c *Callback) SetStateMachineState(status callbackspb.CallbackStatus) { c.Status = status } +// ContextMetadata is used for root CHASM components, so this is only applicable +// for the standalone callback case. func (c *Callback) ContextMetadata(ctx chasm.Context) map[string]string { return map[string]string{ - "request-id": c.RequestId, - // Only set for standalone callbacks. + "request-id": c.RequestId, "callback-id": ctx.ExecutionKey().BusinessID, } } @@ -291,7 +292,7 @@ func callbackCompletionToNexusCompleteOperationOpts( return nexusCompletion, nil default: - return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInvalidArgument("no completion result provided") + return nexusrpc.CompleteOperationOptions{}, serviceerror.NewInternal("no completion result provided") } } From 99901042182d71de3990f3a074c95b89c43c6dd4 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 16 May 2026 10:56:21 -0700 Subject: [PATCH 37/52] Only set TerminalFailure for external errors --- chasm/lib/callback/component.go | 14 ++++++++--- chasm/lib/callback/statemachine.go | 4 +--- chasm/lib/callback/statemachine_test.go | 32 ++++++++++++------------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index a578bd8a32a..0576bfd3c5d 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -350,9 +350,17 @@ func (c *Callback) outcome(ctx chasm.Context) *callbackpb.CallbackExecutionOutco Value: val, } - case callbackspb.CALLBACK_STATUS_FAILED, - callbackspb.CALLBACK_STATUS_TIMED_OUT, - callbackspb.CALLBACK_STATUS_TERMINATED: + case callbackspb.CALLBACK_STATUS_FAILED: + // Organic failures leave TerminalFailure nil, and just set LastAttemptFailure. + val := &callbackpb.CallbackExecutionOutcome_Failure{ + Failure: c.LastAttemptFailure, + } + return &callbackpb.CallbackExecutionOutcome{ + Value: val, + } + + case callbackspb.CALLBACK_STATUS_TERMINATED, + callbackspb.CALLBACK_STATUS_TIMED_OUT: val := &callbackpb.CallbackExecutionOutcome_Failure{ Failure: c.TerminalFailure.Get(ctx), } diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 4541c5181b1..528cefa9eb9 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -101,8 +101,6 @@ var TransitionFailed = chasm.NewTransition( cb.recordAttempt(now) cb.CloseTime = timestamppb.New(now) - // Set the TerminalFailure, but we intentionally leave LastAttemptFailure - // as-is for debugability. failure := &failurepb.Failure{ Message: event.Err.Error(), FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ @@ -111,7 +109,7 @@ var TransitionFailed = chasm.NewTransition( }, }, } - cb.TerminalFailure = chasm.NewDataField(ctx, failure) + cb.LastAttemptFailure = failure return nil }, diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index 45565fee882..f96a8c801d0 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -49,10 +49,9 @@ func TestValidTransitions(t *testing.T) { dt := currentTime.Add(time.Second).Sub(callback.NextAttemptScheduleTime.AsTime()) require.Less(t, dt, time.Millisecond*200) - // Because of the retry policy, the first failure isn't terminal. require.Nil(t, callback.CloseTime) - _, hasTermFailure := callback.TerminalFailure.TryGet(mctx) - require.False(t, hasTermFailure) + _, ok := callback.TerminalFailure.TryGet(mctx) + require.False(t, ok) // Assert backoff task is generated require.Len(t, mctx.Tasks, 1) @@ -74,8 +73,8 @@ func TestValidTransitions(t *testing.T) { require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) require.Nil(t, callback.CloseTime) require.Nil(t, callback.NextAttemptScheduleTime) - _, hasTermFailure := callback.TerminalFailure.TryGet(mctx) - require.False(t, hasTermFailure) + _, ok := callback.TerminalFailure.TryGet(mctx) + require.False(t, ok) // Assert callback task is generated. require.Len(t, mctx.Tasks, 1) @@ -104,9 +103,8 @@ func TestValidTransitions(t *testing.T) { require.Nil(t, callback.NextAttemptScheduleTime) require.Equal(t, currentTime, callback.CloseTime.AsTime()) - // TerminalFailure may explicitly be set to nil. - termFailureValue, hasTermFailure := callback.TerminalFailure.TryGet(mctx) - require.True(t, !hasTermFailure || termFailureValue == nil) + _, ok := callback.TerminalFailure.TryGet(mctx) + require.False(t, ok) // Assert no task is generated on success transition require.Empty(t, mctx.Tasks) @@ -131,16 +129,15 @@ func TestValidTransitions(t *testing.T) { require.Nil(t, callback.NextAttemptScheduleTime) require.Equal(t, currentTime, callback.CloseTime.AsTime()) - // Assert LastAttemptFailure is unchanged. + // Assert LastAttemptFailure is the latest, and final failure. lastFailure := callback.LastAttemptFailure - require.Equal(t, "error message", lastFailure.Message) - require.False(t, lastFailure.GetApplicationFailureInfo().NonRetryable) + require.Equal(t, "new failure msg", lastFailure.Message) + require.True(t, lastFailure.GetApplicationFailureInfo().NonRetryable) - // Assert the TerminalFailure field is set to the final error. - require.NotNil(t, callback.TerminalFailure, "TerminalFailure not set") - termFailure := callback.TerminalFailure.Get(mctx) - require.Equal(t, "new failure msg", termFailure.Message) - require.True(t, termFailure.GetApplicationFailureInfo().NonRetryable) + // Assert the TerminalFailure field is not set. + // (It will only have a value on externa failures, like timeouts.) + _, ok := callback.TerminalFailure.TryGet(mctx) + require.False(t, ok) // Assert no tasks generated. In terminal state. require.Empty(t, mctx.Tasks) @@ -149,6 +146,7 @@ func TestValidTransitions(t *testing.T) { func TestTerminatedTransition(t *testing.T) { initialCallbackState := &callbackspb.CallbackState{ + Status: callbackspb.CALLBACK_STATUS_SCHEDULED, RegistrationTime: timestamppb.New(time.Now()), Callback: &callbackspb.Callback{ Variant: &callbackspb.Callback_Nexus_{ @@ -165,6 +163,7 @@ func TestTerminatedTransition(t *testing.T) { emptyTerminateEvent := EventTerminated{} assertEmptyEventResults := func(t *testing.T, mctx *chasm.MockMutableContext, cb *Callback) { // Confirm default reason, but no additional metadata. + require.Equal(t, callbackspb.CALLBACK_STATUS_TERMINATED, cb.GetStatus()) termFailure := cb.TerminalFailure.Get(mctx) require.Equal(t, "callback execution terminated", termFailure.Message) require.Nil(t, termFailure.GetTerminatedFailureInfo()) @@ -176,6 +175,7 @@ func TestTerminatedTransition(t *testing.T) { } assertPopulatedEventResults := func(t *testing.T, mctx *chasm.MockMutableContext, cb *Callback) { // Confirm user-supplied reason and identity are available. + require.Equal(t, callbackspb.CALLBACK_STATUS_TERMINATED, cb.GetStatus()) termFailure := cb.TerminalFailure.Get(mctx) require.Equal(t, "user-supplied reason", termFailure.Message) gotTermFailureInfo := termFailure.GetTerminatedFailureInfo() From 2fb8a64d3304687d73be3d3e5fec1b55342419ed Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 16 May 2026 11:15:33 -0700 Subject: [PATCH 38/52] Do not clear LastAttemptFailure on successful Callback --- chasm/lib/callback/statemachine.go | 2 +- chasm/lib/callback/statemachine_test.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 528cefa9eb9..74365f5066b 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -127,7 +127,7 @@ var TransitionSucceeded = chasm.NewTransition( now := ctx.Now(cb) cb.recordAttempt(now) cb.CloseTime = timestamppb.New(now) - cb.LastAttemptFailure = nil + // NOTE: We don't clear LastAttemptFailure data intentionally. return nil }, ) diff --git a/chasm/lib/callback/statemachine_test.go b/chasm/lib/callback/statemachine_test.go index f96a8c801d0..64a7ff6fcc0 100644 --- a/chasm/lib/callback/statemachine_test.go +++ b/chasm/lib/callback/statemachine_test.go @@ -98,11 +98,16 @@ func TestValidTransitions(t *testing.T) { // Assert info object is updated. require.Equal(t, callbackspb.CALLBACK_STATUS_SUCCEEDED, callback.StateMachineState()) require.Equal(t, int32(2), callback.Attempt) - require.Nil(t, callback.LastAttemptFailure) require.Equal(t, currentTime, callback.LastAttemptCompleteTime.AsTime()) require.Nil(t, callback.NextAttemptScheduleTime) require.Equal(t, currentTime, callback.CloseTime.AsTime()) + // The LastAttemptFailure and related data remain unchanged. + lastFailure := callback.LastAttemptFailure + require.NotNil(t, lastFailure) + require.Equal(t, "error message", lastFailure.Message) + require.False(t, lastFailure.GetApplicationFailureInfo().NonRetryable) + _, ok := callback.TerminalFailure.TryGet(mctx) require.False(t, ok) From bf3954ecbf907c099f3f45ed45f9dfbe95893fca Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Sat, 16 May 2026 11:26:38 -0700 Subject: [PATCH 39/52] Backout the Temporal NexusCallbackToken from nexusrpc --- chasm/lib/callback/invocable_outbound.go | 5 ++++- common/nexus/nexusrpc/completion.go | 16 +--------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index e175d96ce0b..aa5ce021e73 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -86,7 +86,10 @@ func (n invocableOutbound) Invoke( // If the Nexus callback token is set on the callback, pass it along in completion's headers. if n.callback.GetToken() != "" { - n.completion.CallbackToken = n.callback.GetToken() + if n.completion.Header == nil { + n.completion.Header = nexus.Header{} + } + n.completion.Header.Set(commonnexus.CallbackTokenHeader, n.callback.GetToken()) } err := client.CompleteOperation(ctx, n.callback.Url, n.completion) diff --git a/common/nexus/nexusrpc/completion.go b/common/nexus/nexusrpc/completion.go index e7de8353873..1019283f147 100644 --- a/common/nexus/nexusrpc/completion.go +++ b/common/nexus/nexusrpc/completion.go @@ -12,12 +12,6 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" ) -const ( - // Copy of common/nexus.CallbackTokenHeader, to avoid import cycle. - // Header to identify the callback being resolved for callbacks to resolve Nexus operations. - commonnexusCallbackTokenHeader = "Temporal-Callback-Token" -) - // CompletionHTTPClient is a client for sending Nexus operation completion callbacks via HTTP. type CompletionHTTPClient struct { baseHTTPClient @@ -99,13 +93,8 @@ type CompleteOperationOptions struct { // Note that this is a Nexus header, not an HTTP header. Header nexus.Header // OperationToken is the unique token for this operation. Used when a completion callback is received before a - // started response. (Otherwise, the operation token received from the handler's response to StartOperation will - // be used instead.) + // started response. OperationToken string - // CallbackToken is the unique token for the Nexus operation being completed. It is passed to the handler via - // the CallbackTokenHeader, but is now an explicit field on the Callback proto. Should be set if the completion - // operation is a Nexus callback. (See nexuscommon.CallbackTokenHeader.) - CallbackToken string // StartTime is the time the operation started. Used when a completion callback is received before a started response. StartTime time.Time // CloseTime is the time the operation completed. This may be different from the time the completion callback is delivered. @@ -177,9 +166,6 @@ func (c CompleteOperationOptions) applyToHTTPRequest(cc *CompletionHTTPClient, r if c.Header.Get(nexus.HeaderOperationToken) == "" && c.OperationToken != "" { request.Header.Set(nexus.HeaderOperationToken, c.OperationToken) } - if c.Header.Get(commonnexusCallbackTokenHeader) == "" && c.CallbackToken != "" { - request.Header.Set(commonnexusCallbackTokenHeader, c.CallbackToken) - } if c.Header.Get(headerOperationStartTime) == "" && !c.StartTime.IsZero() { request.Header.Set(headerOperationStartTime, c.StartTime.Format(http.TimeFormat)) } From 908b9c7a4b137d93f84c5c43293e0f47507546b2 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 09:32:16 -0700 Subject: [PATCH 40/52] Address PR feedback --- chasm/lib/callback/component.go | 46 ++++++++++++++++ chasm/lib/callback/handler.go | 94 ++++++++------------------------- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 0576bfd3c5d..524e09005b7 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -16,6 +16,7 @@ import ( commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" queueserrors "go.temporal.io/server/service/history/queues/errors" + "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -76,6 +77,51 @@ func NewEmbeddedCallback( } } +// newStandaloneCallbackInput is the bundle of inputs to the CHASM execution. +type newStandaloneCallbackInput struct { + RequestID string + Callback *callbackspb.Callback + ScheduleToCloseTimeout *durationpb.Duration + Completion *callbackpb.CallbackExecutionCompletion + SearchAttributes map[string]*commonpb.Payload +} + +// newStandaloneCallback constructs a new Callback component in standalone mode. +// The Callback is immediately transitioned to SCHEDULED state to begin invocation. +func newStandaloneCallback( + ctx chasm.MutableContext, + input *newStandaloneCallbackInput, +) (*Callback, error) { + now := timestamppb.Now() + + // Create Callback component. + cb := NewEmbeddedCallback(input.RequestID, now, input.Callback) + cb.ScheduleToCloseTimeout = input.ScheduleToCloseTimeout + cb.SuppliedCompletion = chasm.NewDataField(ctx, input.Completion) + + visibility := chasm.NewVisibilityWithData(ctx, input.SearchAttributes, nil) + cb.Visibility = chasm.NewComponentField(ctx, visibility) + + // Immediately schedule the callback for invocation. + if err := TransitionScheduled.Apply(cb, ctx, EventScheduled{}); err != nil { + return nil, fmt.Errorf("failed to schedule callback: %w", err) + } + + // Schedule the timeout as applicable. + if durationProto := input.ScheduleToCloseTimeout; durationProto != nil { + if duration := durationProto.AsDuration(); duration > 0 { + timeoutTime := now.AsTime().Add(duration) + ctx.AddTask( + cb, + chasm.TaskAttributes{ScheduledTime: timeoutTime}, + &callbackspb.ScheduleToCloseTimeoutTask{}, + ) + } + } + + return cb, nil +} + func (c *Callback) LifecycleState(_ chasm.Context) chasm.LifecycleState { switch c.Status { case callbackspb.CALLBACK_STATUS_SUCCEEDED: diff --git a/chasm/lib/callback/handler.go b/chasm/lib/callback/handler.go index af5c7e899ca..8e1c7ee0b66 100644 --- a/chasm/lib/callback/handler.go +++ b/chasm/lib/callback/handler.go @@ -3,17 +3,12 @@ package callback import ( "context" "errors" - "fmt" - callbackpb "go.temporal.io/api/callback/v1" - commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/contextutil" - "google.golang.org/protobuf/types/known/durationpb" - "google.golang.org/protobuf/types/known/timestamppb" ) type callbackHandler struct { @@ -32,10 +27,10 @@ func (h *callbackHandler) StartCallbackExecution( ctx context.Context, req *callbackspb.StartCallbackExecutionRequest, ) (resp *callbackspb.StartCallbackExecutionResponse, err error) { - frontendReq := req.FrontendRequest + frontendReq := req.GetFrontendRequest() // Gather all the data necessary to create the Callback component. - input := &createStandaloneCallbackInput{ + input := &newStandaloneCallbackInput{ RequestID: frontendReq.GetRequestId(), ScheduleToCloseTimeout: frontendReq.GetScheduleToCloseTimeout(), Completion: frontendReq.GetCompletion(), @@ -59,10 +54,10 @@ func (h *callbackHandler) StartCallbackExecution( result, err := chasm.StartExecution( ctx, chasm.ExecutionKey{ - NamespaceID: req.NamespaceId, + NamespaceID: req.GetNamespaceId(), BusinessID: frontendReq.GetCallbackId(), }, - createStandaloneCallback, + newStandaloneCallback, input, chasm.WithRequestID(frontendReq.GetRequestId()), // Relying on these default policies. No configuration knobs are exposed to users. @@ -112,10 +107,10 @@ func (h *callbackHandler) DescribeCallbackExecution( Info: info, } - if req.FrontendRequest.GetIncludeInput() { + if req.GetFrontendRequest().GetIncludeInput() { resp.Input = c.SuppliedCompletion.Get(ctx) } - if req.FrontendRequest.GetIncludeOutcome() { + if req.GetFrontendRequest().GetIncludeOutcome() { resp.Outcome = c.outcome(ctx) } @@ -127,8 +122,8 @@ func (h *callbackHandler) DescribeCallbackExecution( compRef := chasm.NewComponentRef[*Callback]( chasm.ExecutionKey{ NamespaceID: req.GetNamespaceId(), - BusinessID: req.FrontendRequest.GetCallbackId(), - RunID: req.FrontendRequest.GetRunId(), + BusinessID: req.GetFrontendRequest().GetCallbackId(), + RunID: req.GetFrontendRequest().GetRunId(), }, ) @@ -197,13 +192,13 @@ func (h *callbackHandler) PollCallbackExecution( ref := chasm.NewComponentRef[*Callback]( chasm.ExecutionKey{ - NamespaceID: req.NamespaceId, - BusinessID: req.FrontendRequest.GetCallbackId(), - RunID: req.FrontendRequest.GetRunId(), + NamespaceID: req.GetNamespaceId(), + BusinessID: req.GetFrontendRequest().GetCallbackId(), + RunID: req.GetFrontendRequest().GetRunId(), }, ) - ns := req.FrontendRequest.GetNamespace() + ns := req.GetFrontendRequest().GetNamespace() ctx, cancel := contextutil.WithDeadlineBuffer( ctx, h.config.LongPollTimeout(ns), @@ -240,20 +235,22 @@ func (h *callbackHandler) TerminateCallbackExecution( ctx context.Context, req *callbackspb.TerminateCallbackExecutionRequest, ) (resp *callbackspb.TerminateCallbackExecutionResponse, err error) { + + frontendReq := req.GetFrontendRequest() resp, _, err = chasm.UpdateComponent( ctx, chasm.NewComponentRef[*Callback]( chasm.ExecutionKey{ - NamespaceID: req.NamespaceId, - BusinessID: req.FrontendRequest.GetCallbackId(), - RunID: req.FrontendRequest.GetRunId(), + NamespaceID: req.GetNamespaceId(), + BusinessID: frontendReq.GetCallbackId(), + RunID: frontendReq.GetRunId(), }, ), func(c *Callback, ctx chasm.MutableContext, _ *callbackspb.TerminateCallbackExecutionRequest) (*callbackspb.TerminateCallbackExecutionResponse, error) { if _, err := c.Terminate(ctx, chasm.TerminateComponentRequest{ - Identity: req.FrontendRequest.GetIdentity(), - Reason: req.FrontendRequest.GetReason(), - RequestID: req.FrontendRequest.GetRequestId(), + Identity: frontendReq.GetIdentity(), + Reason: frontendReq.GetReason(), + RequestID: frontendReq.GetRequestId(), }); err != nil { return nil, err } @@ -274,9 +271,9 @@ func (h *callbackHandler) DeleteCallbackExecution( if err = chasm.DeleteExecution[*Callback]( ctx, chasm.ExecutionKey{ - NamespaceID: req.NamespaceId, - BusinessID: req.FrontendRequest.GetCallbackId(), - RunID: req.FrontendRequest.GetRunId(), + NamespaceID: req.GetNamespaceId(), + BusinessID: req.GetFrontendRequest().GetCallbackId(), + RunID: req.GetFrontendRequest().GetRunId(), }, chasm.DeleteExecutionRequest{ TerminateComponentRequest: chasm.TerminateComponentRequest{ @@ -291,48 +288,3 @@ func (h *callbackHandler) DeleteCallbackExecution( FrontendResponse: &workflowservice.DeleteCallbackExecutionResponse{}, }, nil } - -// createStandaloneCallbackInput is the bundle of inputs to the CHASM execution. -type createStandaloneCallbackInput struct { - RequestID string - Callback *callbackspb.Callback - ScheduleToCloseTimeout *durationpb.Duration - Completion *callbackpb.CallbackExecutionCompletion - SearchAttributes map[string]*commonpb.Payload -} - -// createStandaloneCallback constructs a new Callback component in standalone mode. -// The Callback is immediately transitioned to SCHEDULED state to begin invocation. -func createStandaloneCallback( - ctx chasm.MutableContext, - input *createStandaloneCallbackInput, -) (*Callback, error) { - now := timestamppb.Now() - - // Create Callback component. - cb := NewEmbeddedCallback(input.RequestID, now, input.Callback) - cb.ScheduleToCloseTimeout = input.ScheduleToCloseTimeout - cb.SuppliedCompletion = chasm.NewDataField(ctx, input.Completion) - - visibility := chasm.NewVisibilityWithData(ctx, input.SearchAttributes, nil) - cb.Visibility = chasm.NewComponentField(ctx, visibility) - - // Immediately schedule the callback for invocation. - if err := TransitionScheduled.Apply(cb, ctx, EventScheduled{}); err != nil { - return nil, fmt.Errorf("failed to schedule callback: %w", err) - } - - // Schedule the timeout as applicable. - if durationProto := input.ScheduleToCloseTimeout; durationProto != nil { - if duration := durationProto.AsDuration(); duration > 0 { - timeoutTime := now.AsTime().Add(duration) - ctx.AddTask( - cb, - chasm.TaskAttributes{ScheduledTime: timeoutTime}, - &callbackspb.ScheduleToCloseTimeoutTask{}, - ) - } - } - - return cb, nil -} From 47df7d7ba49315bb12e099a44d047fce1597a66c Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 09:44:17 -0700 Subject: [PATCH 41/52] Update go.mod to latest version of api-go SHA --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b2613a9bc7b..c5d1f8e073a 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 go.opentelemetry.io/otel/trace v1.43.0 - go.temporal.io/api v1.62.12-0.20260513212731-4fa5ab4b3909 // DO NOT SUBMIT -- Branch chrsmith/standalone-callbacks + go.temporal.io/api v1.62.13-0.20260518171253-8d40f7825b45 // DO NOT SUBMIT -- Branch chrsmith/standalone-callbacks go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 go.temporal.io/sdk v1.41.1 go.uber.org/fx v1.24.0 diff --git a/go.sum b/go.sum index e5f8d388383..9d878942856 100644 --- a/go.sum +++ b/go.sum @@ -471,8 +471,8 @@ go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:R go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= -go.temporal.io/api v1.62.12-0.20260513212731-4fa5ab4b3909 h1:1sHlmcA+G9aeneEr20SiG9bKpbWejwoKNfpNioiVx+g= -go.temporal.io/api v1.62.12-0.20260513212731-4fa5ab4b3909/go.mod h1:iqSd4FzEdRg8o0TIkhKIc5wIafoP/iix8q+zl5yN8oo= +go.temporal.io/api v1.62.13-0.20260518171253-8d40f7825b45 h1:7yvEzLNJjRcEVqpo35M9gUUywePfqhirhXMir8C1+PQ= +go.temporal.io/api v1.62.13-0.20260518171253-8d40f7825b45/go.mod h1:iqSd4FzEdRg8o0TIkhKIc5wIafoP/iix8q+zl5yN8oo= go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2 h1:1hKeH3GyR6YD6LKMHGCZ76t6h1Sgha0hXVQBxWi3dlQ= go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2/go.mod h1:T8dnzVPeO+gaUTj9eDgm/lT2lZH4+JXNvrGaQGyVi50= go.temporal.io/sdk v1.41.1 h1:yOpvsHyDD1lNuwlGBv/SUodCPhjv9nDeC9lLHW/fJUA= From 04033c1db83209801c49d03e7a399659484cdf0e Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 11:05:31 -0700 Subject: [PATCH 42/52] Use serviceerror.NexusOperationNotStarted --- chasm/lib/callback/component_test.go | 4 ++-- chasm/lib/callback/invocable_outbound.go | 6 +++--- components/nexusoperations/completion.go | 10 ++++------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/chasm/lib/callback/component_test.go b/chasm/lib/callback/component_test.go index 7cf38f19042..6e788f065ed 100644 --- a/chasm/lib/callback/component_test.go +++ b/chasm/lib/callback/component_test.go @@ -5,11 +5,11 @@ import ( "time" "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/backoff" commonnexus "go.temporal.io/server/common/nexus" - "go.temporal.io/server/components/nexusoperations" ) // Confirm that callback delivery failures due to the Nexus operation not having @@ -32,7 +32,7 @@ func TestCallbacksToUnstartedNexusOperations(t *testing.T) { // result being saved on the Callback. mctx := &chasm.MockMutableContext{} _, err := cb.saveResult(mctx, saveResultInput{ - result: invocationResultRetry{err: nexusoperations.ErrOperationNotStarted}, + result: invocationResultRetry{err: serviceerror.NewNexusOperationNotStarted("nexus operation not started", "req-id")}, retryPolicy: backoff.NewExponentialRetryPolicy(time.Second), }) diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index aa5ce021e73..0eed6bf9e57 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -7,6 +7,7 @@ import ( "time" "github.com/nexus-rpc/sdk-go/nexus" + "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/log" @@ -15,7 +16,6 @@ import ( "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" - "go.temporal.io/server/components/nexusoperations" queuescommon "go.temporal.io/server/service/history/queues/common" queueserrors "go.temporal.io/server/service/history/queues/errors" ) @@ -41,7 +41,7 @@ func (n invocableOutbound) WrapError(result invocationResult, err error) error { // If the error is due to a completion of a Nexus operation being delivered before the // operation has officially started, we want to avoid triggering the circuit breakers. // Since the actual destination is working fine, and the failure is due to a data race. - if errors.Is(err, nexusoperations.ErrOperationNotStarted) { + if errors.Is(err, &serviceerror.NexusOperationNotStarted{}) { return err } @@ -107,7 +107,7 @@ func (n invocableOutbound) Invoke( // If the error from trying to resolve a Nexus operation that hasn't yet been marked // as started, it is safe to retry. (n.WrapError will ensure repeated failures of this // kind won't cause the SystemCallback to trip the circuit breaker.) - isErrNotStarted := errors.Is(err, nexusoperations.ErrOperationNotStarted) + isErrNotStarted := errors.Is(err, &serviceerror.NexusOperationNotStarted{}) if n.isSystemCallback() && isErrNotStarted { retryable = true } diff --git a/components/nexusoperations/completion.go b/components/nexusoperations/completion.go index ec10497f7ca..94984231e3c 100644 --- a/components/nexusoperations/completion.go +++ b/components/nexusoperations/completion.go @@ -16,11 +16,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) -// ErrOperationNotStarted is returned when a completion arrives before the operation has -// started and no operation token is provided. This error is used by the callback invocation -// layer to detect this specific condition and retry without triggering the circuit breaker. -var ErrOperationNotStarted = serviceerror.NewFailedPrecondition("nexus operation not started") - func handleSuccessfulOperationResult( node *hsm.Node, operation Operation, @@ -145,8 +140,11 @@ func fabricateStartedEventIfMissing( // reject the request. This handles the race where a completion arrives before the start // handler returns with the operation token. The caller will retry and by then the start // handler will have returned and recorded the token. + // + // TODO(NEXUS-369): We should delay the completion somehow, e.g. using a chasm.PollComponent, + // to allow avoid surfacing any traisent errors to users. if operationToken == "" { - return ErrOperationNotStarted + return serviceerror.NewNexusOperationNotStarted("nexus operation not started", requestID) } eventID, err := hsm.EventIDFromToken(operation.ScheduledEventToken) From d5218c483d2ad92b835117a9ba90248f32d117f4 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 11:17:14 -0700 Subject: [PATCH 43/52] Migrate from EventuallyWithT to Await --- tests/standalone_callbacks_test.go | 84 +++++++++++++++--------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index a6e42db563d..f64f004c5d4 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -11,8 +11,6 @@ import ( "github.com/google/uuid" "github.com/nexus-rpc/sdk-go/nexus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -568,17 +566,17 @@ func (s *StandaloneCallbackSuite) TestPollCallbackExecution() { waitUpTo = 3 * time.Second checkInterval = 200 * time.Millisecond ) - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { pollResp, err := env.FrontendClient().PollCallbackExecution(ctx, &workflowservice.PollCallbackExecutionRequest{ Namespace: env.Namespace().String(), CallbackId: callbackID, }) - require.NoError(t, err) - require.NotNil(t, pollResp.GetOutcome()) + scs.NoError(err) + scs.NotNil(pollResp.GetOutcome()) - require.NotNil(t, pollResp.GetOutcome().GetFailure()) - require.NotNil(t, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo()) - require.Equal(t, enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) + scs.NotNil(pollResp.GetOutcome().GetFailure()) + scs.NotNil(pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo()) + scs.Equal(enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, pollResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) }, waitUpTo, checkInterval) }) } @@ -627,13 +625,13 @@ func (s *StandaloneCallbackSuite) TestDeleteCallbackExecution() { waitUpTo = 3 * time.Second checkInterval = 100 * time.Millisecond ) - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { _, err := env.FrontendClient().DescribeCallbackExecution(ctx, &workflowservice.DescribeCallbackExecutionRequest{ Namespace: env.Namespace().String(), CallbackId: callbackID, RunId: runID, }) - require.ErrorContains(t, err, "not found") + scs.ErrorContains(err, "not found") }, waitUpTo, checkInterval) } @@ -812,47 +810,47 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { s.callStartCallbackExecutionToBogusCallback(ctx, env, callbackIDs[i], time.Minute) } - // List callback executions. Visibility indexing happens be async, so use EventuallyWithT. + // List callback executions. Visibility indexing happens be async, so use s.Await(...). const ( waitUpTo = 5 * time.Second checkInterval = 200 * time.Millisecond ) // Verify returned data includes our callback IDs and has valid fields. - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ Namespace: env.Namespace().String(), PageSize: 10, }) - require.NoError(t, err) - require.GreaterOrEqual(t, len(listResp.GetExecutions()), 2) + scs.NoError(err) + scs.GreaterOrEqual(len(listResp.GetExecutions()), 2) // Collect returned callback IDs and verify fields. foundIDs := make(map[string]bool) for _, exec := range listResp.GetExecutions() { foundIDs[exec.GetCallbackId()] = true - require.NotEmpty(t, exec.GetCallbackId()) - require.NotNil(t, exec.GetCreateTime()) + scs.NotEmpty(exec.GetCallbackId()) + scs.NotNil(exec.GetCreateTime()) } for _, id := range callbackIDs { - require.True(t, foundIDs[id], "expected callback %s in list response", id) + scs.True(foundIDs[id], "expected callback %s in list response", id) } - }, waitUpTo, checkInterval, "Didn't find expected results from ListCallbackExecutions") + }, waitUpTo, checkInterval) // List with ExecutionStatus query filter — newly started callbacks should be "Running". - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ Namespace: env.Namespace().String(), PageSize: 10, Query: fmt.Sprintf(`ExecutionStatus = "Running" AND CallbackId = %q`, callbackIDs[0]), }) - require.NoError(t, err) - require.Len(t, listResp.GetExecutions(), 1) + scs.NoError(err) + scs.Len(listResp.GetExecutions(), 1) gotCb := listResp.GetExecutions()[0] - require.Equal(t, callbackIDs[0], gotCb.GetCallbackId()) - require.NotNil(t, gotCb.CreateTime) - require.Nil(t, gotCb.CloseTime) // Not in terminal state. - }, waitUpTo, checkInterval, "Didn't find Running callback") + scs.Equal(callbackIDs[0], gotCb.GetCallbackId()) + scs.NotNil(gotCb.CreateTime) + scs.Nil(gotCb.CloseTime) // Not in terminal state. + }, waitUpTo, checkInterval) // Terminate one callback to test filtering by terminal status. _, err := env.FrontendClient().TerminateCallbackExecution(ctx, &workflowservice.TerminateCallbackExecutionRequest{ @@ -865,40 +863,40 @@ func (s *StandaloneCallbackSuite) TestListAndCountCallbackExecutions() { s.NoError(err) // List with ExecutionStatus = "Terminated" should find the terminated callback. - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ Namespace: env.Namespace().String(), PageSize: 10, Query: fmt.Sprintf(`ExecutionStatus = "Terminated" AND CallbackId = %q`, callbackIDs[1]), }) - require.NoError(t, err) - require.Len(t, listResp.GetExecutions(), 1) + scs.NoError(err) + scs.Len(listResp.GetExecutions(), 1) gotCb := listResp.GetExecutions()[0] - require.Equal(t, callbackIDs[1], gotCb.GetCallbackId()) - require.NotNil(t, gotCb.CreateTime) - require.NotNil(t, gotCb.CloseTime) + scs.Equal(callbackIDs[1], gotCb.GetCallbackId()) + scs.NotNil(gotCb.CreateTime) + scs.NotNil(gotCb.CloseTime) // If Created and CloseTime are within 1ns, either the system is crazy-fast // or there is a bug where we are not fetching the current time twice. - require.Greater(t, gotCb.CloseTime.AsTime(), gotCb.CreateTime.AsTime()) - }, waitUpTo, checkInterval, "Didn't find Terminated callbacks") + scs.Greater(gotCb.CloseTime.AsTime(), gotCb.CreateTime.AsTime()) + }, waitUpTo, checkInterval) // Count callback executions. - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { countResp, err := env.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ Namespace: env.Namespace().String(), }) - require.NoError(t, err) - require.GreaterOrEqual(t, countResp.GetCount(), int64(2)) + scs.NoError(err) + scs.GreaterOrEqual(countResp.GetCount(), int64(2)) }, 10*time.Second, 200*time.Millisecond) // Count with ExecutionStatus filter should only count scheduled callbacks. - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { countResp, err := env.FrontendClient().CountCallbackExecutions(ctx, &workflowservice.CountCallbackExecutionsRequest{ Namespace: env.Namespace().String(), Query: `ExecutionStatus = "Running"`, }) - require.NoError(t, err) - require.GreaterOrEqual(t, countResp.GetCount(), int64(1)) + scs.NoError(err) + scs.GreaterOrEqual(countResp.GetCount(), int64(1)) }, waitUpTo, checkInterval) } @@ -938,15 +936,15 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_SearchAttributes() s.NoError(err) // Verify the search attribute is queryable via list. - s.EventuallyWithT(func(t *assert.CollectT) { + s.Await(func(scs *StandaloneCallbackSuite) { listResp, err := env.FrontendClient().ListCallbackExecutions(ctx, &workflowservice.ListCallbackExecutionsRequest{ Namespace: env.Namespace().String(), PageSize: 10, Query: fmt.Sprintf(`CustomKeywordField = %q AND CallbackId = %q`, saValue, callbackID), }) - require.NoError(t, err) - require.Len(t, listResp.GetExecutions(), 1) - require.Equal(t, callbackID, listResp.GetExecutions()[0].GetCallbackId()) + scs.NoError(err) + scs.Len(listResp.GetExecutions(), 1) + scs.Equal(callbackID, listResp.GetExecutions()[0].GetCallbackId()) }, 10*time.Second, 200*time.Millisecond) } From ad00970bf415443347887b2700b99774895366f9 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 11:40:00 -0700 Subject: [PATCH 44/52] Add unit tests for invocable_outbound --- chasm/lib/callback/invocable_outbound.go | 6 +- chasm/lib/callback/invocable_outbound_test.go | 111 ++++++++++++++++++ 2 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 chasm/lib/callback/invocable_outbound_test.go diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 0eed6bf9e57..145d771903e 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -41,7 +41,8 @@ func (n invocableOutbound) WrapError(result invocationResult, err error) error { // If the error is due to a completion of a Nexus operation being delivered before the // operation has officially started, we want to avoid triggering the circuit breakers. // Since the actual destination is working fine, and the failure is due to a data race. - if errors.Is(err, &serviceerror.NexusOperationNotStarted{}) { + var opNotStarted *serviceerror.NexusOperationNotStarted + if errors.As(err, &opNotStarted) { return err } @@ -107,7 +108,8 @@ func (n invocableOutbound) Invoke( // If the error from trying to resolve a Nexus operation that hasn't yet been marked // as started, it is safe to retry. (n.WrapError will ensure repeated failures of this // kind won't cause the SystemCallback to trip the circuit breaker.) - isErrNotStarted := errors.Is(err, &serviceerror.NexusOperationNotStarted{}) + var opNotStarted *serviceerror.NexusOperationNotStarted + isErrNotStarted := errors.As(err, &opNotStarted) if n.isSystemCallback() && isErrNotStarted { retryable = true } diff --git a/chasm/lib/callback/invocable_outbound_test.go b/chasm/lib/callback/invocable_outbound_test.go new file mode 100644 index 00000000000..63e34c91950 --- /dev/null +++ b/chasm/lib/callback/invocable_outbound_test.go @@ -0,0 +1,111 @@ +package callback + +import ( + "context" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + commonnexus "go.temporal.io/server/common/nexus" + queuescommon "go.temporal.io/server/service/history/queues/common" +) + +func TestInvocableOutbound_Invoke(t *testing.T) { + factory := namespace.NewDefaultReplicationResolverFactory() + nsDetail := &persistencespb.NamespaceDetail{ + Info: &persistencespb.NamespaceInfo{ + Id: "namespace-id", + Name: "namespace-name", + }, + Config: &persistencespb.NamespaceConfig{}, + } + ns, err := namespace.FromPersistentState(nsDetail, factory(nsDetail)) + require.NoError(t, err) + + tests := []struct { + name string + callbackURL string + caller HTTPCaller + assertResult func(*testing.T, invocationResult) + }{ + { + name: "happy-path", + callbackURL: "http://localhost/callback", + caller: func(_ *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Body: http.NoBody}, nil + }, + assertResult: func(t *testing.T, result invocationResult) { + require.IsType(t, invocationResultOK{}, result) + }, + }, + { + name: "non-retryable-http-error", + callbackURL: "http://localhost/callback", + caller: func(_ *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 400, Body: http.NoBody}, nil + }, + assertResult: func(t *testing.T, result invocationResult) { + require.IsType(t, invocationResultFail{}, result) + require.ErrorContains(t, result.error(), "handler error (BAD_REQUEST)") + }, + }, + { + name: "retryable-http-error", + callbackURL: "http://localhost/callback", + caller: func(_ *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 500, Body: http.NoBody}, nil + }, + assertResult: func(t *testing.T, result invocationResult) { + require.IsType(t, invocationResultRetry{}, result) + require.ErrorContains(t, result.error(), "handler error (INTERNAL)") + }, + }, + { + name: "operation-not-started-for-system-callback", + callbackURL: commonnexus.SystemCallbackURL, + caller: func(_ *http.Request) (*http.Response, error) { + return nil, serviceerror.NewNexusOperationNotStarted("nexus operation not started", "request-id") + }, + assertResult: func(t *testing.T, result invocationResult) { + // The completion must remain retryable so it can be redelivered after the + // start handler returns, and the original error type must be preserved so + // WrapError can avoid tripping the circuit breaker. + require.IsType(t, invocationResultRetry{}, result) + var opNotStarted *serviceerror.NexusOperationNotStarted + require.ErrorAs(t, result.error(), &opNotStarted) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + handler := &invocationTaskHandler{ + metricsHandler: metrics.NoopMetricsHandler, + logger: log.NewTestLogger(), + httpCallerProvider: func(_ queuescommon.NamespaceIDAndDestination) HTTPCaller { + return tc.caller + }, + } + + invokable := invocableOutbound{ + callback: &callbackspb.Callback_Nexus{Url: tc.callbackURL}, + } + + result := invokable.Invoke( + context.Background(), + ns, + handler, + nil, + chasm.TaskAttributes{Destination: tc.callbackURL}, + ) + tc.assertResult(t, result) + }) + } +} From a1427e1cb6348a4add7f500a12801817ab9c0e5b Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 12:00:26 -0700 Subject: [PATCH 45/52] Remove no longer applicable comment --- chasm/lib/callback/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index ec5a1222b51..57d29fd663f 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -82,9 +82,6 @@ type Config struct { BlobSizeLimitError dynamicconfig.IntPropertyFnWithNamespaceFilter BlobSizeLimitWarn dynamicconfig.IntPropertyFnWithNamespaceFilter MaxIDLength dynamicconfig.IntPropertyFn // Used to check CallbackID, RequestID, etc. - - // NOTE: The configuration setting defining the allowlist of supported callback - // addresses is defined in components/callbacks/config.go, via AllowedAddresses. } func ConfigProvider(dc *dynamicconfig.Collection) *Config { From 36c713486e11d092cd4901df9584583fdef9f2b8 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 12:07:40 -0700 Subject: [PATCH 46/52] Reword comment slightly --- service/frontend/configs/quotas.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/frontend/configs/quotas.go b/service/frontend/configs/quotas.go index 348245ee590..9571456ea8e 100644 --- a/service/frontend/configs/quotas.go +++ b/service/frontend/configs/quotas.go @@ -207,7 +207,7 @@ var ( // P5: Low priority APIs // GetWorkflowExecutionHistory with WaitNewEvent set to true is a long poll API. - // Similarly, `Describe{resource}Execution` are a long poll API if LongPollToken is set. + // Describe{resource}Execution APIs are also considered a long poll if LongPollToken is set. // Treat these as long-poll but lower priority (5) so spikes don’t block Poll* APIs. PollWorkflowHistoryAPIName: 5, PollActivityExecutionAPIName: 5, From 78dc137bf77c1cdf1acef4a0243cc08e20262614 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 14:20:04 -0700 Subject: [PATCH 47/52] Rerun 'make proto' to sync .pb.go with actual defs --- chasm/lib/callback/gen/callbackpb/v1/message.pb.go | 2 +- chasm/lib/callback/gen/callbackpb/v1/service.pb.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go index 542cfba8cdb..13cfa7b6289 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/message.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/message.pb.go @@ -46,7 +46,7 @@ const ( // Callback was terminated by request. Only relevant for standalone callbacks. CALLBACK_STATUS_TERMINATED CallbackStatus = 6 // Callback exceeded the schedule-to-close timeout. - CALLBACK_STATUS_TIMED_OUT CallbackStatus = 7 // TODO(chrsmith): Wire this new state in. + CALLBACK_STATUS_TIMED_OUT CallbackStatus = 7 ) // Enum value maps for CallbackStatus. diff --git a/chasm/lib/callback/gen/callbackpb/v1/service.pb.go b/chasm/lib/callback/gen/callbackpb/v1/service.pb.go index 2abe4351309..df578628e3c 100644 --- a/chasm/lib/callback/gen/callbackpb/v1/service.pb.go +++ b/chasm/lib/callback/gen/callbackpb/v1/service.pb.go @@ -30,7 +30,7 @@ const file_temporal_server_chasm_lib_callback_proto_v1_service_proto_rawDesc = " "9temporal/server/chasm/lib/callback/proto/v1/service.proto\x12,temporal.server.chasm.lib.callbacks.proto.v1\x1aBtemporal/server/chasm/lib/callback/proto/v1/request_response.proto\x1a0temporal/server/api/common/v1/api_category.proto\x1a.temporal/server/api/routing/v1/extension.proto2\x86\t\n" + "\x0fCallbackService\x12\xdd\x01\n" + "\x16StartCallbackExecution\x12K.temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionRequest\x1aL.temporal.server.chasm.lib.callbacks.proto.v1.StartCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xe6\x01\n" + - "\x19DescribeCallbackExecution\x12N.temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionRequest\x1aO.temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xda\x01\n" + + "\x19DescribeCallbackExecution\x12N.temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionRequest\x1aO.temporal.server.chasm.lib.callbacks.proto.v1.DescribeCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x02\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xda\x01\n" + "\x15PollCallbackExecution\x12J.temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionRequest\x1aK.temporal.server.chasm.lib.callbacks.proto.v1.PollCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x02\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xe9\x01\n" + "\x1aTerminateCallbackExecution\x12O.temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionRequest\x1aP.temporal.server.chasm.lib.callbacks.proto.v1.TerminateCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_id\x12\xe0\x01\n" + "\x17DeleteCallbackExecution\x12L.temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionRequest\x1aM.temporal.server.chasm.lib.callbacks.proto.v1.DeleteCallbackExecutionResponse\"(\x8a\xb5\x18\x02\b\x01\xd2\xc3\x18\x1e\x1a\x1cfrontend_request.callback_idBGZEgo.temporal.io/server/chasm/lib/callbacks/gen/callbackspb;callbackspbb\x06proto3" From f1b13e23df58522a063c8ed7c31ad7da649a596f Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 18 May 2026 15:37:14 -0700 Subject: [PATCH 48/52] Update callbacks test to match new behavior --- tests/callbacks_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/callbacks_test.go b/tests/callbacks_test.go index 89b13bb5797..82af1782b06 100644 --- a/tests/callbacks_test.go +++ b/tests/callbacks_test.go @@ -407,7 +407,7 @@ func (s *CallbacksSuite) TestWorkflowNexusCallbacks_CarriedOver() { ) } else { require.Equal(col, enumspb.CALLBACK_STATE_SUCCEEDED, callbackInfo.State) - require.Nil(col, callbackInfo.LastAttemptFailure) + // callbackInfo.LastAttemptFailure will be preserved, even if a later attempt is successful. } descCbs = append(descCbs, callbackInfo.Callback) } From ab5ccb9838add3fae9f8ae68e7301d96f0f70bcd Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Tue, 19 May 2026 13:26:25 -0700 Subject: [PATCH 49/52] Move ScheduleToCloseTimeoutTask to statemachine.go --- chasm/lib/callback/component.go | 12 ------------ chasm/lib/callback/statemachine.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/chasm/lib/callback/component.go b/chasm/lib/callback/component.go index 524e09005b7..7decf0bbf64 100644 --- a/chasm/lib/callback/component.go +++ b/chasm/lib/callback/component.go @@ -107,18 +107,6 @@ func newStandaloneCallback( return nil, fmt.Errorf("failed to schedule callback: %w", err) } - // Schedule the timeout as applicable. - if durationProto := input.ScheduleToCloseTimeout; durationProto != nil { - if duration := durationProto.AsDuration(); duration > 0 { - timeoutTime := now.AsTime().Add(duration) - ctx.AddTask( - cb, - chasm.TaskAttributes{ScheduledTime: timeoutTime}, - &callbackspb.ScheduleToCloseTimeoutTask{}, - ) - } - } - return cb, nil } diff --git a/chasm/lib/callback/statemachine.go b/chasm/lib/callback/statemachine.go index 74365f5066b..67190e39410 100644 --- a/chasm/lib/callback/statemachine.go +++ b/chasm/lib/callback/statemachine.go @@ -27,6 +27,20 @@ var TransitionScheduled = chasm.NewTransition( return fmt.Errorf("failed to parse URL: %v: %w", cb.Callback, err) } ctx.AddTask(cb, chasm.TaskAttributes{Destination: u.Scheme + "://" + u.Host}, &callbackspb.InvocationTask{}) + + // Schedule the timeout as applicable. + if durationProto := cb.ScheduleToCloseTimeout; durationProto != nil { + now := ctx.Now(cb) + if duration := durationProto.AsDuration(); duration > 0 { + timeoutTime := now.Add(duration) + ctx.AddTask( + cb, + chasm.TaskAttributes{ScheduledTime: timeoutTime}, + &callbackspb.ScheduleToCloseTimeoutTask{}, + ) + } + } + return nil }, ) From e3408bf1c3f9c243ee5b8549f6fc033440166ae5 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Tue, 19 May 2026 13:46:05 -0700 Subject: [PATCH 50/52] Update invocable_outbound_test.go to use more representative errors --- chasm/lib/callback/component_test.go | 52 ------------------- chasm/lib/callback/invocable_outbound_test.go | 44 +++++++++++----- 2 files changed, 32 insertions(+), 64 deletions(-) delete mode 100644 chasm/lib/callback/component_test.go diff --git a/chasm/lib/callback/component_test.go b/chasm/lib/callback/component_test.go deleted file mode 100644 index 6e788f065ed..00000000000 --- a/chasm/lib/callback/component_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package callback - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.temporal.io/api/serviceerror" - "go.temporal.io/server/chasm" - callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" - "go.temporal.io/server/common/backoff" - commonnexus "go.temporal.io/server/common/nexus" -) - -// Confirm that callback delivery failures due to the Nexus operation not having -// started will be retried and not trigger circuit breakers. -func TestCallbacksToUnstartedNexusOperations(t *testing.T) { - cb := &Callback{ - CallbackState: &callbackspb.CallbackState{ - Callback: &callbackspb.Callback{ - Variant: &callbackspb.Callback_Nexus_{ - Nexus: &callbackspb.Callback_Nexus{ - Url: commonnexus.SystemCallbackURL, - }, - }, - }, - Status: callbackspb.CALLBACK_STATUS_SCHEDULED, - }, - } - - // Simulate the InvocationTask being executed, which ends with the invocation's - // result being saved on the Callback. - mctx := &chasm.MockMutableContext{} - _, err := cb.saveResult(mctx, saveResultInput{ - result: invocationResultRetry{err: serviceerror.NewNexusOperationNotStarted("nexus operation not started", "req-id")}, - retryPolicy: backoff.NewExponentialRetryPolicy(time.Second), - }) - - require.NoError(t, err) - require.Equal(t, callbackspb.CALLBACK_STATUS_BACKING_OFF, cb.StateMachineState()) - require.Equal(t, int32(1), cb.Attempt) - require.Equal(t, "nexus operation not started", cb.LastAttemptFailure.Message) - require.False(t, cb.LastAttemptFailure.GetApplicationFailureInfo().NonRetryable) - require.NotNil(t, cb.NextAttemptScheduleTime) - - _, ok := cb.TerminalFailure.TryGet(mctx) - require.False(t, ok) - - // Confirm backoff task was generated. - require.Len(t, mctx.Tasks, 1) - require.IsType(t, &callbackspb.BackoffTask{}, mctx.Tasks[0].Payload) -} diff --git a/chasm/lib/callback/invocable_outbound_test.go b/chasm/lib/callback/invocable_outbound_test.go index 63e34c91950..c73c08a3795 100644 --- a/chasm/lib/callback/invocable_outbound_test.go +++ b/chasm/lib/callback/invocable_outbound_test.go @@ -2,18 +2,18 @@ package callback import ( "context" + "net" "net/http" + "os" "testing" "github.com/stretchr/testify/require" - "go.temporal.io/api/serviceerror" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" - commonnexus "go.temporal.io/server/common/nexus" queuescommon "go.temporal.io/server/service/history/queues/common" ) @@ -46,7 +46,7 @@ func TestInvocableOutbound_Invoke(t *testing.T) { }, }, { - name: "non-retryable-http-error", + name: "non-retryable-status-code", callbackURL: "http://localhost/callback", caller: func(_ *http.Request) (*http.Response, error) { return &http.Response{StatusCode: 400, Body: http.NoBody}, nil @@ -57,7 +57,7 @@ func TestInvocableOutbound_Invoke(t *testing.T) { }, }, { - name: "retryable-http-error", + name: "retryable-status-code", callbackURL: "http://localhost/callback", caller: func(_ *http.Request) (*http.Response, error) { return &http.Response{StatusCode: 500, Body: http.NoBody}, nil @@ -68,18 +68,38 @@ func TestInvocableOutbound_Invoke(t *testing.T) { }, }, { - name: "operation-not-started-for-system-callback", - callbackURL: commonnexus.SystemCallbackURL, + name: "retryable-error-dns", + callbackURL: "http://localhost/callback", + caller: func(_ *http.Request) (*http.Response, error) { + return nil, &net.DNSError{ + Err: "no such host", + Name: "example.invalid", + Server: "1.1.1.1:11", + IsNotFound: true, + IsTimeout: false, + IsTemporary: false, + } + }, + assertResult: func(t *testing.T, result invocationResult) { + require.IsType(t, invocationResultRetry{}, result) + require.ErrorContains(t, result.error(), "lookup example.invalid on 1.1.1.1:11: no such host") + }, + }, + { + name: "retryable-error-conntimeout", + callbackURL: "http://localhost/callback", caller: func(_ *http.Request) (*http.Response, error) { - return nil, serviceerror.NewNexusOperationNotStarted("nexus operation not started", "request-id") + return nil, &net.OpError{ + Op: "dial", + Net: "tcp", + Source: nil, + Addr: &net.TCPAddr{IP: net.ParseIP("192.168.0.1"), Port: 80}, + Err: os.ErrDeadlineExceeded, + } }, assertResult: func(t *testing.T, result invocationResult) { - // The completion must remain retryable so it can be redelivered after the - // start handler returns, and the original error type must be preserved so - // WrapError can avoid tripping the circuit breaker. require.IsType(t, invocationResultRetry{}, result) - var opNotStarted *serviceerror.NexusOperationNotStarted - require.ErrorAs(t, result.error(), &opNotStarted) + require.ErrorContains(t, result.error(), "dial tcp 192.168.0.1:80: i/o timeout") }, }, } From ae85bb1c6f52ffbc38fcc53674868646793298fb Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Tue, 19 May 2026 14:44:09 -0700 Subject: [PATCH 51/52] Remove MaxCallbackScheduleToCloseTimeout; add unit tests for exported methods of frontendRequestValidator --- chasm/lib/callback/config.go | 27 +- chasm/lib/callback/frontend_validation.go | 15 +- .../lib/callback/frontend_validation_test.go | 270 ++++++++++++++++++ tests/standalone_callbacks_test.go | 2 +- 4 files changed, 283 insertions(+), 31 deletions(-) diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 57d29fd663f..d0bab821a0f 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -46,13 +46,6 @@ var RequestTimeout = dynamicconfig.NewDestinationDurationSetting( `RequestTimeout is the timeout for executing a single callback request.`, ) -var MaxCallbackScheduleToCloseTimeout = dynamicconfig.NewNamespaceDurationSetting( - "callback.limit.scheduleToCloseTimeout", - 0, - `Maximum allowed duration of a callback execution. Commands that specify no schedule-to-close timeout -or a longer timeout than permitted will have their schedule-to-close timeout capped to this value. 0 implies no limit.`, -) - var RetryPolicyInitialInterval = dynamicconfig.NewGlobalDurationSetting( "callback.retryPolicy.initialInterval", time.Second, @@ -67,12 +60,11 @@ var RetryPolicyMaximumInterval = dynamicconfig.NewGlobalDurationSetting( type Config struct { // callback.* settings. - EnableStandaloneExecutions dynamicconfig.BoolPropertyFnWithNamespaceFilter - LongPollBuffer dynamicconfig.DurationPropertyFnWithNamespaceFilter - LongPollTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter - MaxCallbackScheduleToCloseTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter - RequestTimeout dynamicconfig.DurationPropertyFnWithDestinationFilter - RetryPolicy func() backoff.RetryPolicy + EnableStandaloneExecutions dynamicconfig.BoolPropertyFnWithNamespaceFilter + LongPollBuffer dynamicconfig.DurationPropertyFnWithNamespaceFilter + LongPollTimeout dynamicconfig.DurationPropertyFnWithNamespaceFilter + RequestTimeout dynamicconfig.DurationPropertyFnWithDestinationFilter + RetryPolicy func() backoff.RetryPolicy // Settings defined elsewhere. CHASMEnabled dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -86,11 +78,10 @@ type Config struct { func ConfigProvider(dc *dynamicconfig.Collection) *Config { return &Config{ - EnableStandaloneExecutions: EnableStandaloneExecutions.Get(dc), - LongPollBuffer: LongPollBuffer.Get(dc), - LongPollTimeout: LongPollTimeout.Get(dc), - MaxCallbackScheduleToCloseTimeout: MaxCallbackScheduleToCloseTimeout.Get(dc), - RequestTimeout: RequestTimeout.Get(dc), + EnableStandaloneExecutions: EnableStandaloneExecutions.Get(dc), + LongPollBuffer: LongPollBuffer.Get(dc), + LongPollTimeout: LongPollTimeout.Get(dc), + RequestTimeout: RequestTimeout.Get(dc), RetryPolicy: func() backoff.RetryPolicy { return backoff.NewExponentialRetryPolicy( RetryPolicyInitialInterval.Get(dc)(), diff --git a/chasm/lib/callback/frontend_validation.go b/chasm/lib/callback/frontend_validation.go index 4c8dbef52de..66404e579b9 100644 --- a/chasm/lib/callback/frontend_validation.go +++ b/chasm/lib/callback/frontend_validation.go @@ -14,9 +14,9 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/searchattribute" "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/durationpb" ) // Returns a serviceerror.InvalidArgument error for a missing required field. @@ -112,17 +112,8 @@ func (rv *frontendRequestValidator) ValidateAndNormalizeStartCallbackExecution( } // ScheduleToCloseTimeout - if schedToCloseTimeout := req.GetScheduleToCloseTimeout(); schedToCloseTimeout != nil { - if schedToCloseTimeout.AsDuration() <= 0 { - return serviceerror.NewInvalidArgument("schedule_to_close_timeout must be positive") - } - - // Clamp the ScheduleToCloseTimeout to the maximum allowed if set. - maxAllowed := rv.config.MaxCallbackScheduleToCloseTimeout(req.Namespace) - if maxAllowed > 0 { - clamped := min(schedToCloseTimeout.AsDuration(), maxAllowed) - req.ScheduleToCloseTimeout = durationpb.New(clamped) - } + if err := timestamp.ValidateAndCapProtoDuration(req.GetScheduleToCloseTimeout()); err != nil { + return serviceerror.NewInvalidArgumentf("schedule_to_close_timeout is invalid: %v", err) } // Validate the input data to deliver to the callback URL, currently only one kind is supported (Completion). diff --git a/chasm/lib/callback/frontend_validation_test.go b/chasm/lib/callback/frontend_validation_test.go index a896e878bf3..e6e3e0be898 100644 --- a/chasm/lib/callback/frontend_validation_test.go +++ b/chasm/lib/callback/frontend_validation_test.go @@ -1,10 +1,22 @@ package callback import ( + "context" "fmt" + "strings" "testing" + "time" + "github.com/google/uuid" "github.com/stretchr/testify/require" + callbackpb "go.temporal.io/api/callback/v1" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/workflowservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" ) func TestRequiredStringFields(t *testing.T) { @@ -40,3 +52,261 @@ func TestRequiredStringFields(t *testing.T) { } require.ErrorContains(t, mixedTests.Validate(), "Mixed Field2 is required") } + +type noopCallbackValidator struct{} + +func (noopCallbackValidator) Validate(context.Context, string, []*commonpb.Callback) error { + return nil +} + +func newTestRequestValidator(maxIDLength, blobLimit int) *frontendRequestValidator { + return &frontendRequestValidator{ + config: &Config{ + MaxIDLength: func() int { return maxIDLength }, + BlobSizeLimitError: func(string) int { return blobLimit }, + BlobSizeLimitWarn: func(string) int { return blobLimit }, + }, + cbValidator: noopCallbackValidator{}, + logger: log.NewNoopLogger(), + } +} + +type validationTest[T any] struct { + name string + mutate func(*T) + wantErr string +} + +func runValidationTests[T any]( + t *testing.T, + newValidRequestProto func() *T, + validate func(*T) error, + tests []validationTest[T], +) { + t.Helper() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := newValidRequestProto() + tt.mutate(req) + err := validate(req) + if tt.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantErr) + } + }) + } +} + +// TestFrontendRequestValidator covers all of the exported functions from the +// frontendRequestValidator type, using the generic `runValidationTests[T]` to +// cut down on boilerplate. +func TestFrontendRequestValidator(t *testing.T) { + const ( + maxIDLength = 64 + maxBlobSize = 1024 + ) + rv := newTestRequestValidator(maxIDLength, maxBlobSize) + + longID := strings.Repeat("a", maxIDLength+1) + validUUID := uuid.NewString() + + t.Run("ValidateAndNormalizeStartCallbackExecution", func(t *testing.T) { + type ReqProto = workflowservice.StartCallbackExecutionRequest + newValidRequestProto := func() *ReqProto { + return &ReqProto{ + Namespace: "namespace", + Identity: "identity", + CallbackId: "callback-id", + Input: &workflowservice.StartCallbackExecutionRequest_Completion{ + Completion: &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{Success: &commonpb.Payload{}}, + }, + }, + } + } + + validate := func(req *ReqProto) error { + ctx := context.Background() + return rv.ValidateAndNormalizeStartCallbackExecution(ctx, req) + } + + runValidationTests(t, newValidRequestProto, validate, []validationTest[ReqProto]{ + {"valid", func(*ReqProto) {}, ""}, + {"missing namespace", func(r *ReqProto) { r.Namespace = "" }, "namespace is required"}, + {"missing identity", func(r *ReqProto) { r.Identity = "" }, "identity is required"}, + {"missing callback_id", func(r *ReqProto) { r.CallbackId = "" }, "callback_id is required"}, + {"callback_id too long", func(r *ReqProto) { r.CallbackId = longID }, "callback_id exceeds length limit"}, + {"identity too long", func(r *ReqProto) { r.Identity = longID }, "identity exceeds length limit"}, + {"request_id too long", func(r *ReqProto) { r.RequestId = longID }, "request_id exceeds length limit"}, + { + "negative schedule_to_close timeout", + func(r *ReqProto) { + r.ScheduleToCloseTimeout = durationpb.New(-time.Second) + }, + "schedule_to_close_timeout is invalid", + }, + {"missing completion", func(r *ReqProto) { r.Input = nil }, "completion is not set"}, + { + "completion empty", + func(r *ReqProto) { + r.Input = &workflowservice.StartCallbackExecutionRequest_Completion{ + Completion: &callbackpb.CallbackExecutionCompletion{}, + } + }, + "completion must have either success or failure set", + }, + { + "completion oversized", + func(r *ReqProto) { + massivePayload := strings.Repeat("x", maxBlobSize+1) + massiveCompletion := &workflowservice.StartCallbackExecutionRequest_Completion{ + Completion: &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: &commonpb.Payload{ + Data: []byte(massivePayload), + }, + }, + }, + } + r.Input = massiveCompletion + }, + "Blob data size exceeds limit", + }, + }) + + t.Run("RequestID is auto-assigned", func(t *testing.T) { + req := newValidRequestProto() + require.NoError(t, validate(req)) + require.NotEmpty(t, req.RequestId) + }) + }) + + t.Run("ValidateDescribeCallbackExecution", func(t *testing.T) { + nsID := namespace.ID("ns-id") + tokenFor := func(nsID string) []byte { + b, err := proto.Marshal(&persistencespb.ChasmComponentRef{ + NamespaceId: nsID, + BusinessId: "callback-id", + }) + require.NoError(t, err) + return b + } + + type ReqProto = workflowservice.DescribeCallbackExecutionRequest + newValidRequestProto := func() *ReqProto { + return &ReqProto{ + Namespace: "ns", + CallbackId: "cb", + } + } + validate := func(req *ReqProto) error { + return rv.ValidateDescribeCallbackExecution(req, nsID) + } + + runValidationTests(t, newValidRequestProto, validate, []validationTest[ReqProto]{ + {"valid", func(*ReqProto) {}, ""}, + {"valid run_id", func(r *ReqProto) { r.RunId = validUUID }, ""}, + {"missing namespace", func(r *ReqProto) { r.Namespace = "" }, "namespace is required"}, + {"missing callback_id", func(r *ReqProto) { r.CallbackId = "" }, "callback_id is required"}, + {"invalid run_id", func(r *ReqProto) { r.RunId = "not-uuid" }, "invalid run_id"}, + {"callback_id too long", func(r *ReqProto) { r.CallbackId = longID }, "callback_id exceeds length limit"}, + {"long_poll without run_id", func(r *ReqProto) { r.LongPollToken = []byte("x") }, "run_id is required when long_poll_token"}, + { + "malformed long_poll", + func(r *ReqProto) { + r.RunId, r.LongPollToken = validUUID, []byte("garbage") + }, + "invalid long poll token", + }, + { + "long_poll namespace mismatch", + func(r *ReqProto) { + r.RunId, r.LongPollToken = validUUID, tokenFor("other-ns-id") + }, + "long poll token does not match", + }, + { + "long_poll match", + func(r *ReqProto) { + r.RunId, r.LongPollToken = validUUID, tokenFor(nsID.String()) + }, + "", + }, + }) + }) + + t.Run("ValidatePollCallbackExecution", func(t *testing.T) { + type ReqProto = workflowservice.PollCallbackExecutionRequest + newValidRequestProto := func() *ReqProto { + return &ReqProto{Namespace: "ns", CallbackId: "cb"} + } + + runValidationTests(t, newValidRequestProto, rv.ValidatePollCallbackExecution, []validationTest[ReqProto]{ + {"valid", func(*ReqProto) {}, ""}, + {"valid run_id", func(r *ReqProto) { r.RunId = uuid.NewString() }, ""}, + {"missing namespace", func(r *ReqProto) { r.Namespace = "" }, "namespace is required"}, + {"missing callback_id", func(r *ReqProto) { r.CallbackId = "" }, "callback_id is required"}, + {"invalid run_id", func(r *ReqProto) { r.RunId = "not-uuid" }, "invalid run_id"}, + {"callback_id too long", func(r *ReqProto) { r.CallbackId = longID }, "callback_id exceeds length limit"}, + }) + }) + + t.Run("ValidateAndNormalizeTerminateCallbackExecution", func(t *testing.T) { + type ReqProto = workflowservice.TerminateCallbackExecutionRequest + newValidRequestProto := func() *ReqProto { + return &ReqProto{ + Namespace: "ns", + CallbackId: "callback-id", + } + } + + runValidationTests(t, newValidRequestProto, rv.ValidateAndNormalizeTerminateCallbackExecution, []validationTest[ReqProto]{ + {"valid", func(*ReqProto) {}, ""}, + {"missing namespace", func(r *ReqProto) { r.Namespace = "" }, "namespace is required"}, + {"missing callback_id", func(r *ReqProto) { r.CallbackId = "" }, "callback_id is required"}, + {"invalid run_id", func(r *ReqProto) { r.RunId = "not-uuid" }, "invalid run_id"}, + {"callback_id too long", func(r *ReqProto) { r.CallbackId = longID }, "callback_id exceeds length limit"}, + {"identity too long", func(r *ReqProto) { r.Identity = longID }, "identity exceeds length limit"}, + {"request_id too long", func(r *ReqProto) { r.RequestId = longID }, "request_id exceeds length limit"}, + {"reason too long", func(r *ReqProto) { r.Reason = longID }, "reason exceeds length limit"}, + }) + + t.Run("RequestID is auto-assigned", func(t *testing.T) { + req := newValidRequestProto() + require.NoError(t, rv.ValidateAndNormalizeTerminateCallbackExecution(req)) + require.NotEmpty(t, req.RequestId) + }) + }) + + t.Run("ValidateDeleteCallbackExecution", func(t *testing.T) { + type ReqProto = workflowservice.DeleteCallbackExecutionRequest + newValidRequestProto := func() *ReqProto { + return &ReqProto{ + Namespace: "ns", + CallbackId: "callback-id", + } + } + + runValidationTests(t, newValidRequestProto, rv.ValidateDeleteCallbackExecution, []validationTest[ReqProto]{ + {"valid", func(*ReqProto) {}, ""}, + {"missing namespace", func(r *ReqProto) { r.Namespace = "" }, "namespace is required"}, + {"missing callback_id", func(r *ReqProto) { r.CallbackId = "" }, "callback_id is required"}, + {"callback_id too long", func(r *ReqProto) { r.CallbackId = longID }, "callback_id exceeds length limit"}, + }) + }) + + t.Run("ValidateListCallbackExecutions", func(t *testing.T) { + // Only validates the Namespace is set. + rv := newTestRequestValidator(10, 20) + require.NoError(t, rv.ValidateListCallbackExecutions(&workflowservice.ListCallbackExecutionsRequest{Namespace: "ns"})) + require.ErrorContains(t, rv.ValidateListCallbackExecutions(&workflowservice.ListCallbackExecutionsRequest{}), "namespace is required") + }) + + t.Run("ValidateCountCallbackExecutions", func(t *testing.T) { + // Only validates the Namespace is set. + rv := newTestRequestValidator(10, 20) + require.NoError(t, rv.ValidateCountCallbackExecutions(&workflowservice.CountCallbackExecutionsRequest{Namespace: "ns"})) + require.ErrorContains(t, rv.ValidateCountCallbackExecutions(&workflowservice.CountCallbackExecutionsRequest{}), "namespace is required") + }) +} diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index 2fda2a5859d..bbb4290590a 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -718,7 +718,7 @@ func (s *StandaloneCallbackSuite) TestStartCallbackExecution_InvalidArguments() mutate: func(req *workflowservice.StartCallbackExecutionRequest) { req.ScheduleToCloseTimeout = durationpb.New(-time.Second) }, - errMsg: "schedule_to_close_timeout must be positive", + errMsg: "schedule_to_close_timeout is invalid: negative duration", }, } From 2c6bce478f0597cb5493f4c1f2af34668d5a52f5 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Tue, 19 May 2026 15:53:24 -0700 Subject: [PATCH 52/52] Properly wire 'nexus op not started' as a nexus.HandlerError --- chasm/lib/callback/invocable_outbound.go | 29 +---- components/nexusoperations/completion.go | 6 +- tests/standalone_callbacks_test.go | 147 +++++++++++++++++++++++ 3 files changed, 153 insertions(+), 29 deletions(-) diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 145d771903e..11bd43dee7f 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -7,7 +7,6 @@ import ( "time" "github.com/nexus-rpc/sdk-go/nexus" - "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/log" @@ -29,23 +28,7 @@ type invocableOutbound struct { attempt int32 } -func (n invocableOutbound) isSystemCallback() bool { - c := n.callback - if c == nil { - return false - } - return c.Url == commonnexus.SystemCallbackURL -} - func (n invocableOutbound) WrapError(result invocationResult, err error) error { - // If the error is due to a completion of a Nexus operation being delivered before the - // operation has officially started, we want to avoid triggering the circuit breakers. - // Since the actual destination is working fine, and the failure is due to a data race. - var opNotStarted *serviceerror.NexusOperationNotStarted - if errors.As(err, &opNotStarted) { - return err - } - if retry, ok := result.(invocationResultRetry); ok { return queueserrors.NewDestinationDownError(retry.err.Error(), err) } @@ -85,7 +68,7 @@ func (n invocableOutbound) Invoke( startTime := time.Now() n.completion.Header = n.callback.Header - // If the Nexus callback token is set on the callback, pass it along in completion's headers. + // If the Nexus callback token is set on the supplied callback, pass it along in completion's headers. if n.callback.GetToken() != "" { if n.completion.Header == nil { n.completion.Header = nexus.Header{} @@ -104,16 +87,6 @@ func (n invocableOutbound) Invoke( if err != nil { retryable := isRetryableCallError(err) h.logger.Error("Callback request failed", tag.Error(err), tag.Bool("retryable", retryable)) - - // If the error from trying to resolve a Nexus operation that hasn't yet been marked - // as started, it is safe to retry. (n.WrapError will ensure repeated failures of this - // kind won't cause the SystemCallback to trip the circuit breaker.) - var opNotStarted *serviceerror.NexusOperationNotStarted - isErrNotStarted := errors.As(err, &opNotStarted) - if n.isSystemCallback() && isErrNotStarted { - retryable = true - } - if retryable { return invocationResultRetry{err} } diff --git a/components/nexusoperations/completion.go b/components/nexusoperations/completion.go index 94984231e3c..de8cd923851 100644 --- a/components/nexusoperations/completion.go +++ b/components/nexusoperations/completion.go @@ -144,7 +144,11 @@ func fabricateStartedEventIfMissing( // TODO(NEXUS-369): We should delay the completion somehow, e.g. using a chasm.PollComponent, // to allow avoid surfacing any traisent errors to users. if operationToken == "" { - return serviceerror.NewNexusOperationNotStarted("nexus operation not started", requestID) + return &nexus.HandlerError{ + Type: nexus.HandlerErrorTypeBadRequest, + Message: "nexus operation not started", + RetryBehavior: nexus.HandlerErrorRetryBehaviorRetryable, + } } eventID, err := hsm.EventIDFromToken(operation.ScheduledEventToken) diff --git a/tests/standalone_callbacks_test.go b/tests/standalone_callbacks_test.go index bbb4290590a..b115b3daf00 100644 --- a/tests/standalone_callbacks_test.go +++ b/tests/standalone_callbacks_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "slices" + "sync" "testing" "time" @@ -1226,3 +1227,149 @@ func (s *StandaloneCallbackSuite) TestScheduleToCloseTimeout() { s.NotNil(descResp.GetOutcome().GetFailure().GetTimeoutFailureInfo()) s.Equal(enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, descResp.GetOutcome().GetFailure().GetTimeoutFailureInfo().GetTimeoutType()) } + +// TestCallbackBeforeNexusOperationStart tests the situation where the standalone callback is +// called BEFORE the NexusStartOperation request is complete. (And the Temporal server sees +// a completion for a Nexus operation that doesn't yet exist.) +func (s *StandaloneCallbackSuite) TestCallbackBeforeNexusOperationStart() { + env := s.newEnv() + ctx := env.Context() + taskQueue := testcore.RandomizeStr(s.T().Name()) + + fakeSvc := startFakeExternalService(ctx, env.FrontendClient()) + + // Wait group used in the nexusHandler to block returning from + // OnStartOperation until we can inspect the results of the async + // StartCallbackExecution call. + var waitForExternalSvc sync.WaitGroup + waitForExternalSvc.Add(1) + + nexusEndpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) + nexusHandler := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + // Send the request to the external service, which will call + // StartCallbackExecution. + completion := &callbackpb.CallbackExecutionCompletion{ + Result: &callbackpb.CallbackExecutionCompletion_Success{ + Success: testcore.MustToPayload(s.T(), "success"), + }, + } + fakeSvc.incomingRequests <- externalRequestInfo{ + Namespace: env.Namespace().String(), + Token: options.CallbackHeader.Get(commonnexus.CallbackTokenHeader), + URL: options.CallbackURL, + + Result: completion, + } + + // Block waiting for the result from the fake svc to get the result + // BEFORE we return from the StartNexusOperation call. + waitForExternalSvc.Wait() + + // End the Nexus operation, reporting back to Temporal the Nexus operation + // has officially started. + return &nexus.HandlerStartOperationResultAsync{ + OperationToken: fmt.Sprintf("operation-token-%s", uuid.NewString()), + }, nil + }, + } + + // Start the Nexus server, register the endpoint, start a workflow to + // invoke the Nexus operation. + listenAddr := nexustest.AllocListenAddress() + nexustest.NewNexusServer(s.T(), listenAddr, nexusHandler) + createNexusEndpointReq := &operatorservice.CreateNexusEndpointRequest{ + Spec: &nexuspb.EndpointSpec{ + Name: nexusEndpointName, + Target: &nexuspb.EndpointTarget{ + Variant: &nexuspb.EndpointTarget_External_{ + External: &nexuspb.EndpointTarget_External{ + Url: "http://" + listenAddr, + }, + }, + }, + }, + } + _, err := env.OperatorClient().CreateNexusEndpoint(ctx, createNexusEndpointReq) + s.NoError(err, "Error registering Nexus endpoint") + + // Calling Workflow + callerWf := func(ctx workflow.Context) (string, error) { + c := workflow.NewNexusClient(nexusEndpointName, "nexus-service") + fut := c.ExecuteOperation(ctx, "nexus-op", "input", workflow.NexusOperationOptions{}) + + var nexusOpResult string + err := fut.Get(ctx, &nexusOpResult) + return nexusOpResult, err + } + + callerWfWorker := worker.New(env.SdkClient(), taskQueue, worker.Options{}) + callerWfWorker.RegisterWorkflow(callerWf) + s.NoError(callerWfWorker.Start(), "Error starting calling workflow Worker") + defer callerWfWorker.Stop() + + // Start the workflow, triggering the Nexus operation. + startOpts := client.StartWorkflowOptions{ + TaskQueue: taskQueue, + } + callerWfRun, err := env.SdkClient().ExecuteWorkflow(ctx, startOpts, callerWf) + s.NoError(err, "Error running the caller Workflow") + + // By pulling an item from the channel, we can confirm the fake service was able + // to successfully call StartCallbackExecution before the Nexus handler responded. + gotResult := <-fakeSvc.requestResults + env.NoError(gotResult.Error) + env.NotEmpty(gotResult.CallbackID) + env.NotEmpty(gotResult.RunID) + + // Get the callback's status on the Temporal side. Wait until we see some sort of failure + // when trying to invoke the callback. + var gotLastFailure *failurepb.Failure + const ( + waitUpTo = 3 * time.Second + checkInterval = 200 * time.Millisecond + ) + s.Await(func(scs *StandaloneCallbackSuite) { + shortTimeout, cancelFn := context.WithTimeout(ctx, time.Second) + callbackState, err := env.FrontendClient().DescribeCallbackExecution(shortTimeout, &workflowservice.DescribeCallbackExecutionRequest{ + Namespace: env.Namespace().String(), + CallbackId: gotResult.CallbackID, + RunId: gotResult.RunID, + }) + cancelFn() + + scs.NoError(err) + + exState := callbackState.GetInfo() + scs.NotNil(exState) + + // The callback starts in STANDBY. We are waiting for it to have been invoked at least once. + scs.NotEqual(enumspb.CALLBACK_STATE_STANDBY, exState.State) + // It shouldn't be in a terminal state. + scs.NotEqual(enumspb.CALLBACK_STATE_FAILED, exState.State) + scs.NotEqual(enumspb.CALLBACK_STATE_SUCCEEDED, exState.State) + + gotLastFailure = exState.LastAttemptFailure + scs.NotNil(gotLastFailure) + }, waitUpTo, checkInterval) + + // Confirm the last failure is what we expect. The frontend returns a retryable BAD_REQUEST nexus.HandlerError, + // but that gets observed as a retryable ISE. + env.Equal("handler error (INTERNAL)", gotLastFailure.Message) + + // We are done inspecting the started StandaloneCallbackExecution, now have the Nexus operation + // officially "start". This will then have the callback retry again, and this time not fail + // with the NexusOperationNotStarted error. + waitForExternalSvc.Done() + + // At this point, wait until the workflow has completed successfully. This implicitly verifies + // that the callback was retried and successfully completed. + var gotPayload string + s.NoError(callerWfRun.Get(ctx, &gotPayload)) + s.Equal("success", gotPayload) +}