fix: silence expected user-workflow failures from Sentry error alerts (#526)

Co-authored-by: Will Zimmerman <will@avaprotocol.org>

Author: chrisli30

aggregator/rpc_server.go

@@ -455,7 +455,11 @@ func (r *RpcServer) WithdrawFunds(ctx context.Context, payload *avsproto.Withdra
 	)
 
 	if err != nil {
-		r.config.Logger.Error("failed to send withdrawal UserOp",
+		// See preset.LogBundlerError: Warn on on-chain revert (user's withdrawal
+		// reverted — e.g. ERC20 transfer to blacklisted recipient, insufficient
+		// token balance after race), Error on infra/AA (bundler down, AA21, etc.).
+		preset.LogBundlerError(r.config.Logger, err,
+			"failed to send withdrawal UserOp",
 			"error", err,
 			"user", user.Address.String(),
 			"recipient", payload.RecipientAddress,

core/taskengine/engine.go

@@ -3080,7 +3080,10 @@ func (n *Engine) SimulateTask(user *model.User, trigger *avsproto.TaskTrigger, n
 		cleanErrorMsg = stackTraceRegex.ReplaceAllString(cleanErrorMsg, "")
 		cleanErrorMsg = strings.TrimSpace(cleanErrorMsg)
 
-		n.logger.Error("workflow simulation completed with failures",
+		// User-workflow simulation failure: per-step errors are captured in the
+		// persisted execution steps. Log summary at Warn so it stays out of Sentry
+		// error alerts.
+		n.logger.Warn("workflow simulation completed with failures",
 			"error", cleanErrorMsg,
 			"task_id", task.Id,
 			"simulation_id", simulationID,

core/taskengine/executor.go

@@ -653,7 +653,10 @@ func (x *TaskExecutor) RunTask(task *model.Task, queueData *QueueExecutionData)
 	case ExecutionSuccess:
 		x.logger.Info("task execution completed successfully", "task_id", task.Id, "execution_id", queueData.ExecutionID, "total_steps", len(vm.ExecutionLogs))
 	case ExecutionFailed:
-		x.logger.Error("task execution completed with failures",
+		// User-workflow failure: per-step errors are already logged at their sites
+		// and the ExecutionStatus_EXECUTION_STATUS_FAILED is persisted below. Log
+		// the summary at Warn so it stays out of Sentry error alerts.
+		x.logger.Warn("task execution completed with failures",
 			"error", executionError,
 			"task_id", task.Id,
 			"execution_id", queueData.ExecutionID,

core/taskengine/tenderly_client.go

@@ -1049,7 +1049,10 @@ func (tc *TenderlyClient) SimulateContractWrite(ctx context.Context, contractAdd
 				if status, ok := sim["status"].(bool); ok && !status {
 					result.Success = false
 					if em, ok := sim["error_message"].(string); ok && em != "" {
-						tc.logger.Error("❌ Tenderly simulation failed: simulation.status=false",
+						// Simulation catching a future revert is the feature working
+						// as intended — user-workflow failure, not infra. Log at Warn
+						// so it stays out of Sentry error alerts.
+						tc.logger.Warn("tenderly simulation failed: simulation.status=false",
 							"contract", contractAddress,
 							"method", methodName,
 							"error_message", em,
@@ -1081,7 +1084,9 @@ func (tc *TenderlyClient) SimulateContractWrite(ctx context.Context, contractAdd
 								// Look for error in nested calls (like ERC20 transferFrom failures)
 								if errMsg, ok := callMap["error"].(string); ok && errMsg != "" {
 									errorMsg = errMsg
-									tc.logger.Error("❌ Tenderly simulation failed: transaction reverted",
+									// User-workflow revert caught by simulation — log at
+									// Warn to keep out of Sentry error alerts.
+									tc.logger.Warn("tenderly simulation failed: transaction reverted",
 										"contract", contractAddress,
 										"method", methodName,
 										"error_from_call_trace", errMsg,

core/taskengine/vm_runner_contract_write.go

@@ -814,7 +814,12 @@ func (r *ContractWriteProcessor) executeRealUserOpTransaction(ctx context.Contex
 			}
 		}
 
-		r.vm.logger.Error("🚫 BUNDLER FAILED - UserOp transaction failed, workflow execution FAILED",
+		// preset.LogBundlerError picks Error vs Warn based on the error: on-chain
+		// reverts (expected user-workflow outcomes) log at Warn so they don't page
+		// Sentry; real infra/AA failures (bundler down, AA21/AA23/AA25, paymaster
+		// revert) stay at Error.
+		preset.LogBundlerError(r.vm.logger, err,
+			"bundler: UserOp transaction failed, workflow execution FAILED",
 			"bundler_error", err,
 			"bundler_url", r.smartWalletConfig.BundlerURL,
 			"method", methodName,
@@ -1209,43 +1214,35 @@ func (r *ContractWriteProcessor) convertTenderlyResultToFlexibleFormat(result *C
 
 	receipt, _ := structpb.NewValue(receiptMap)
 
-	// Extract return value from Tenderly response
+	// Extract return value from Tenderly response.
+	// ReturnData is nil when the provider did not return output data (e.g. simulation
+	// reverted — tenderly_client.go clears ReturnData in that case). That path leaves
+	// Value as nil, which is the expected behavior.
 	var returnValue *structpb.Value
 	if result.ReturnData != nil {
-		r.vm.logger.Info("🔍 CRITICAL DEBUG - ReturnData found",
-			"method", result.MethodName,
-			"returnData_name", result.ReturnData.Name,
-			"returnData_type", result.ReturnData.Type,
-			"returnData_value", result.ReturnData.Value)
-
 		// Parse the JSON value from ReturnData and convert to protobuf
 		var parsedValue interface{}
 		if err := json.Unmarshal([]byte(result.ReturnData.Value), &parsedValue); err == nil {
 			// Successfully parsed JSON, convert to protobuf
 			if valueProto, err := structpb.NewValue(parsedValue); err == nil {
 				returnValue = valueProto
-				r.vm.logger.Info("✅ CRITICAL DEBUG - Successfully created returnValue protobuf",
-					"method", result.MethodName,
-					"parsedValue", parsedValue)
 			} else {
-				r.vm.logger.Error("❌ CRITICAL DEBUG - Failed to create protobuf from parsedValue",
+				r.vm.logger.Debug("failed to create protobuf from parsed ReturnData",
 					"method", result.MethodName,
 					"error", err)
 			}
 		} else {
-			r.vm.logger.Error("❌ CRITICAL DEBUG - Failed to unmarshal JSON from ReturnData.Value",
+			// Non-JSON return types (bytes32, address, etc.) are expected; fall through
+			// to raw-string handling below.
+			r.vm.logger.Debug("ReturnData is not JSON, falling back to raw string",
 				"method", result.MethodName,
-				"error", err,
-				"raw_value", result.ReturnData.Value)
+				"error", err)
 
 			// Fallback: treat as raw string if JSON parsing fails
 			if valueProto, err := structpb.NewValue(result.ReturnData.Value); err == nil {
 				returnValue = valueProto
 			}
 		}
-	} else {
-		r.vm.logger.Error("❌ CRITICAL DEBUG - ReturnData is nil",
-			"method", result.MethodName)
 	}
 
 	// No fallback default value. If provider does not return output data, Value remains nil
@@ -1624,7 +1621,11 @@ func (r *ContractWriteProcessor) Execute(stepID string, node *avsproto.ContractW
 				}
 			}
 		} else {
-			r.vm.logger.Error("🚨 DEPLOYED WORKFLOW: Method execution failed",
+			// User-workflow failure: method returned success=false. The concrete
+			// cause is already logged by the upstream site (Tenderly simulation at
+			// tenderly_client.go, or bundler/AA at line ~817). Re-logging at Warn
+			// here keeps operator-visible context without paging Sentry.
+			r.vm.logger.Warn("deployed workflow: method execution failed",
 				"method_name", result.MethodName,
 				"error_message", result.Error,
 				"error_length", len(result.Error),

core/taskengine/vm_runner_eth_transfer.go

@@ -344,7 +344,9 @@ func (p *ETHTransferProcessor) executeRealETHTransfer(stepID, destination, amoun
 	)
 
 	if err != nil {
-		p.vm.logger.Error("🚫 BUNDLER FAILED - ETH transfer UserOp transaction failed",
+		// See preset.LogBundlerError: Warn on on-chain revert, Error on infra/AA.
+		preset.LogBundlerError(p.vm.logger, err,
+			"bundler: ETH transfer UserOp transaction failed",
 			"bundler_error", err,
 			"bundler_url", p.smartWalletConfig.BundlerURL,
 			"destination", destination,

pkg/erc4337/preset/bundler_error.go

@@ -0,0 +1,43 @@
+package preset
+
+import (
+	"strings"
+
+	"github.com/AvaProtocol/EigenLayer-AVS/pkg/logger"
+)
+
+// userOpRevertMarker identifies errors returned by SendUserOp when the UserOp
+// was included on-chain but the target contract call reverted. The marker
+// string is emitted from waitForUserOpConfirmation via fmt.Errorf.
+const userOpRevertMarker = "success=false in UserOperationEvent"
+
+// IsUserOpRevert reports whether err represents an on-chain revert of the user
+// target contract (UserOp was mined but UserOperationEvent.success == false),
+// as distinct from infra/AA failures such as bundler unreachable, AA21 prefund,
+// AA23 reverted, AA25 invalid nonce, or paymaster revert.
+//
+// On-chain reverts are expected user-workflow outcomes and should not escalate
+// to Sentry error alerts. Infra/AA failures should.
+func IsUserOpRevert(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), userOpRevertMarker)
+}
+
+// LogBundlerError logs a bundler/UserOp failure at the severity appropriate
+// for its cause: Warn for on-chain reverts (see IsUserOpRevert) so they do not
+// page Sentry, Error for real infra/AA failures that operators must see.
+//
+// Callers pass the error both for classification (the first argument) and,
+// conventionally, as a tag value so the logged record includes the full error.
+func LogBundlerError(lgr logger.Logger, err error, msg string, tags ...any) {
+	if lgr == nil {
+		return
+	}
+	if IsUserOpRevert(err) {
+		lgr.Warn(msg, tags...)
+		return
+	}
+	lgr.Error(msg, tags...)
+}

pkg/erc4337/preset/bundler_error_test.go

@@ -0,0 +1,82 @@
+package preset
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+
+	sdklogging "github.com/Layr-Labs/eigensdk-go/logging"
+)
+
+func TestIsUserOpRevert(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"unrelated", errors.New("dial tcp: connection refused"), false},
+		{"AA21 prefund", errors.New("AA21 didn't pay prefund"), false},
+		{"AA25 nonce", errors.New("AA25 invalid account nonce"), false},
+		{"direct marker", errors.New("UserOp execution failed (success=false in UserOperationEvent) - tx: 0xabc"), true},
+		{"wrapped marker", fmt.Errorf("UserOp execution failed: %w", errors.New("UserOp execution failed (success=false in UserOperationEvent) - tx: 0xabc")), true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := IsUserOpRevert(tc.err); got != tc.want {
+				t.Errorf("IsUserOpRevert(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+// bundlerErrorSpyLogger captures which severity method was invoked for LogBundlerError.
+type bundlerErrorSpyLogger struct {
+	mu    sync.Mutex
+	calls []string // method names in order
+}
+
+func (s *bundlerErrorSpyLogger) record(method string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.calls = append(s.calls, method)
+}
+
+func (s *bundlerErrorSpyLogger) Debug(string, ...any)          {}
+func (s *bundlerErrorSpyLogger) Debugf(string, ...any)         {}
+func (s *bundlerErrorSpyLogger) Info(string, ...any)           {}
+func (s *bundlerErrorSpyLogger) Infof(string, ...any)          {}
+func (s *bundlerErrorSpyLogger) Warn(string, ...any)           { s.record("Warn") }
+func (s *bundlerErrorSpyLogger) Warnf(string, ...any)          { s.record("Warn") }
+func (s *bundlerErrorSpyLogger) Error(string, ...any)          { s.record("Error") }
+func (s *bundlerErrorSpyLogger) Errorf(string, ...any)         { s.record("Error") }
+func (s *bundlerErrorSpyLogger) Fatal(string, ...any)          {}
+func (s *bundlerErrorSpyLogger) Fatalf(string, ...any)         {}
+func (s *bundlerErrorSpyLogger) With(...any) sdklogging.Logger { return s }
+
+func TestLogBundlerError(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{"on-chain revert → Warn", errors.New("UserOp execution failed (success=false in UserOperationEvent) - tx: 0xabc"), "Warn"},
+		{"AA21 infra → Error", errors.New("AA21 didn't pay prefund"), "Error"},
+		{"bundler down → Error", errors.New("dial tcp: connection refused"), "Error"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			spy := &bundlerErrorSpyLogger{}
+			LogBundlerError(spy, tc.err, "bundler failed", "err", tc.err)
+			if len(spy.calls) != 1 || spy.calls[0] != tc.want {
+				t.Errorf("expected single %s call, got %v", tc.want, spy.calls)
+			}
+		})
+	}
+}
+
+func TestLogBundlerError_NilLogger(t *testing.T) {
+	// Must not panic.
+	LogBundlerError(nil, errors.New("anything"), "msg")
+}