feat(runner): add timeout support (#134)

Author: skylenet

config.example.yaml

@@ -298,6 +298,9 @@ runner:
       # Useful for clients like Erigon that need time to complete internal sync pipelines
       # after their RPC endpoint becomes available.
       # wait_after_rpc_ready: 30s
+      # Optional: Maximum duration for the test execution phase.
+      # If exceeded, the run is cancelled with "timed_out" status. Partial results are kept.
+      # run_timeout: 2h
       # Optional: Retry engine_newPayload calls when client returns SYNCING status.
       # Useful for clients with internal sync pipelines (e.g., Erigon) that may return
       # SYNCING while still processing blocks internally.
@@ -443,6 +446,7 @@ runner:
       #   tmpfs_max_size: "16g"
       #   wait_after_tcp_drop_connections: "10s"
       # wait_after_rpc_ready: 60s  # Instance-level override (optional)
+      # run_timeout: 1h  # Instance-level override (optional)
       # retry_new_payloads_syncing_state:  # Instance-level override (optional)
       #   enabled: true
       #   max_retries: 10

docs/configuration.md

@@ -506,6 +506,7 @@ runner:
 | `rollback_strategy` | string | `rpc-debug-setHead` | Rollback strategy after each test (see below) |
 | `checkpoint_restore_strategy_options` | object | - | Options for the checkpoint-restore rollback strategy (see [Checkpoint Restore Strategy Options](#checkpoint-restore-strategy-options)) |
 | `wait_after_rpc_ready` | string | - | Duration to wait after RPC becomes ready (see below) |
+| `run_timeout` | string | - | Maximum duration for test execution before the run is timed out (see below) |
 | `retry_new_payloads_syncing_state` | object | - | Retry config for SYNCING responses (see below) |
 | `resource_limits` | object | - | Container resource constraints (see [Resource Limits](#resource-limits)) |
 | `post_test_rpc_calls` | []object | - | Arbitrary RPC calls to execute after each test step (see [Post-Test RPC Calls](#post-test-rpc-calls)) |
@@ -666,6 +667,26 @@ The value is a Go duration string (e.g., `30s`, `1m`, `500ms`). If not set, no a
 - When you observe `SYNCING` responses from Engine API calls despite the RPC being available
 - When starting from pre-populated data directories where clients may need time to validate state
 
+##### Run Timeout
+
+The `run_timeout` option sets a maximum duration for the test execution phase of a run. If the timeout is exceeded, the run is cancelled with a `timed_out` status. Partial results collected before the timeout are still written and published.
+
+```yaml
+runner:
+  client:
+    config:
+      run_timeout: 2h
+```
+
+The value is a Go duration string (e.g., `30m`, `1h`, `2h30m`). If not set, no timeout is applied.
+
+The timeout covers only the test execution phase — container setup, image pulling, and RPC readiness checks are not included.
+
+**When to use:**
+- When running large test suites that may hang or take unexpectedly long
+- When you want to enforce a maximum wall-clock time for benchmark runs
+- When running in CI/CD environments with time constraints
+
 ##### Retry New Payloads Syncing State
 
 When `engine_newPayload` returns a `SYNCING` status, it indicates the client hasn't fully processed the parent block yet. The `retry_new_payloads_syncing_state` option configures automatic retries with exponential backoff.
@@ -833,6 +854,7 @@ runner:
 | `rollback_strategy` | string | No | From `runner.client.config` | Instance-specific rollback strategy |
 | `checkpoint_restore_strategy_options` | object | No | From `runner.client.config` | Instance-specific checkpoint-restore strategy options (replaces global) |
 | `wait_after_rpc_ready` | string | No | From `runner.client.config` | Instance-specific RPC ready wait duration |
+| `run_timeout` | string | No | From `runner.client.config` | Instance-specific run timeout duration |
 | `retry_new_payloads_syncing_state` | object | No | From `runner.client.config` | Instance-specific retry config for SYNCING responses |
 | `resource_limits` | object | No | From `runner.client.config` | Instance-specific resource limits |
 | `post_test_rpc_calls` | []object | No | From `runner.client.config` | Instance-specific post-test RPC calls (replaces global) |

pkg/api/indexstore/indexstore.go

@@ -327,7 +327,7 @@ func (s *store) ListRunIDs(
 }
 
 // terminalStatuses are run statuses that will not change.
-var terminalStatuses = []string{"completed", "failed", "cancelled", "container_died"}
+var terminalStatuses = []string{"completed", "failed", "cancelled", "container_died", "timeout"}
 
 // ListIncompleteRunIDs returns run IDs where the result has not been indexed
 // and the run is still potentially in progress. A run is considered

pkg/config/config.go

@@ -593,6 +593,7 @@ type ClientDefaults struct {
 	ResourceLimits                   *ResourceLimits                   `yaml:"resource_limits,omitempty" mapstructure:"resource_limits"`
 	RetryNewPayloadsSyncingState     *RetryNewPayloadsSyncingConfig    `yaml:"retry_new_payloads_syncing_state,omitempty" mapstructure:"retry_new_payloads_syncing_state"`
 	WaitAfterRPCReady                string                            `yaml:"wait_after_rpc_ready,omitempty" mapstructure:"wait_after_rpc_ready"`
+	RunTimeout                       string                            `yaml:"run_timeout,omitempty" mapstructure:"run_timeout"`
 	PostTestRPCCalls                 []PostTestRPCCall                 `yaml:"post_test_rpc_calls,omitempty" mapstructure:"post_test_rpc_calls"`
 	BootstrapFCU                     *BootstrapFCUConfig               `yaml:"bootstrap_fcu,omitempty" mapstructure:"bootstrap_fcu"`
 	CheckpointRestoreStrategyOptions *CheckpointRestoreStrategyOptions `yaml:"checkpoint_restore_strategy_options,omitempty" mapstructure:"checkpoint_restore_strategy_options"`
@@ -616,6 +617,7 @@ type ClientInstance struct {
 	ResourceLimits                   *ResourceLimits                   `yaml:"resource_limits,omitempty" mapstructure:"resource_limits"`
 	RetryNewPayloadsSyncingState     *RetryNewPayloadsSyncingConfig    `yaml:"retry_new_payloads_syncing_state,omitempty" mapstructure:"retry_new_payloads_syncing_state"`
 	WaitAfterRPCReady                string                            `yaml:"wait_after_rpc_ready,omitempty" mapstructure:"wait_after_rpc_ready"`
+	RunTimeout                       string                            `yaml:"run_timeout,omitempty" mapstructure:"run_timeout"`
 	PostTestRPCCalls                 []PostTestRPCCall                 `yaml:"post_test_rpc_calls,omitempty" mapstructure:"post_test_rpc_calls"`
 	BootstrapFCU                     *BootstrapFCUConfig               `yaml:"bootstrap_fcu,omitempty" mapstructure:"bootstrap_fcu"`
 	CheckpointRestoreStrategyOptions *CheckpointRestoreStrategyOptions `yaml:"checkpoint_restore_strategy_options,omitempty" mapstructure:"checkpoint_restore_strategy_options"`
@@ -1038,6 +1040,11 @@ func (c *Config) Validate(opts ...ValidateOpts) error {
 		return err
 	}
 
+	// Validate run_timeout settings.
+	if err := c.validateRunTimeout(); err != nil {
+		return err
+	}
+
 	// Validate post_test_rpc_calls settings.
 	if err := c.validatePostTestRPCCalls(); err != nil {
 		return err
@@ -1282,6 +1289,29 @@ func (c *Config) GetWaitAfterRPCReady(instance *ClientInstance) time.Duration {
 	return d
 }
 
+// GetRunTimeout returns the maximum duration for test execution.
+// Instance-level config takes precedence over global defaults. Returns 0 if not set.
+func (c *Config) GetRunTimeout(instance *ClientInstance) time.Duration {
+	var s string
+
+	if instance.RunTimeout != "" {
+		s = instance.RunTimeout
+	} else {
+		s = c.Runner.Client.Config.RunTimeout
+	}
+
+	if s == "" {
+		return 0
+	}
+
+	d, err := time.ParseDuration(s)
+	if err != nil {
+		return 0
+	}
+
+	return d
+}
+
 // GetPostTestRPCCalls returns the post-test RPC calls for an instance.
 // Instance-level config completely replaces the global default.
 // Returns nil if not configured at either level.
@@ -1641,6 +1671,25 @@ func (c *Config) validateWaitAfterRPCReady() error {
 	return nil
 }
 
+// validateRunTimeout validates run_timeout settings.
+func (c *Config) validateRunTimeout() error {
+	for _, instance := range c.Runner.Instances {
+		s := instance.RunTimeout
+		if s == "" {
+			s = c.Runner.Client.Config.RunTimeout
+		}
+
+		if s != "" {
+			if _, err := time.ParseDuration(s); err != nil {
+				return fmt.Errorf("instance %q: invalid run_timeout %q: %w",
+					instance.ID, s, err)
+			}
+		}
+	}
+
+	return nil
+}
+
 // validatePostTestRPCCalls validates post_test_rpc_calls settings.
 func (c *Config) validatePostTestRPCCalls() error {
 	// Validate global-level calls.

pkg/config/config_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -2370,3 +2371,125 @@ func TestValidateAPIIndexing(t *testing.T) {
 		})
 	}
 }
+
+func TestGetRunTimeout(t *testing.T) {
+	tests := []struct {
+		name     string
+		global   string
+		instance string
+		expected time.Duration
+	}{
+		{
+			name:     "empty returns zero",
+			global:   "",
+			instance: "",
+			expected: 0,
+		},
+		{
+			name:     "global value used",
+			global:   "30m",
+			instance: "",
+			expected: 30 * time.Minute,
+		},
+		{
+			name:     "instance overrides global",
+			global:   "30m",
+			instance: "1h",
+			expected: 1 * time.Hour,
+		},
+		{
+			name:     "instance only",
+			global:   "",
+			instance: "45m",
+			expected: 45 * time.Minute,
+		},
+		{
+			name:     "invalid returns zero",
+			global:   "not-a-duration",
+			instance: "",
+			expected: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := &Config{
+				Runner: RunnerConfig{
+					Client: ClientConfig{
+						Config: ClientDefaults{
+							RunTimeout: tt.global,
+						},
+					},
+				},
+			}
+			instance := &ClientInstance{
+				RunTimeout: tt.instance,
+			}
+			assert.Equal(t, tt.expected, cfg.GetRunTimeout(instance))
+		})
+	}
+}
+
+func TestValidateRunTimeout(t *testing.T) {
+	tests := []struct {
+		name      string
+		global    string
+		instance  string
+		wantErr   bool
+		errSubstr string
+	}{
+		{
+			name:     "empty is valid",
+			global:   "",
+			instance: "",
+		},
+		{
+			name:   "valid global",
+			global: "30m",
+		},
+		{
+			name:     "valid instance",
+			instance: "1h",
+		},
+		{
+			name:      "invalid global",
+			global:    "bad",
+			wantErr:   true,
+			errSubstr: "invalid run_timeout",
+		},
+		{
+			name:      "invalid instance",
+			instance:  "bad",
+			wantErr:   true,
+			errSubstr: "invalid run_timeout",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := &Config{
+				Runner: RunnerConfig{
+					Client: ClientConfig{
+						Config: ClientDefaults{
+							RunTimeout: tt.global,
+						},
+					},
+					Instances: []ClientInstance{
+						{
+							ID:         "test",
+							Client:     "geth",
+							RunTimeout: tt.instance,
+						},
+					},
+				},
+			}
+			err := cfg.validateRunTimeout()
+			if tt.wantErr {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.errSubstr)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}

pkg/runner/runner.go

@@ -115,6 +115,7 @@ const (
 	RunStatusFailed        = "failed"
 	RunStatusContainerDied = "container_died"
 	RunStatusCancelled     = "cancelled"
+	RunStatusTimedOut      = "timeout"
 )
 
 // SystemInfo contains system hardware and OS information.
@@ -181,6 +182,7 @@ type ResolvedInstance struct {
 	RollbackStrategy                 string                                   `json:"rollback_strategy,omitempty"`
 	DropMemoryCaches                 string                                   `json:"drop_memory_caches,omitempty"`
 	WaitAfterRPCReady                string                                   `json:"wait_after_rpc_ready,omitempty"`
+	RunTimeout                       string                                   `json:"run_timeout,omitempty"`
 	RetryNewPayloadsSyncingState     *config.RetryNewPayloadsSyncingConfig    `json:"retry_new_payloads_syncing_state,omitempty"`
 	ResourceLimits                   *ResolvedResourceLimits                  `json:"resource_limits,omitempty"`
 	PostTestRPCCalls                 []config.PostTestRPCCall                 `json:"post_test_rpc_calls,omitempty"`
@@ -1040,6 +1042,16 @@ func (r *runner) runContainerLifecycle(
 		}
 	}
 
+	// Resolve run_timeout for config output and timeout enforcement.
+	var runTimeoutStr string
+	var runTimeout time.Duration
+	if r.cfg.FullConfig != nil {
+		runTimeout = r.cfg.FullConfig.GetRunTimeout(instance)
+		if runTimeout > 0 {
+			runTimeoutStr = runTimeout.String()
+		}
+	}
+
 	// Write run configuration with resolved values.
 	runConfig := &RunConfig{
 		Timestamp: params.RunTimestamp,
@@ -1070,6 +1082,7 @@ func (r *runner) runContainerLifecycle(
 			}(),
 			DropMemoryCaches:  dropMemoryCaches,
 			WaitAfterRPCReady: waitAfterRPCReadyStr,
+			RunTimeout:        runTimeoutStr,
 			RetryNewPayloadsSyncingState: func() *config.RetryNewPayloadsSyncingConfig {
 				if r.cfg.FullConfig != nil {
 					return r.cfg.FullConfig.GetRetryNewPayloadsSyncingState(instance)
@@ -1232,8 +1245,19 @@ func (r *runner) runContainerLifecycle(
 
 	log.Info("Container started")
 
+	// Apply run timeout if configured.
+	testCtx := ctx
+	var timeoutCancel context.CancelFunc
+
+	if runTimeout > 0 {
+		testCtx, timeoutCancel = context.WithTimeout(ctx, runTimeout)
+		defer timeoutCancel()
+
+		log.WithField("timeout", runTimeout).Info("Run timeout configured")
+	}
+
 	// Start container death monitoring.
-	execCtx, execCancel := context.WithCancel(ctx)
+	execCtx, execCancel := context.WithCancel(testCtx)
 	defer execCancel()
 
 	var containerDied bool
@@ -1460,14 +1484,14 @@ func (r *runner) runContainerLifecycle(
 			switch rollbackStrategy {
 			case config.RollbackStrategyCheckpointRestore:
 				result, execErr = r.runTestsWithCheckpointRestore(
-					ctx, params, spec, containerID, containerIP,
+					testCtx, params, spec, containerID, containerIP,
 					dropMemoryCaches, dropCachesPath,
 					runResultsDir, &logCancel, &logDone, benchmarkoorLogFile,
 					&localCleanupFuncs, localCleanupStarted,
 				)
 			default:
 				result, execErr = r.runTestsWithContainerStrategy(
-					ctx, params, spec, containerID, containerIP,
+					testCtx, params, spec, containerID, containerIP,
 					rollbackStrategy, dropMemoryCaches, dropCachesPath,
 					runResultsDir, &logCancel, &logDone, benchmarkoorLogFile,
 					&localCleanupFuncs, localCleanupStarted,
@@ -1569,8 +1593,13 @@ func (r *runner) runContainerLifecycle(
 	}
 
 	// Determine final run status (don't overwrite if already set by executor).
+	// Timeout is checked first because when the deadline fires, the context
+	// cancellation can cause the container to stop, which sets containerDied.
 	mu.Lock()
-	if containerDied {
+	if timeoutCancel != nil && testCtx.Err() == context.DeadlineExceeded {
+		runConfig.Status = RunStatusTimedOut
+		runConfig.TerminationReason = fmt.Sprintf("the run_timeout of %s was reached", runTimeout)
+	} else if containerDied {
 		runConfig.Status = RunStatusContainerDied
 		runConfig.TerminationReason = "container exited during test execution"
 		runConfig.ContainerExitCode = containerExitCode

ui/src/api/types.ts

@@ -5,7 +5,7 @@ export interface Index {
 }
 
 // Run status type
-export type RunStatus = 'completed' | 'container_died' | 'cancelled'
+export type RunStatus = 'completed' | 'container_died' | 'cancelled' | 'timeout'
 
 export interface IndexEntry {
   run_id: string
@@ -235,6 +235,7 @@ export interface InstanceConfig {
   rollback_strategy?: string
   drop_memory_caches?: string
   wait_after_rpc_ready?: string
+  run_timeout?: string
   retry_new_payloads_syncing_state?: RetryNewPayloadsSyncingConfig
   resource_limits?: ResourceLimitsConfig
   post_test_rpc_calls?: PostTestRPCCallConfig[]