feat(orchestrator): classify execution end as killed, paused, or crashed

Record orchestrator.sandbox.execution.duration from a single termination
observer instead of inline in Delete and Pause, so abnormal exits are
captured too. The end_reason label now distinguishes killed (the kill
reason), paused, and crashed.

Add Metadata.endReason (guarded like startedAt/endAt) set by the teardown
initiator: Delete sets the kill reason, Pause sets "pause", and Checkpoint
marks the old sandbox "checkpoint" so its in-place hand-off is not counted.
setupSandboxLifecycle, the one goroutine that observes every sandbox's exit,
reads the reason and records the sample; an unset reason with a non-nil exit
error is classified as crashed, otherwise unknown.

Author: jakubno

packages/orchestrator/pkg/sandbox/sandbox.go

@@ -222,9 +222,39 @@ type Metadata struct {
 	Config         *Config
 	Runtime        RuntimeMetadata
 
-	rwmu      sync.RWMutex // protects startedAt, endAt
+	rwmu      sync.RWMutex // protects startedAt, endAt, endReason
 	startedAt time.Time
 	endAt     time.Time
+
+	// endReason records why the execution ended (e.g. a kill reason, "pause",
+	// or "checkpoint"). It is set by whoever initiates the teardown and read at
+	// the single termination observer (setupSandboxLifecycle) to label the
+	// execution-duration metric. An empty value means the sandbox terminated
+	// without an initiated teardown — i.e. it crashed.
+	endReason string
+}
+
+// GetEndReason returns the reason the execution ended in a thread-safe manner.
+// An empty string means no teardown reason was set (the sandbox crashed).
+func (m *Metadata) GetEndReason() string {
+	m.rwmu.RLock()
+	defer m.rwmu.RUnlock()
+
+	return m.endReason
+}
+
+// SetEndReason records why the execution ended in a thread-safe manner. The
+// first non-empty reason wins; later calls are ignored so an initiated teardown
+// (kill/pause/checkpoint) is never overwritten by a subsequent stop.
+func (m *Metadata) SetEndReason(reason string) {
+	m.rwmu.Lock()
+	defer m.rwmu.Unlock()
+
+	if m.endReason != "" {
+		return
+	}
+
+	m.endReason = reason
 }
 
 // GetEndAt returns the sandbox end time in a thread-safe manner.

packages/orchestrator/pkg/server/sandboxes.go

@@ -532,6 +532,11 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR
 
 	sbxlogger.E(sbx).Info(ctx, "Killing sandbox", zap.String("kill_reason", killReason))
 
+	// Record why the execution is ending before triggering the stop, so the
+	// lifecycle observer reads the kill reason rather than treating it as a
+	// crash. The execution-duration sample is emitted there, not here.
+	sbx.SetEndReason(killReason)
+
 	// Check health metrics before stopping the sandbox
 	sbx.Checks.Healthcheck(ctx, true)
 
@@ -552,7 +557,6 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR
 	eventData[executionEventDataKey] = s.getSandboxExecutionData(sbx)
 	addKillReason(eventData, killReason)
 	recordSandboxKill(ctx, s.sandboxKilledCounter, killReason)
-	s.recordExecutionDuration(ctx, sbx, killReason)
 
 	eventType := events.SandboxKilledEventPair
 	go s.sbxEventsService.Publish(
@@ -595,14 +599,47 @@ func recordSandboxKill(ctx context.Context, counter metric.Int64Counter, killRea
 	counter.Add(ctx, 1, metric.WithAttributes(attribute.String("kill_reason", killReason)))
 }
 
-// endReasonPause labels execution-duration samples for executions that ended
-// because the sandbox was paused rather than killed.
-const endReasonPause = "pause"
+const (
+	// endReasonPause labels execution-duration samples for executions that
+	// ended because the sandbox was paused.
+	endReasonPause = "pause"
+
+	// endReasonCrashed labels executions that ended without an initiated
+	// teardown — the Firecracker process exited on its own (crash, OOM, host
+	// failure) and no kill/pause set an end reason.
+	endReasonCrashed = "crashed"
+
+	// endReasonCheckpoint marks the old sandbox object replaced by an
+	// in-place checkpoint resume. The ExecutionID continues on the resumed
+	// sandbox, so this stop is not an execution boundary and must not emit a
+	// duration sample.
+	endReasonCheckpoint = "checkpoint"
+)
+
+// resolveEndReason maps a sandbox's recorded teardown reason to the
+// execution-duration end_reason label. An unset reason means no teardown was
+// initiated: a non-nil exit error is a crash, otherwise it is unknown.
+func resolveEndReason(reason string, exitErr error) string {
+	if reason != "" {
+		return reason
+	}
+
+	if exitErr != nil {
+		return endReasonCrashed
+	}
+
+	return killReasonUnknown
+}
 
 // recordExecutionDuration records the duration of a single sandbox execution
 // (start/resume until pause or kill) tagged with the reason the execution
-// ended. For kills this is the kill reason; for pauses it is endReasonPause.
+// ended.
 func (s *Server) recordExecutionDuration(ctx context.Context, sbx *sandbox.Sandbox, endReason string) {
+	if endReason == endReasonCheckpoint {
+		// An in-place checkpoint resume is not an execution boundary.
+		return
+	}
+
 	startedAt := sbx.GetStartedAt()
 	if startedAt.IsZero() {
 		// A zero start time means the sandbox never finished starting (e.g. it
@@ -664,6 +701,11 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest
 
 	sbxlogger.E(sbx).Info(ctx, "Pausing sandbox")
 
+	// Mark this execution as ending in a pause before the stop is triggered, so
+	// the lifecycle observer records it as paused rather than crashed. The
+	// execution-duration sample is emitted there, not here.
+	sbx.SetEndReason(endReasonPause)
+
 	// Stop the old sandbox in background after we're done
 	defer s.stopSandboxAsync(context.WithoutCancel(ctx), sbx)
 
@@ -677,8 +719,6 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest
 
 	s.uploadSnapshotAsync(ctx, sbx, res)
 
-	s.recordExecutionDuration(ctx, sbx, endReasonPause)
-
 	teamID, buildId, eventData := s.prepareSandboxEventData(ctx, sbx)
 	eventData[executionEventDataKey] = s.getSandboxExecutionData(sbx)
 
@@ -818,6 +858,14 @@ func (s *Server) Checkpoint(ctx context.Context, in *orchestrator.SandboxCheckpo
 	// after the last checkpoint.
 	resumedSbx.SetStartedAt(sbx.GetStartedAt())
 
+	// The resume succeeded and resumedSbx now carries this execution. Mark the
+	// old sandbox object as a checkpoint hand-off so its impending stop is not
+	// recorded as an execution end (or a crash) — the execution continues under
+	// the same ExecutionID and will be recorded when resumedSbx is paused or
+	// killed. On checkpoint failure we return before this point, so the old
+	// sandbox's abnormal stop is still recorded as a crash.
+	sbx.SetEndReason(endReasonCheckpoint)
+
 	// Collect prefetch data immediately after resume while it's most accurate
 	prefetchData, prefetchErr := resumedSbx.MemoryPrefetchData(ctx)
 	if prefetchErr != nil {
@@ -1063,6 +1111,12 @@ func (s *Server) setupSandboxLifecycle(ctx context.Context, sbx *sandbox.Sandbox
 			sbxlogger.I(sbx).Error(ctx, "failed to wait for sandbox, cleaning up", zap.Error(waitErr))
 		}
 
+		// This is the single observer of every execution end (kill, pause, or
+		// crash), so record the execution duration here. An unset end reason
+		// means no teardown was initiated: derive killed/paused from the reason
+		// set by Delete/Pause, otherwise classify by the exit error as crashed.
+		s.recordExecutionDuration(ctx, sbx, resolveEndReason(sbx.GetEndReason(), waitErr))
+
 		cleanupErr := sbx.Close(ctx)
 		if cleanupErr != nil {
 			sbxlogger.I(sbx).Error(ctx, "failed to cleanup sandbox, will remove from cache", zap.Error(cleanupErr))

packages/orchestrator/pkg/server/sandboxes_test.go

@@ -4,6 +4,7 @@ package server
 
 import (
 	"context"
+	"errors"
 	"net"
 	"reflect"
 	"testing"
@@ -212,12 +213,17 @@ func TestRecordExecutionDuration(t *testing.T) {
 
 	s.recordExecutionDuration(context.Background(), newSbx(), "timeout")
 	s.recordExecutionDuration(context.Background(), newSbx(), endReasonPause)
+	s.recordExecutionDuration(context.Background(), newSbx(), endReasonCrashed)
 	s.recordExecutionDuration(context.Background(), newSbx(), "")
 
 	// A sandbox that never finished starting has a zero start time and must not
 	// emit a sample (it would be a massive time.Since(zero) outlier).
 	s.recordExecutionDuration(context.Background(), &sandbox.Sandbox{Metadata: &sandbox.Metadata{}}, "timeout")
 
+	// An in-place checkpoint hand-off is not an execution boundary and must not
+	// emit a sample.
+	s.recordExecutionDuration(context.Background(), newSbx(), endReasonCheckpoint)
+
 	var rm metricdata.ResourceMetrics
 	require.NoError(t, reader.Collect(context.Background(), &rm))
 
@@ -241,5 +247,22 @@ func TestRecordExecutionDuration(t *testing.T) {
 
 	assert.Equal(t, uint64(1), got["timeout"])
 	assert.Equal(t, uint64(1), got[endReasonPause])
+	assert.Equal(t, uint64(1), got[endReasonCrashed])
 	assert.Equal(t, uint64(1), got[killReasonUnknown])
+	assert.Zero(t, got[endReasonCheckpoint])
+}
+
+func TestResolveEndReason(t *testing.T) {
+	t.Parallel()
+
+	someErr := errors.New("fc exited")
+
+	// An explicit reason always wins, regardless of the exit error.
+	assert.Equal(t, "timeout", resolveEndReason("timeout", nil))
+	assert.Equal(t, endReasonPause, resolveEndReason(endReasonPause, someErr))
+	assert.Equal(t, endReasonCheckpoint, resolveEndReason(endReasonCheckpoint, someErr))
+
+	// No reason set: a non-nil exit error is a crash, otherwise unknown.
+	assert.Equal(t, endReasonCrashed, resolveEndReason("", someErr))
+	assert.Equal(t, killReasonUnknown, resolveEndReason("", nil))
 }