Skip to content

Commit 5a5830f

Browse files
committed
fix(orchestrator): include phase/step context and original error chain in build cancellation messages
Previously, when a build was cancelled or timed out, the user-facing error message was always the generic "build was cancelled" or "build timed out", regardless of the underlying cause. This made it very difficult to diagnose why a build actually failed (e.g. envd crash due to GLIBC incompatibility, VM process exit, timeout during a specific phase). Changes: - PhaseBuildError.Error() now includes phase and step context when available, e.g. "context canceled (phase: provision, step: 1)" - phases.Run() annotates context cancellation/timeout errors with the metadata of the phase that was active when the interruption occurred - WrapContextAsUserError() preserves the original error chain instead of replacing it, so diagnostic details like "wait for envd: failed to init new envd" are visible to the user - Added sanitizeErrorMessage() to handle newlines from errors.Join Before: Build failed: build was cancelled After: Build failed: build was cancelled: wait for envd: failed to init new envd: context canceled or: Build failed: context canceled (phase: provision, step: 1)
1 parent f046fcf commit 5a5830f

3 files changed

Lines changed: 106 additions & 12 deletions

File tree

packages/orchestrator/pkg/template/build/builderrors/errors.go

Lines changed: 60 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ package builderrors
55
import (
66
"context"
77
"errors"
8+
"fmt"
9+
"strings"
810

911
"github.com/e2b-dev/infra/packages/orchestrator/pkg/template/build/phases"
1012
template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager"
@@ -28,29 +30,80 @@ func IsUserError(err error) bool {
2830

2931
// WrapContextAsUserError wraps context.Canceled as a user error if no user error already exists.
3032
// This ensures that user-initiated cancellations are tracked as user errors in metrics.
33+
//
34+
// When the error already contains a PhaseBuildError (e.g. annotated by phases.Run
35+
// with the active phase/step), the phase metadata is extracted and re-wrapped with
36+
// the standardized ErrCanceled/ErrTimeout sentinel so that errors.Is still works.
37+
// The original error chain is preserved via a sanitizedError wrapper that cleans
38+
// up newlines in Error() while keeping Unwrap() intact.
3139
func WrapContextAsUserError(err error) error {
3240
if err == nil {
3341
return nil
3442
}
3543

36-
// If it's already a user error, return as-is
37-
if IsUserError(err) {
38-
return err
44+
// Extract phase metadata if present, so we can re-wrap with the standard sentinel.
45+
// When err is exactly the PhaseBuildError (not wrapped by an outer error like
46+
// errors.Join), we unwrap to pbe.Err to avoid duplicating phase/step in the
47+
// message. When err has outer wrapping context, we keep the full err so that
48+
// diagnostic details are not lost.
49+
var phase, step string
50+
errToWrap := err
51+
if pbe := phases.UnwrapPhaseBuildError(err); pbe != nil {
52+
phase = pbe.Phase
53+
step = pbe.Step
54+
// Only unwrap when err is the PhaseBuildError itself; if there is
55+
// outer wrapping (e.g. errors.Join), preserve the full context.
56+
if err == error(pbe) {
57+
errToWrap = pbe.Err
58+
}
59+
// For non-cancel/timeout user errors, return as-is.
60+
if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
61+
return err
62+
}
63+
}
64+
65+
// wrapWithPhase creates a PhaseBuildError preserving any phase/step context
66+
// extracted above, with the given sentinel wrapping the original error chain.
67+
wrapWithPhase := func(sentinel error) error {
68+
return &phases.PhaseBuildError{
69+
Phase: phase,
70+
Step: step,
71+
Err: fmt.Errorf("%w: %w", sentinel, sanitizedError{errToWrap}),
72+
}
3973
}
4074

41-
// If it's a canceled context, wrap it as a user error
75+
// If it's a canceled context, wrap it as a user error while preserving the
76+
// original error chain for diagnostics.
4277
if errors.Is(err, context.Canceled) {
43-
return phases.NewPhaseBuildError(phases.PhaseMeta{}, ErrCanceled)
78+
return wrapWithPhase(ErrCanceled)
4479
}
4580

46-
// If it's a timeout context, wrap it as a user error
81+
// If it's a timeout context, wrap it as a user error while preserving the
82+
// original error chain for diagnostics.
4783
if errors.Is(err, context.DeadlineExceeded) {
48-
return phases.NewPhaseBuildError(phases.PhaseMeta{}, ErrTimeout)
84+
return wrapWithPhase(ErrTimeout)
4985
}
5086

5187
return err
5288
}
5389

90+
// sanitizedError wraps an error, replacing newlines in Error() output
91+
// (e.g. from errors.Join) while preserving the original error chain via Unwrap.
92+
type sanitizedError struct {
93+
err error
94+
}
95+
96+
func (e sanitizedError) Error() string {
97+
if e.err == nil {
98+
return ""
99+
}
100+
return strings.ReplaceAll(e.err.Error(), "\n", "; ")
101+
}
102+
103+
func (e sanitizedError) Unwrap() error {
104+
return e.err
105+
}
106+
54107
func UnwrapUserError(err error) *template_manager.TemplateBuildStatusReason {
55108
phaseBuildError := phases.UnwrapPhaseBuildError(err)
56109
if phaseBuildError != nil {

packages/orchestrator/pkg/template/build/phases/errors.go

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ package phases
44

55
import (
66
"errors"
7+
"fmt"
78
)
89

910
type PhaseBuildError struct {
@@ -13,7 +14,27 @@ type PhaseBuildError struct {
1314
}
1415

1516
func (e *PhaseBuildError) Error() string {
16-
return e.Err.Error()
17+
if e.Err == nil {
18+
if e.Phase == "" && e.Step == "" {
19+
return "unknown build error"
20+
}
21+
if e.Step == "" {
22+
return fmt.Sprintf("build error (phase: %s)", e.Phase)
23+
}
24+
return fmt.Sprintf("build error (phase: %s, step: %s)", e.Phase, e.Step)
25+
}
26+
27+
// If the inner error chain already contains a PhaseBuildError, skip
28+
// appending phase/step to avoid duplicate metadata in the message.
29+
if (e.Phase == "" && e.Step == "") || UnwrapPhaseBuildError(e.Err) != nil {
30+
return e.Err.Error()
31+
}
32+
33+
if e.Step == "" {
34+
return fmt.Sprintf("%s (phase: %s)", e.Err.Error(), e.Phase)
35+
}
36+
37+
return fmt.Sprintf("%s (phase: %s, step: %s)", e.Err.Error(), e.Phase, e.Step)
1738
}
1839

1940
func (e *PhaseBuildError) Unwrap() error {

packages/orchestrator/pkg/template/build/phases/phase.go

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,26 @@ func Run(
7575

7676
sourceLayer := LayerResult{}
7777

78+
// wrapContextErr annotates a context cancellation/timeout error with the
79+
// phase metadata that was active when the error occurred, so that the
80+
// user-facing message includes which build step was interrupted.
81+
wrapContextErr := func(err error, meta PhaseMeta) error {
82+
if err == nil {
83+
return nil
84+
}
85+
86+
// Already a PhaseBuildError — keep it as-is.
87+
if UnwrapPhaseBuildError(err) != nil {
88+
return err
89+
}
90+
91+
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
92+
return NewPhaseBuildError(meta, err)
93+
}
94+
95+
return err
96+
}
97+
7898
for _, builder := range builders {
7999
meta := builder.Metadata()
80100

@@ -91,19 +111,19 @@ func Run(
91111
phaseStartTime := time.Now()
92112
hash, err := builder.Hash(ctx, sourceLayer)
93113
if err != nil {
94-
return LayerResult{}, fmt.Errorf("getting hash: %w", err)
114+
return LayerResult{}, wrapContextErr(fmt.Errorf("getting hash: %w", err), meta)
95115
}
96116

97117
currentLayer, err := builder.Layer(ctx, sourceLayer, hash)
98118
if err != nil {
99-
return LayerResult{}, fmt.Errorf("getting layer: %w", err)
119+
return LayerResult{}, wrapContextErr(fmt.Errorf("getting layer: %w", err), meta)
100120
}
101121
metrics.RecordCacheResult(ctx, meta.Phase, meta.StepType, currentLayer.Cached)
102122

103123
prefix := builder.Prefix()
104124
source, err := builder.String(ctx)
105125
if err != nil {
106-
return LayerResult{}, fmt.Errorf("getting source: %w", err)
126+
return LayerResult{}, wrapContextErr(fmt.Errorf("getting source: %w", err), meta)
107127
}
108128
stepUserLogger.Info(ctx, layerInfo(currentLayer.Cached, prefix, source, currentLayer.Hash))
109129

@@ -127,7 +147,7 @@ func Run(
127147
metrics.RecordPhaseDuration(ctx, phaseDuration, meta.Phase, meta.StepType, false)
128148

129149
if err != nil {
130-
return LayerResult{}, err
150+
return LayerResult{}, wrapContextErr(err, meta)
131151
}
132152

133153
sourceLayer = res

0 commit comments

Comments
 (0)