fix: template build panic handling (#1289)

Author: dobrac

packages/orchestrator/internal/template/build/builder.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"golang.org/x/sync/errgroup"
@@ -32,6 +33,7 @@ import (
 	"github.com/e2b-dev/infra/packages/shared/pkg/smap"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage"
 	"github.com/e2b-dev/infra/packages/shared/pkg/storage/header"
+	"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
 )
 
 const progressDelay = 5 * time.Second
@@ -134,6 +136,13 @@ func (b *Builder) Build(ctx context.Context, template storage.TemplateFiles, con
 		}
 	}()
 
+	defer func() {
+		if r := recover(); r != nil {
+			telemetry.ReportCriticalError(ctx, "recovered from panic in template build", nil, attribute.String("panic", fmt.Sprintf("%v", r)), telemetry.WithTemplateID(config.TemplateID), telemetry.WithBuildID(template.BuildID))
+			e = errors.New("fatal error occurred during template build, please contact us")
+		}
+	}()
+
 	if isV1Build {
 		hookedCore, done := writer.NewPostProcessor(progressDelay, logsCore)
 		defer done()

packages/orchestrator/internal/template/server/create_template.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"go.opentelemetry.io/otel/attribute"
@@ -98,7 +99,8 @@ func (s *ServerStore) TemplateCreate(ctx context.Context, templateRequest *templ
 
 		defer func() {
 			if r := recover(); r != nil {
-				zap.L().Error("recovered from panic in template build", zap.Any("panic", r), logger.WithTemplateID(cfg.TemplateID), logger.WithBuildID(cfg.BuildID))
+				telemetry.ReportCriticalError(ctx, "recovered from panic in template build handler", nil, attribute.String("panic", fmt.Sprintf("%v", r)), telemetry.WithTemplateID(cfg.TemplateID), telemetry.WithBuildID(cfg.BuildID))
+				buildInfo.SetFail(builderrors.UnwrapUserError(errors.New("fatal error occurred, please contact us")))
 			}
 		}()