feat(api): soft-delete build layers in DB on user delete (#3121)

This switches user delete to soft-delete the build layers in DB:
- New `deleted` build status, mapped to a dedicated `deleted`
status_group (migration updates `compute_status_group`).
- New `MarkExclusiveTemplateBuildsDeleted` query that marks builds
assigned only to this template as `deleted`, reusing the exclusivity
predicate so builds shared with other templates are untouched.
- Template delete and snapshot/paused delete now run the mark + delete
in one transaction. The `env_builds` rows (and their team/env
attribution) are preserved so a future GC can reclaim storage. Mirrors
how `failed` keeps attribution.

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: ValentaTomas

packages/api/internal/cache/templates/cache.go

@@ -175,7 +175,7 @@ func (c *TemplateCache) fetchTemplateWithBuild(templateID string, tag *string) f
 		}
 
 		build := &result.EnvBuild
-		template := result.Env
+		template := result.ActiveEnv
 		clusterID := clusters.WithClusterFallback(template.ClusterID)
 
 		tagValue := sharedUtils.DerefOrDefault(tag, id.DefaultTag)

packages/api/internal/cache/templates/template_build.go

@@ -84,12 +84,12 @@ func (c *TemplatesBuildCache) fetchFromDB(templateID string, buildID uuid.UUID)
 		}
 
 		return TemplateBuildInfo{
-			TeamID:      result.Env.TeamID,
-			TemplateID:  result.Env.ID,
+			TeamID:      result.ActiveEnv.TeamID,
+			TemplateID:  result.ActiveEnv.ID,
 			BuildStatus: result.EnvBuild.StatusGroup,
 			Reason:      result.EnvBuild.Reason,
 			Version:     result.EnvBuild.Version,
-			ClusterID:   clusters.WithClusterFallback(result.Env.ClusterID),
+			ClusterID:   clusters.WithClusterFallback(result.ActiveEnv.ClusterID),
 			NodeID:      result.EnvBuild.ClusterNodeID,
 		}, nil
 	}

packages/api/internal/handlers/deprecated_template_request_build.go

@@ -107,7 +107,7 @@ func (a *APIStore) PostTemplatesTemplateID(c *gin.Context, rawTemplateID api.Tem
 		return
 	}
 
-	templateDB, err := a.sqlcDB.GetTemplateByID(ctx, templateID)
+	templateDB, err := a.sqlcDB.GetTemplateById(ctx, templateID)
 	switch {
 	case err == nil:
 		if templateDB.TeamID != team.ID {

packages/api/internal/handlers/deprecated_template_start_build.go

@@ -120,7 +120,7 @@ func (a *APIStore) PostTemplatesTemplateIDBuildsBuildID(c *gin.Context, template
 	var team *types.Team
 	// Check if the user has access to the template
 	for _, t := range teams {
-		if t.Team.ID == templateBuildDB.Env.TeamID {
+		if t.Team.ID == templateBuildDB.ActiveEnv.TeamID {
 			team = t.Team
 
 			break

packages/api/internal/handlers/sandbox_kill.go

@@ -14,7 +14,6 @@ import (
 	"github.com/e2b-dev/infra/packages/api/internal/sandbox"
 	"github.com/e2b-dev/infra/packages/api/internal/utils"
 	"github.com/e2b-dev/infra/packages/auth/pkg/auth"
-	"github.com/e2b-dev/infra/packages/db/queries"
 	"github.com/e2b-dev/infra/packages/shared/pkg/logger"
 	"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
 )
@@ -25,10 +24,7 @@ func (a *APIStore) deleteSnapshot(ctx context.Context, sandboxID string, teamID
 		return err
 	}
 
-	aliasKeys, dbErr := a.sqlcDB.DeleteTemplate(ctx, queries.DeleteTemplateParams{
-		TeamID:     teamID,
-		TemplateID: snapshot.TemplateID,
-	})
+	aliasKeys, dbErr := a.softDeleteTemplate(ctx, teamID, snapshot.TemplateID)
 	if dbErr != nil {
 		return fmt.Errorf("error deleting template from db: %w", dbErr)
 	}

packages/api/internal/handlers/template_delete.go

@@ -6,16 +6,54 @@ import (
 	"net/http"
 
 	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
 	"go.opentelemetry.io/otel/attribute"
 
 	"github.com/e2b-dev/infra/packages/api/internal/api"
 	"github.com/e2b-dev/infra/packages/api/internal/sandbox"
+	"github.com/e2b-dev/infra/packages/db/pkg/dberrors"
 	"github.com/e2b-dev/infra/packages/db/queries"
 	"github.com/e2b-dev/infra/packages/shared/pkg/id"
 	"github.com/e2b-dev/infra/packages/shared/pkg/logger"
 	"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
 )
 
+// softDeleteTemplate soft-deletes the env, releases its aliases, and clears its
+// active build rows, in a transaction. The env-locking soft-delete runs first so
+// a concurrent build registration commits before the alias/active-build cleanup,
+// whose fresh per-statement snapshots then see those rows. Returns the released
+// alias cache keys.
+func (a *APIStore) softDeleteTemplate(ctx context.Context, teamID uuid.UUID, templateID string) ([]string, error) {
+	txDB, tx, err := a.sqlcDB.WithTx(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("begin tx: %w", err)
+	}
+	defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
+
+	if _, err := txDB.SoftDeleteTemplate(ctx, queries.SoftDeleteTemplateParams{TemplateID: templateID, TeamID: teamID}); err != nil {
+		if dberrors.IsNotFoundError(err) {
+			return nil, nil // already deleted or not owned by the team
+		}
+
+		return nil, fmt.Errorf("soft delete template: %w", err)
+	}
+
+	aliasKeys, err := txDB.ReleaseTemplateAliases(ctx, templateID)
+	if err != nil {
+		return nil, fmt.Errorf("release aliases: %w", err)
+	}
+
+	if err := txDB.DeleteActiveTemplateBuilds(ctx, templateID); err != nil {
+		return nil, fmt.Errorf("delete active builds: %w", err)
+	}
+
+	if err := tx.Commit(ctx); err != nil {
+		return nil, fmt.Errorf("commit: %w", err)
+	}
+
+	return aliasKeys, nil
+}
+
 // DeleteTemplatesTemplateID serves to delete a template (e.g. in CLI)
 func (a *APIStore) DeleteTemplatesTemplateID(c *gin.Context, aliasOrTemplateID api.TemplateID) {
 	ctx := c.Request.Context()
@@ -80,15 +118,7 @@ func (a *APIStore) DeleteTemplatesTemplateID(c *gin.Context, aliasOrTemplateID a
 		return
 	}
 
-	// Delete the template from DB (cascades to env_build_assignments, env_aliases, snapshot_templates).
-	// Returns alias cache keys captured before cascade deletion for cache invalidation.
-	// Build artifacts are intentionally NOT deleted from storage here because builds are layered diffs
-	// that may be referenced by other builds' header mappings.
-	// [ENG-3477] a future GC mechanism will handle orphaned storage.
-	aliasKeys, err := a.sqlcDB.DeleteTemplate(ctx, queries.DeleteTemplateParams{
-		TemplateID: templateID,
-		TeamID:     team.ID,
-	})
+	aliasKeys, err := a.softDeleteTemplate(ctx, team.ID, templateID)
 	if err != nil {
 		telemetry.ReportCriticalError(ctx, "error when deleting template from db", err)
 		a.sendAPIStoreError(c, http.StatusInternalServerError, "Error when deleting template")

packages/api/internal/handlers/template_layer_files_upload.go

@@ -26,8 +26,8 @@ func (a *APIStore) GetTemplatesTemplateIDFilesHash(c *gin.Context, templateID ap
 		return
 	}
 
-	// Check if the user has access to the template
-	templateDB, err := a.sqlcDB.GetTemplateByID(ctx, templateID)
+	// Resolve via the active-envs view so a soft-deleted template is not found.
+	templateDB, err := a.sqlcDB.GetTemplateById(ctx, templateID)
 	if err != nil {
 		a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Error when getting template: %s", err))
 		telemetry.ReportCriticalError(ctx, "error when getting env", err, telemetry.WithTemplateID(templateID))

packages/api/internal/handlers/template_start_build_v2.go

@@ -71,7 +71,7 @@ func (a *APIStore) PostV2TemplatesTemplateIDBuildsBuildID(c *gin.Context, templa
 		return
 	}
 
-	dbTeamID := templateBuildDB.Env.TeamID.String()
+	dbTeamID := templateBuildDB.ActiveEnv.TeamID.String()
 	team, apiErr := a.GetTeam(ctx, c, &dbTeamID)
 	if apiErr != nil {
 		a.sendAPIStoreError(c, apiErr.Code, apiErr.ClientMsg)
@@ -80,7 +80,7 @@ func (a *APIStore) PostV2TemplatesTemplateIDBuildsBuildID(c *gin.Context, templa
 		return
 	}
 
-	if team.ID != templateBuildDB.Env.TeamID {
+	if team.ID != templateBuildDB.ActiveEnv.TeamID {
 		a.sendAPIStoreError(c, http.StatusForbidden, "User does not have access to the template")
 
 		telemetry.ReportCriticalError(ctx, "user does not have access to the template", err, telemetry.WithTemplateID(templateID))

packages/api/internal/handlers/template_tags.go

@@ -103,7 +103,7 @@ func (a *APIStore) PostTemplatesTags(c *gin.Context) {
 		return
 	}
 
-	template := result.Env
+	template := result.ActiveEnv
 	buildID := result.EnvBuild.ID
 
 	telemetry.SetAttributes(ctx,
@@ -129,7 +129,7 @@ func (a *APIStore) PostTemplatesTags(c *gin.Context) {
 
 	// Create the tag assignments
 	for _, tag := range tags {
-		err = client.CreateTemplateBuildAssignment(ctx, queries.CreateTemplateBuildAssignmentParams{
+		rows, err := client.CreateTemplateBuildAssignment(ctx, queries.CreateTemplateBuildAssignmentParams{
 			TemplateID: template.ID,
 			BuildID:    buildID,
 			Tag:        tag,
@@ -138,6 +138,11 @@ func (a *APIStore) PostTemplatesTags(c *gin.Context) {
 			telemetry.ReportCriticalError(ctx, "error when creating tag assignment", err)
 			a.sendAPIStoreError(c, http.StatusInternalServerError, "Error creating tag assignment")
 
+			return
+		}
+		if rows == 0 {
+			a.sendAPIStoreError(c, http.StatusNotFound, fmt.Sprintf("Template '%s' not found", template.ID))
+
 			return
 		}
 	}

packages/api/internal/handlers/templates_list.go

@@ -69,19 +69,19 @@ func (a *APIStore) GetTemplates(c *gin.Context, params api.GetTemplatesParams) {
 		}
 
 		templates = append(templates, &api.Template{
-			TemplateID:    item.Env.ID,
+			TemplateID:    item.ActiveEnv.ID,
 			BuildID:       item.BuildID.String(),
 			CpuCount:      api.CPUCount(item.BuildVcpu),
 			MemoryMB:      api.MemoryMB(item.BuildRamMb),
 			DiskSizeMB:    api.DiskSizeMB(diskMB),
-			Public:        item.Env.Public,
+			Public:        item.ActiveEnv.Public,
 			Aliases:       item.Aliases,
 			Names:         item.Names,
-			CreatedAt:     item.Env.CreatedAt,
-			UpdatedAt:     item.Env.UpdatedAt,
-			LastSpawnedAt: item.Env.LastSpawnedAt,
-			SpawnCount:    item.Env.SpawnCount,
-			BuildCount:    item.Env.BuildCount,
+			CreatedAt:     item.ActiveEnv.CreatedAt,
+			UpdatedAt:     item.ActiveEnv.UpdatedAt,
+			LastSpawnedAt: item.ActiveEnv.LastSpawnedAt,
+			SpawnCount:    item.ActiveEnv.SpawnCount,
+			BuildCount:    item.ActiveEnv.BuildCount,
 			BuildStatus:   getCorrespondingTemplateBuildStatus(ctx, item.BuildStatus),
 			CreatedBy:     createdBy,
 			EnvdVersion:   envdVersion,