feat(orchestrator): improve concurrent benchmark tracing and enable huge pages (#2327)

* feat(orchestrator): add concurrency and sandbox attributes to benchmark spans

Add concurrency level, sandbox index, and sandbox ID attributes to the
bench-resume span so traces can be filtered by concurrency level in
Grafana/Tempo (e.g. {span.concurrency=5}).

* feat(orchestrator): enable huge pages by default in concurrent benchmark

Production uses huge pages, so the benchmark should too. Disable with
DISABLE_HUGE_PAGES=true for comparison. Uses a separate build ID per
mode to avoid cache collisions.

* feat(orchestrator): add spans for uffd socket and rootfs waits in resume-fc

During concurrent sandbox creation, resume-fc blocks on several
parallel waits before it can load the snapshot. These waits were
previously invisible — only covered by point-in-time ReportEvent
calls that do not capture duration.

Adding duration spans makes them visible as bars in the Grafana
waterfall view. This is important because wait-rootfs-path turned out to
be the primary bottleneck, growing significantly as more sandboxes are
created simultaneously.

* fix(orchestrator): parse DISABLE_HUGE_PAGES as a boolean flag

Use strconv.ParseBool so that common boolean env values like 1, TRUE,
or True are accepted, not just the exact string "true".

* Update packages/orchestrator/pkg/sandbox/fc/process.go

Co-authored-by: Jakub Novák <jakub@e2b.dev>

---------

Co-authored-by: Jakub Novák <jakub@e2b.dev>

Author: arkamar

packages/orchestrator/benchmarks/concurrent_benchmark_test.go

@@ -33,8 +33,10 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/metric/noop"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/e2b-dev/infra/packages/clickhouse/pkg/hoststats"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/cfg"
@@ -137,11 +139,20 @@ func BenchmarkConcurrentResume(b *testing.B) {
 		kernelVersion   = "vmlinux-6.1.158"
 		fcVersion       = featureflags.DefaultFirecrackerVersion
 		templateID      = "fcb33d09-3141-42c4-8d3b-c2df411681db"
-		buildID         = "ba6aae36-74f7-487a-b6f7-74fd7c94e479"
-		useHugePages    = false
 		templateVersion = "v2.0.0"
+
+		buildIDNormal    = "ba6aae36-74f7-487a-b6f7-74fd7c94e479"
+		buildIDHugePages = "ba6aae36-74f7-487a-b6f7-74fd7c94e480"
 	)
 
+	disableHP, _ := strconv.ParseBool(os.Getenv("DISABLE_HUGE_PAGES"))
+	useHugePages := !disableHP
+	buildID := buildIDHugePages
+	if !useHugePages {
+		buildID = buildIDNormal
+		b.Log("huge pages disabled")
+	}
+
 	// cache & ephemeral paths
 	persistenceDir := getPersistenceDir()
 	kernelsDir := filepath.Join(persistenceDir, "kernels")
@@ -423,9 +434,17 @@ func runConcurrentResume(
 			// Wait for the barrier.
 			<-gate
 
+			ctx, span := tracer.Start(b.Context(), "bench-resume",
+				trace.WithAttributes(
+					attribute.Int("concurrency", n),
+					attribute.Int("sandbox.index", i),
+					attribute.String("sandbox.id", runtime.SandboxID),
+				),
+			)
+
 			start := time.Now()
 			sbx, err := factory.ResumeSandbox(
-				b.Context(),
+				ctx,
 				tmpl,
 				config,
 				runtime,
@@ -434,6 +453,7 @@ func runConcurrentResume(
 				nil,
 			)
 			elapsed := time.Since(start)
+			span.End()
 
 			results[i] = concurrencyResult{
 				sandboxID: runtime.SandboxID,

packages/orchestrator/pkg/sandbox/fc/process.go

@@ -489,7 +489,10 @@ func (p *Process) Resume(
 	})
 
 	eg.Go(func() error {
-		err := socket.Wait(egCtx, uffdSocketPath)
+		ctx, uffdSpan := tracer.Start(egCtx, "wait-uffd-socket")
+		err := socket.Wait(ctx, uffdSocketPath)
+		uffdSpan.End()
+
 		if err != nil {
 			return fmt.Errorf("error waiting for uffd socket: %w", err)
 		}
@@ -500,7 +503,10 @@ func (p *Process) Resume(
 	})
 
 	eg.Go(func() error {
+		_, rootfsSpan := tracer.Start(egCtx, "wait-rootfs-path")
 		rootfsPath, err := p.rootfsProvider.Path()
+		rootfsSpan.End()
+
 		if err != nil {
 			return fmt.Errorf("error getting rootfs path: %w", err)
 		}