feat(orchestrator): add self-reported status gauge with version label (#2667)

Adds an 'orchestrator.status' observable gauge that emits 1 on each
collection cycle labelled with status, version, and commit, so each
orchestrator's health and build can be tracked individually in metrics.

Author: jakubno

packages/orchestrator/pkg/server/main.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/jellydator/ttlcache/v3"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
 
@@ -124,6 +125,25 @@ func New(ctx context.Context, cfg ServiceConfig) (*Server, error) {
 		return nil, fmt.Errorf("failed to register sandbox count metric: %w", err)
 	}
 
+	statusGauge, err := telemetry.GetGaugeInt(meter, telemetry.OrchestratorStatusGaugeName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create orchestrator status gauge: %w", err)
+	}
+
+	_, err = meter.RegisterCallback(
+		func(_ context.Context, obs metric.Observer) error {
+			obs.ObserveInt64(statusGauge, 1, metric.WithAttributes(
+				attribute.String("status", server.info.GetStatus().String()),
+				attribute.String("version", server.info.SourceVersion),
+				attribute.String("commit", server.info.SourceCommit),
+			))
+
+			return nil
+		}, statusGauge)
+	if err != nil {
+		return nil, fmt.Errorf("failed to register orchestrator status gauge: %w", err)
+	}
+
 	go server.refreshStartingSandboxesLimit(ctx)
 
 	return server, nil

packages/shared/pkg/telemetry/meters.go

@@ -129,6 +129,7 @@ const (
 
 const (
 	ApiOrchestratorCountMeterName GaugeIntType = "api.orchestrator.status"
+	OrchestratorStatusGaugeName   GaugeIntType = "orchestrator.status"
 
 	// Sandbox metrics
 	SandboxRamUsedGaugeName   GaugeIntType = "e2b.sandbox.ram.used"
@@ -243,6 +244,7 @@ var gaugeFloatUnits = map[GaugeFloatType]string{
 
 var gaugeIntDesc = map[GaugeIntType]string{
 	ApiOrchestratorCountMeterName: "Counter of running orchestrators.",
+	OrchestratorStatusGaugeName:   "Self-reported orchestrator status (always 1, labelled with status and version).",
 	SandboxRamUsedGaugeName:       "Amount of RAM used by the sandbox.",
 	SandboxRamTotalGaugeName:      "Amount of RAM available to the sandbox.",
 	SandboxRamCacheGaugeName:      "Amount of RAM used by the page cache in the sandbox.",
@@ -255,6 +257,7 @@ var gaugeIntDesc = map[GaugeIntType]string{
 
 var gaugeIntUnits = map[GaugeIntType]string{
 	ApiOrchestratorCountMeterName: "{orchestrator}",
+	OrchestratorStatusGaugeName:   "{orchestrator}",
 	SandboxRamUsedGaugeName:       "{By}",
 	SandboxRamTotalGaugeName:      "{By}",
 	SandboxRamCacheGaugeName:      "{By}",