index.ts

import { SupervisorSession } from "@trigger.dev/core/v3/workers";
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
import { formatLogLine, startTelnetLogServer } from "@trigger.dev/core/v3/telnetLogServer";
import { env } from "./env.js";
import { WorkloadServer } from "./workloadServer/index.js";
import type { WorkloadManagerOptions, WorkloadManager } from "./workloadManager/types.js";
import Docker from "dockerode";
import { z } from "zod";
import { type DequeuedMessage } from "@trigger.dev/core/v3";
import {
  DockerResourceMonitor,
  KubernetesResourceMonitor,
  NoopResourceMonitor,
  type ResourceMonitor,
} from "./resourceMonitor.js";
import { KubernetesWorkloadManager } from "./workloadManager/kubernetes.js";
import { DockerWorkloadManager } from "./workloadManager/docker.js";
import { ComputeWorkloadManager } from "./workloadManager/compute.js";
import {
  HttpServer,
  CheckpointClient,
  isKubernetesEnvironment,
} from "@trigger.dev/core/v3/serverOnly";
import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js";
import { collectDefaultMetrics, Gauge, Histogram } from "prom-client";
import { register } from "./metrics.js";
import { PodCleaner } from "./services/podCleaner.js";
import { FailedPodHandler } from "./services/failedPodHandler.js";
import { getWorkerToken } from "./workerToken.js";
import { OtlpTraceService } from "./services/otlpTraceService.js";
import {
  WarmStartVerificationService,
  type WarmStartTimings,
} from "./services/warmStartVerificationService.js";
import { extractTraceparent, getRestoreRunnerId } from "./util.js";
import { Redis } from "ioredis";
import { BackpressureMonitor } from "./backpressure/backpressureMonitor.js";
import { RedisBackpressureSignalSource } from "./backpressure/redisBackpressureSignalSource.js";
import { BackpressureMetrics } from "./backpressure/backpressureMetrics.js";
import { K8sPodCountSignalSource } from "./backpressure/k8sPodCountSignalSource.js";
import {
  fromContext,
  recordPhaseSince,
  runWideEvent,
  setExtra,
  setMeta,
  type WideEventOptions,
} from "./wideEvents/index.js";

if (env.METRICS_COLLECT_DEFAULTS) {
  collectDefaultMetrics({ register });
}

const workloadCreateDuration = new Histogram({
  name: "workload_create_duration_seconds",
  help: "Duration of workload manager create calls. A create may include backend-internal retries, so one observation can span multiple attempts.",
  labelNames: ["backend", "outcome"],
  buckets: [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60],
  registers: [register],
});

class ManagedSupervisor {
  private readonly workerSession: SupervisorSession;
  private readonly metricsServer?: HttpServer;
  private readonly workloadServer: WorkloadServer;
  private readonly workloadManager: WorkloadManager;
  private readonly workloadManagerBackend: "compute" | "kubernetes" | "docker";
  private readonly computeManager?: ComputeWorkloadManager;
  private readonly logger = new SimpleStructuredLogger("managed-supervisor");
  private readonly resourceMonitor: ResourceMonitor;
  private readonly checkpointClient?: CheckpointClient;
  private readonly warmStartVerifier?: WarmStartVerificationService;

  private readonly podCleaner?: PodCleaner;
  private readonly failedPodHandler?: FailedPodHandler;
  private readonly tracing?: OtlpTraceService;
  private readonly backpressureMonitors: BackpressureMonitor[] = [];
  private readonly backpressureRedis?: Redis;

  private readonly isKubernetes = isKubernetesEnvironment(env.KUBERNETES_FORCE_ENABLED);
  private readonly warmStartUrl = env.TRIGGER_WARM_START_URL;

  private readonly wideEventOpts: WideEventOptions = {
    service: "supervisor",
    env: { nodeId: env.TRIGGER_WORKER_INSTANCE_NAME },
    enabled: env.TRIGGER_WIDE_EVENTS_ENABLED,
  };
  private readonly wideEventsNoisyRoutes = env.TRIGGER_WIDE_EVENTS_NOISY_ROUTES;

  constructor() {
    // Strip secret-like env vars before debug-logging the rest. Add any new
    // secret env var here so it never lands in the DEBUG "Starting up" log.
    const {
      TRIGGER_WORKER_TOKEN,
      MANAGED_WORKER_SECRET,
      COMPUTE_GATEWAY_AUTH_TOKEN,
      DOCKER_REGISTRY_PASSWORD,
      TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD,
      ...envWithoutSecrets
    } = env;

    if (env.DEBUG) {
      this.logger.debug("Starting up", { envWithoutSecrets });
    }

    if (this.warmStartUrl) {
      this.logger.log("🔥 Warm starts enabled", {
        warmStartUrl: this.warmStartUrl,
      });
    }

    const workloadManagerOptions = {
      workloadApiProtocol: env.TRIGGER_WORKLOAD_API_PROTOCOL,
      workloadApiDomain: env.TRIGGER_WORKLOAD_API_DOMAIN,
      workloadApiPort: env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL,
      warmStartUrl: this.warmStartUrl,
      metadataUrl: env.TRIGGER_METADATA_URL,
      imagePullSecrets: env.KUBERNETES_IMAGE_PULL_SECRETS?.split(","),
      heartbeatIntervalSeconds: env.RUNNER_HEARTBEAT_INTERVAL_SECONDS,
      snapshotPollIntervalSeconds: env.RUNNER_SNAPSHOT_POLL_INTERVAL_SECONDS,
      additionalEnvVars: env.RUNNER_ADDITIONAL_ENV_VARS,
      dockerAutoremove: env.DOCKER_AUTOREMOVE_EXITED_CONTAINERS,
    } satisfies WorkloadManagerOptions;

    this.resourceMonitor = env.RESOURCE_MONITOR_ENABLED
      ? this.isKubernetes
        ? new KubernetesResourceMonitor(createK8sApi(), env.TRIGGER_WORKER_INSTANCE_NAME)
        : new DockerResourceMonitor(new Docker())
      : new NoopResourceMonitor();

    if (env.COMPUTE_GATEWAY_URL) {
      if (!env.TRIGGER_WORKLOAD_API_DOMAIN) {
        throw new Error("TRIGGER_WORKLOAD_API_DOMAIN is not set, cannot create compute manager");
      }

      const callbackUrl = `${env.TRIGGER_WORKLOAD_API_PROTOCOL}://${env.TRIGGER_WORKLOAD_API_DOMAIN}:${env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL}/api/v1/compute/snapshot-complete`;

      if (env.COMPUTE_TRACE_SPANS_ENABLED) {
        this.tracing = new OtlpTraceService({
          endpointUrl: env.COMPUTE_TRACE_OTLP_ENDPOINT,
        });
      }

      const computeManager = new ComputeWorkloadManager({
        ...workloadManagerOptions,
        gateway: {
          url: env.COMPUTE_GATEWAY_URL,
          authToken: env.COMPUTE_GATEWAY_AUTH_TOKEN,
          timeoutMs: env.COMPUTE_GATEWAY_TIMEOUT_MS,
        },
        snapshots: {
          enabled: env.COMPUTE_SNAPSHOTS_ENABLED,
          delayMs: env.COMPUTE_SNAPSHOT_DELAY_MS,
          dispatchLimit: env.COMPUTE_SNAPSHOT_DISPATCH_LIMIT,
          callbackUrl,
        },
        tracing: this.tracing,
        runner: {
          instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
          otelEndpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT,
          prettyLogs: env.RUNNER_PRETTY_LOGS,
          sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS,
        },
        createRetry: {
          maxAttempts: env.COMPUTE_INSTANCE_CREATE_MAX_ATTEMPTS,
          baseDelayMs: env.COMPUTE_INSTANCE_CREATE_RETRY_BASE_DELAY_MS,
        },
      });
      this.computeManager = computeManager;
      this.workloadManager = computeManager;
      this.workloadManagerBackend = "compute";
    } else if (this.isKubernetes) {
      this.workloadManager = new KubernetesWorkloadManager(workloadManagerOptions);
      this.workloadManagerBackend = "kubernetes";
    } else {
      this.workloadManager = new DockerWorkloadManager(workloadManagerOptions);
      this.workloadManagerBackend = "docker";
    }

    if (this.isKubernetes) {
      if (env.POD_CLEANER_ENABLED) {
        this.logger.log("🧹 Pod cleaner enabled", {
          namespace: env.KUBERNETES_NAMESPACE,
          batchSize: env.POD_CLEANER_BATCH_SIZE,
          intervalMs: env.POD_CLEANER_INTERVAL_MS,
        });
        this.podCleaner = new PodCleaner({
          register,
          namespace: env.KUBERNETES_NAMESPACE,
          batchSize: env.POD_CLEANER_BATCH_SIZE,
          intervalMs: env.POD_CLEANER_INTERVAL_MS,
        });
      } else {
        this.logger.warn("Pod cleaner disabled");
      }

      if (env.FAILED_POD_HANDLER_ENABLED) {
        this.logger.log("🔁 Failed pod handler enabled", {
          namespace: env.KUBERNETES_NAMESPACE,
          reconnectIntervalMs: env.FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS,
        });
        this.failedPodHandler = new FailedPodHandler({
          register,
          namespace: env.KUBERNETES_NAMESPACE,
          reconnectIntervalMs: env.FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS,
        });
      } else {
        this.logger.warn("Failed pod handler disabled");
      }
    }

    if (env.TRIGGER_DEQUEUE_INTERVAL_MS > env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS) {
      this.logger.warn(
        `⚠️  TRIGGER_DEQUEUE_INTERVAL_MS (${env.TRIGGER_DEQUEUE_INTERVAL_MS}) is greater than TRIGGER_DEQUEUE_IDLE_INTERVAL_MS (${env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS}) - did you mix them up?`
      );
    }

    // Redis-verdict source (external aggregator). Keeps existing metric names.
    if (env.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED) {
      this.backpressureRedis = new Redis({
        host: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST,
        port: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PORT,
        username: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_USERNAME,
        password: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD,
        ...(env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_TLS_DISABLED ? {} : { tls: {} }),
        maxRetriesPerRequest: null,
      });
      this.backpressureRedis.on("error", (error) =>
        this.logger.error("Backpressure redis error", { error: error.message })
      );
      this.backpressureMonitors.push(
        new BackpressureMonitor({
          enabled: true,
          source: new RedisBackpressureSignalSource(
            this.backpressureRedis,
            env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY
          ),
          refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS,
          maxVerdictAgeMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS,
          rampMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS,
          dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN,
          logger: this.logger,
          metrics: new BackpressureMetrics({ register }),
        })
      );
      this.logger.log("🛑 Dequeue backpressure enabled (redis source)", {
        key: env.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_KEY,
        refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_REFRESH_MS,
        dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_DRY_RUN,
      });
    }

    // Pod-count source (in-process apiserver scrape). Namespaced metrics so the
    // redis source's metric names are preserved.
    if (env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED) {
      // RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here.
      const podCountGauge = new Gauge({
        name: "supervisor_cluster_pod_count",
        help: "Total pod objects stored in the cluster, scraped for backpressure",
        registers: [register],
      });
      this.backpressureMonitors.push(
        new BackpressureMonitor({
          enabled: true,
          source: new K8sPodCountSignalSource({
            fetchMetrics: createApiserverMetricsFetcher(
              env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS
            ),
            engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,
            releaseThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE,
            reportPodCount: (count) => podCountGauge.set(count),
          }),
          refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS,
          maxVerdictAgeMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_MAX_VERDICT_AGE_MS,
          rampMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_RAMP_MS,
          dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN,
          logger: this.logger,
          metrics: new BackpressureMetrics({
            register,
            prefix: "supervisor_backpressure_pod_count",
          }),
        })
      );
      this.logger.log("🛑 Dequeue backpressure enabled (pod-count source)", {
        engage: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,
        release: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE,
        refreshIntervalMs: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS,
        dryRun: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_DRY_RUN,
      });
    }

    this.workerSession = new SupervisorSession({
      workerToken: getWorkerToken(),
      apiUrl: env.TRIGGER_API_URL,
      instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
      managedWorkerSecret: env.MANAGED_WORKER_SECRET,
      dequeueIntervalMs: env.TRIGGER_DEQUEUE_INTERVAL_MS,
      dequeueIdleIntervalMs: env.TRIGGER_DEQUEUE_IDLE_INTERVAL_MS,
      queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED,
      maxRunCount: env.TRIGGER_DEQUEUE_MAX_RUN_COUNT,
      queueClass: env.TRIGGER_WORKER_QUEUE_CLASS,
      metricsRegistry: register,
      scaling: {
        strategy: env.TRIGGER_DEQUEUE_SCALING_STRATEGY,
        minConsumerCount: env.TRIGGER_DEQUEUE_MIN_CONSUMER_COUNT,
        maxConsumerCount: env.TRIGGER_DEQUEUE_MAX_CONSUMER_COUNT,
        scaleUpCooldownMs: env.TRIGGER_DEQUEUE_SCALING_UP_COOLDOWN_MS,
        scaleDownCooldownMs: env.TRIGGER_DEQUEUE_SCALING_DOWN_COOLDOWN_MS,
        targetRatio: env.TRIGGER_DEQUEUE_SCALING_TARGET_RATIO,
        ewmaAlpha: env.TRIGGER_DEQUEUE_SCALING_EWMA_ALPHA,
        batchWindowMs: env.TRIGGER_DEQUEUE_SCALING_BATCH_WINDOW_MS,
        dampingFactor: env.TRIGGER_DEQUEUE_SCALING_DAMPING_FACTOR,
        // Freeze scale-up while backpressure is hard-engaged (not during the resume
        // ramp). Undefined when backpressure is disabled → no effect on scaling.
        shouldPauseScaling: () => this.backpressureMonitors.some((m) => m.isEngaged()),
      },
      runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED,
      heartbeatIntervalSeconds: env.TRIGGER_WORKER_HEARTBEAT_INTERVAL_SECONDS,
      sendRunDebugLogs: env.SEND_RUN_DEBUG_LOGS,
      preDequeue: async () => {
        // Synchronous, hot-path-safe cached read; false when no monitors are active.
        const skipForBackpressure = this.backpressureMonitors.some((m) => m.shouldSkipDequeue());

        if (!env.RESOURCE_MONITOR_ENABLED || this.isKubernetes) {
          // Resource monitor is not used in k8s; backpressure is the only gate there.
          return { skipDequeue: skipForBackpressure };
        }

        const resources = await this.resourceMonitor.getNodeResources();

        return {
          maxResources: {
            cpu: resources.cpuAvailable,
            memory: resources.memoryAvailable,
          },
          skipDequeue:
            skipForBackpressure ||
            resources.cpuAvailable < 0.25 ||
            resources.memoryAvailable < 0.25,
        };
      },
      preSkip: async () => {
        // When the node is full, it should still try to warm start runs
        // await this.tryWarmStartAllThisNode();
      },
    });

    if (env.TRIGGER_CHECKPOINT_URL) {
      this.logger.log("🥶 Checkpoints enabled", {
        checkpointUrl: env.TRIGGER_CHECKPOINT_URL,
      });

      this.checkpointClient = new CheckpointClient({
        apiUrl: new URL(env.TRIGGER_CHECKPOINT_URL),
        workerClient: this.workerSession.httpClient,
        orchestrator: this.isKubernetes ? "KUBERNETES" : "DOCKER",
      });
    }

    if (env.TRIGGER_WARM_START_VERIFY_ENABLED && this.warmStartUrl) {
      this.logger.log("Warm-start delivery verification enabled", {
        delayMs: env.TRIGGER_WARM_START_VERIFY_DELAY_MS,
      });

      this.warmStartVerifier = new WarmStartVerificationService({
        workerClient: this.workerSession.httpClient,
        delayMs: env.TRIGGER_WARM_START_VERIFY_DELAY_MS,
        createWorkload: (message, timings) => this.createWorkload(message, timings),
        wideEventOpts: this.wideEventOpts,
      });
    }

    this.workerSession.on("runNotification", async ({ time, run }) => {
      this.logger.verbose("runNotification", { time, run });

      this.workloadServer.notifyRun({ run });
    });

    this.workerSession.on(
      "runQueueMessage",
      async ({ time, message, dequeueResponseMs, pollingIntervalMs }) => {
        this.logger.verbose(`Received message with timestamp ${time.toLocaleString()}`, message);

        const traceparent = extractTraceparent(message.run.traceContext);

        await runWideEvent(
          {
            ...this.wideEventOpts,
            op: "dequeue",
            kind: "inbound",
            traceparent,
            setup: (state) => {
              setMeta(state, "run_id", message.run.friendlyId);
              setMeta(state, "env_id", message.environment.id);
              setMeta(state, "org_id", message.organization.id);
              setMeta(state, "project_id", message.project.id);
              if (message.deployment.friendlyId) {
                setMeta(state, "deployment_id", message.deployment.friendlyId);
              }
              setMeta(state, "machine_preset", message.run.machine.name);
              state.extras.iteration = "dequeue";
              state.extras.dequeue_response_ms = dequeueResponseMs;
              state.extras.polling_interval_ms = pollingIntervalMs;
              state.extras.completed_waitpoints = message.completedWaitpoints.length;
            },
          },
          async () => {
            if (message.completedWaitpoints.length > 0) {
              this.logger.debug("Run has completed waitpoints", {
                runId: message.run.id,
                completedWaitpoints: message.completedWaitpoints.length,
              });
            }

            if (!message.image) {
              setExtra(fromContext(), "path_taken", "skipped_no_image");
              this.logger.error("Run has no image", { runId: message.run.id });
              return;
            }

            const { checkpoint, ...rest } = message;

            // Register trace context early so snapshot spans work for all paths
            // (cold create, restore, warm start). Re-registration on restore is safe
            // since dequeue always provides fresh context.
            if (this.computeManager?.traceSpansEnabled && traceparent) {
              this.workloadServer.registerRunTraceContext(message.run.friendlyId, {
                traceparent,
                envId: message.environment.id,
                orgId: message.organization.id,
                projectId: message.project.id,
              });
            }

            if (checkpoint) {
              setExtra(fromContext(), "path_taken", "restore");
              this.logger.debug("Restoring run", { runId: message.run.id });

              if (this.computeManager) {
                const restoreStart = performance.now();
                try {
                  const runnerId = getRestoreRunnerId(message.run.friendlyId, checkpoint.id);

                  const didRestore = await this.computeManager.restore({
                    snapshotId: checkpoint.location,
                    runnerId,
                    runFriendlyId: message.run.friendlyId,
                    snapshotFriendlyId: message.snapshot.friendlyId,
                    machine: message.run.machine,
                    traceContext: message.run.traceContext,
                    envId: message.environment.id,
                    orgId: message.organization.id,
                    projectId: message.project.id,
                    hasPrivateLink: message.organization.hasPrivateLink,
                    dequeuedAt: message.dequeuedAt,
                  });
                  recordPhaseSince("restore", restoreStart, undefined);
                  setExtra(fromContext(), "did_restore", didRestore);

                  if (didRestore) {
                    this.logger.debug("Compute restore successful", {
                      runId: message.run.id,
                      runnerId,
                    });
                  } else {
                    this.logger.error("Compute restore failed", {
                      runId: message.run.id,
                      runnerId,
                    });
                  }
                } catch (error) {
                  recordPhaseSince(
                    "restore",
                    restoreStart,
                    error instanceof Error ? error : new Error(String(error))
                  );
                  this.logger.error("Failed to restore run (compute)", { error });
                }

                return;
              }

              if (!this.checkpointClient) {
                this.logger.error("No checkpoint client", { runId: message.run.id });
                return;
              }

              const restoreStart = performance.now();
              try {
                const didRestore = await this.checkpointClient.restoreRun({
                  runFriendlyId: message.run.friendlyId,
                  snapshotFriendlyId: message.snapshot.friendlyId,
                  body: {
                    ...rest,
                    checkpoint,
                  },
                });
                recordPhaseSince("restore", restoreStart, undefined);
                setExtra(fromContext(), "did_restore", didRestore);

                if (didRestore) {
                  this.logger.debug("Restore successful", { runId: message.run.id });
                } else {
                  this.logger.error("Restore failed", { runId: message.run.id });
                }
              } catch (error) {
                recordPhaseSince(
                  "restore",
                  restoreStart,
                  error instanceof Error ? error : new Error(String(error))
                );
                this.logger.error("Failed to restore run", { error });
              }

              return;
            }

            this.logger.debug("Scheduling run", { runId: message.run.id });

            const warmStartStart = performance.now();
            const didWarmStart = await this.tryWarmStart(message, traceparent);
            const warmStartCheckMs = Math.round(performance.now() - warmStartStart);
            recordPhaseSince("warm_start", warmStartStart, undefined);
            setExtra(fromContext(), "did_warm_start", didWarmStart);

            if (didWarmStart) {
              setExtra(fromContext(), "path_taken", "warm_start");
              this.logger.debug("Warm start successful", { runId: message.run.id });
              // A hit only means the response was written to the long-poll
              // socket, not that the runner received it. Schedule a delivery
              // verification that cold-starts the run if nobody acts on it.
              this.warmStartVerifier?.schedule(message, {
                dequeueResponseMs,
                pollingIntervalMs,
                warmStartCheckMs,
              });
              return;
            }

            setExtra(fromContext(), "path_taken", "cold_create");

            await this.createWorkload(message, {
              dequeueResponseMs,
              pollingIntervalMs,
              warmStartCheckMs,
            });
          }
        );
      }
    );

    if (env.METRICS_ENABLED) {
      this.metricsServer = new HttpServer({
        port: env.METRICS_PORT,
        host: env.METRICS_HOST,
        metrics: {
          register,
          expose: true,
        },
      });
    }

    // Responds to workload requests only
    this.workloadServer = new WorkloadServer({
      port: env.TRIGGER_WORKLOAD_API_PORT_INTERNAL,
      host: env.TRIGGER_WORKLOAD_API_HOST_INTERNAL,
      workerClient: this.workerSession.httpClient,
      checkpointClient: this.checkpointClient,
      computeManager: this.computeManager,
      tracing: this.tracing,
      wideEventOpts: this.wideEventOpts,
      wideEventsNoisyRoutes: this.wideEventsNoisyRoutes,
    });

    this.workloadServer.on("runConnected", this.onRunConnected.bind(this));
    this.workloadServer.on("runDisconnected", this.onRunDisconnected.bind(this));
  }

  async onRunConnected({ run }: { run: { friendlyId: string } }) {
    this.logger.debug("Run connected", { run });
    // The dispatched run reached a runner on this node - no fallback needed.
    this.warmStartVerifier?.cancel(run.friendlyId);
    this.workerSession.subscribeToRunNotifications([run.friendlyId]);
  }

  async onRunDisconnected({ run }: { run: { friendlyId: string } }) {
    this.logger.debug("Run disconnected", { run });
    this.workerSession.unsubscribeFromRunNotifications([run.friendlyId]);
  }

  private async createWorkload(message: DequeuedMessage, timings: WarmStartTimings) {
    const createStart = performance.now();
    try {
      if (!message.deployment.friendlyId) {
        // mostly a type guard, deployments always exists for deployed environments
        // a proper fix would be to use a discriminated union schema to differentiate between dequeued runs in dev and in deployed environments.
        throw new Error("Deployment is missing");
      }

      if (!message.image) {
        // same type-guard situation as deployment above
        throw new Error("Image is missing");
      }

      await this.workloadManager.create({
        dequeuedAt: message.dequeuedAt,
        dequeueResponseMs: timings.dequeueResponseMs,
        pollingIntervalMs: timings.pollingIntervalMs,
        warmStartCheckMs: timings.warmStartCheckMs,
        envId: message.environment.id,
        envType: message.environment.type,
        image: message.image,
        machine: message.run.machine,
        orgId: message.organization.id,
        projectId: message.project.id,
        deploymentFriendlyId: message.deployment.friendlyId,
        deploymentVersion: message.backgroundWorker.version,
        runId: message.run.id,
        runFriendlyId: message.run.friendlyId,
        version: message.version,
        nextAttemptNumber: message.run.attemptNumber,
        snapshotId: message.snapshot.id,
        snapshotFriendlyId: message.snapshot.friendlyId,
        placementTags: message.placementTags,
        traceContext: message.run.traceContext,
        annotations: message.run.annotations,
        hasPrivateLink: message.organization.hasPrivateLink,
      });
      recordPhaseSince("workload_create", createStart, undefined);
      workloadCreateDuration.observe(
        { backend: this.workloadManagerBackend, outcome: "success" },
        (performance.now() - createStart) / 1000
      );

      // Disabled for now
      // this.resourceMonitor.blockResources({
      //   cpu: message.run.machine.cpu,
      //   memory: message.run.machine.memory,
      // });
    } catch (error) {
      recordPhaseSince(
        "workload_create",
        createStart,
        error instanceof Error ? error : new Error(String(error))
      );
      workloadCreateDuration.observe(
        { backend: this.workloadManagerBackend, outcome: "error" },
        (performance.now() - createStart) / 1000
      );
      this.logger.error("Failed to create workload", {
        runId: message.run.friendlyId,
        error,
      });
    }
  }

  private async tryWarmStart(
    dequeuedMessage: DequeuedMessage,
    traceparent: string | undefined
  ): Promise<boolean> {
    if (!this.warmStartUrl) {
      return false;
    }

    const warmStartUrlWithPath = new URL("/warm-start", this.warmStartUrl);

    const headers: Record<string, string> = {
      "Content-Type": "application/json",
    };
    // Propagate the inbound W3C traceparent so the upstream warm-start
    // receiver continues the same trace instead of minting a new one. Gated
    // by the same kill switch as the wide-event emission so the whole PR is
    // a no-op on the wire when disabled.
    if (this.wideEventOpts.enabled && traceparent) {
      headers.traceparent = traceparent;
    }

    try {
      const res = await fetch(warmStartUrlWithPath.href, {
        method: "POST",
        headers,
        body: JSON.stringify({ dequeuedMessage }),
      });

      if (!res.ok) {
        this.logger.error("Warm start failed", {
          runId: dequeuedMessage.run.id,
        });
        return false;
      }

      const data = await res.json();
      const parsedData = z.object({ didWarmStart: z.boolean() }).safeParse(data);

      if (!parsedData.success) {
        this.logger.error("Warm start response invalid", {
          runId: dequeuedMessage.run.id,
          data,
        });
        return false;
      }

      return parsedData.data.didWarmStart;
    } catch (error) {
      this.logger.error("Warm start error", {
        runId: dequeuedMessage.run.id,
        error,
      });
      return false;
    }
  }

  async start() {
    this.logger.log("Starting up");

    // Optional services
    this.backpressureMonitors.forEach((m) => m.start());
    await this.podCleaner?.start();
    await this.failedPodHandler?.start();
    await this.metricsServer?.start();

    if (env.TRIGGER_WORKLOAD_API_ENABLED) {
      this.logger.log("Workload API enabled", {
        protocol: env.TRIGGER_WORKLOAD_API_PROTOCOL,
        domain: env.TRIGGER_WORKLOAD_API_DOMAIN,
        port: env.TRIGGER_WORKLOAD_API_PORT_INTERNAL,
      });
      await this.workloadServer.start();
    } else {
      this.logger.warn("Workload API disabled");
    }

    await this.workerSession.start();
  }

  async stop() {
    this.logger.log("Shutting down");
    // Stop the verifier first: its timer can otherwise fire mid-shutdown and
    // cold-create a workload on a node that is going down.
    this.warmStartVerifier?.stop();
    await this.workloadServer.stop();
    await this.workerSession.stop();

    // Optional services
    this.backpressureMonitors.forEach((m) => m.stop());
    await this.backpressureRedis?.quit();
    await this.podCleaner?.stop();
    await this.failedPodHandler?.stop();
    await this.metricsServer?.stop();
  }
}

// Opt-in, dev-only: mirror this process's structured logs to a local telnet/TCP stream.
if (env.SUPERVISOR_TELNET_LOGS_PORT && env.SUPERVISOR_TELNET_LOGS_PORT > 0) {
  const telnetLogServer = startTelnetLogServer({
    port: env.SUPERVISOR_TELNET_LOGS_PORT,
    name: "supervisor",
  });
  SimpleStructuredLogger.onLog = (log) => telnetLogServer.broadcast(formatLogLine(log));
}

const worker = new ManagedSupervisor();
worker.start();