Supervised daemon reclaims a stale manifest after reboot

A real reboot test (with integration data in the DB) surfaced an intermittent
crash-loop: across a reboot the pre-reboot server.json persists with the old
pid, and pids recycle — so isPidAlive(oldPid) often returns true for an
unrelated process, making the startup guard treat it as a live-but-unreachable
server and refuse to start. Under launchd/systemd KeepAlive that loops forever
and the daemon never serves.

Since the OS service manager is the real singleton (kickstart -k kills the old
instance before starting the new), a supervised daemon now force-removes any
pre-existing manifest and reclaims it instead of asserting. The daemon-pointer
check already required reachability, so only the manifest path needed this.

Author: RhysSullivan

apps/cli/src/local-server-manifest.ts

@@ -74,6 +74,25 @@ export const removeLocalServerManifestIfOwnedBy = (input: {
     yield* fs.remove(manifestPath, { force: true });
   });
 
+/**
+ * Remove the server manifest unconditionally. Used by an OS-supervised daemon
+ * to reclaim a stale `server.json` left by a previous boot: across a reboot the
+ * recorded pid is meaningless (pids recycle, so it may now belong to an
+ * unrelated process), and launchd/systemd already guarantee a single supervised
+ * instance — so any pre-existing manifest is stale and the supervised daemon
+ * owns it.
+ */
+export const removeLocalServerManifest = (): Effect.Effect<
+  void,
+  PlatformError,
+  FileSystem.FileSystem | Path.Path
+> =>
+  Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const path = yield* Path.Path;
+    yield* fs.remove(localServerManifestPath(path), { force: true });
+  });
+
 const StartupLockPayload = Schema.Struct({
   pid: Schema.Number,
 });

apps/cli/src/main.ts

@@ -95,6 +95,7 @@ import {
   readLocalServerManifest,
   readServiceKey,
   releaseLocalServerStartLock,
+  removeLocalServerManifest,
   removeLocalServerManifestIfOwnedBy,
   removeServiceKey,
   resolveExecutorDataDir,
@@ -898,7 +899,19 @@ const runDaemonSession = (input: {
       let token: string | null = null;
 
       try {
-        yield* assertNoOtherActiveLocalServer();
+        // A supervised daemon (launchd/systemd) is the OS-guaranteed singleton
+        // — kickstart -k kills the old instance before starting the new — so any
+        // server.json from a previous boot is stale. Reclaim it rather than
+        // refusing: across a reboot the recorded pid may have been recycled by
+        // an unrelated process, which would otherwise make the "is one already
+        // running?" check treat it as alive-but-unreachable, refuse to start,
+        // and crash-loop under KeepAlive. (Found by a real reboot test with
+        // integration data in the DB.)
+        if (process.env.EXECUTOR_SUPERVISED) {
+          yield* removeLocalServerManifest().pipe(Effect.ignore);
+        } else {
+          yield* assertNoOtherActiveLocalServer();
+        }
 
         const existing = yield* readDaemonPointer({ hostname: daemonHost, scopeId });
 

apps/cli/src/service.ts

@@ -95,7 +95,9 @@ const runCommand = (
   env?: Record<string, string | undefined>,
 ): Effect.Effect<CommandResult, Error> =>
   Effect.callback<CommandResult, Error>((resume) => {
-    const options = env ? { encoding: "utf8" as const, env: { ...process.env, ...env } } : { encoding: "utf8" as const };
+    const options = env
+      ? { encoding: "utf8" as const, env: { ...process.env, ...env } }
+      : { encoding: "utf8" as const };
     execFile(cmd, [...args], options, (error, stdout, stderr) => {
       // A string `code` (ENOENT etc.) means the command could not be spawned.
       if (error && typeof (error as { code?: unknown }).code === "string") {
@@ -389,7 +391,9 @@ const makeSystemdBackend = (): ServiceBackend => {
         // it if the caller's environment lacks it (e.g. a non-login shell) so
         // install is robust regardless of how it was invoked.
         const username = userInfo().username;
-        const sdEnv = { XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}` };
+        const sdEnv = {
+          XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}`,
+        };
         yield* runCommand("systemctl", ["--user", "daemon-reload"], sdEnv).pipe(Effect.ignore);
         const enable = yield* runCommand(
           "systemctl",
@@ -398,7 +402,9 @@ const makeSystemdBackend = (): ServiceBackend => {
         );
         if (enable.code !== 0) {
           return yield* Effect.fail(
-            new Error(`systemctl --user enable failed (exit ${enable.code}): ${enable.stderr.trim()}`),
+            new Error(
+              `systemctl --user enable failed (exit ${enable.code}): ${enable.stderr.trim()}`,
+            ),
           );
         }
         // Enable lingering so the user manager — and this enabled service —
@@ -412,7 +418,9 @@ const makeSystemdBackend = (): ServiceBackend => {
       Effect.gen(function* () {
         const fs = yield* FileSystem.FileSystem;
         const path = yield* Path.Path;
-        const sdEnv = { XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}` };
+        const sdEnv = {
+          XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}`,
+        };
         yield* runCommand("systemctl", ["--user", "disable", "--now", unitName], sdEnv).pipe(
           Effect.ignore,
         );
@@ -426,7 +434,9 @@ const makeSystemdBackend = (): ServiceBackend => {
       Effect.gen(function* () {
         const fs = yield* FileSystem.FileSystem;
         const path = yield* Path.Path;
-        const sdEnv = { XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}` };
+        const sdEnv = {
+          XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}`,
+        };
         const registered = yield* fs.exists(systemdUnitPath(path));
         const active = yield* runCommand("systemctl", ["--user", "is-active", unitName], sdEnv);
         const running = active.stdout.trim() === "active";
@@ -452,7 +462,9 @@ const makeSystemdBackend = (): ServiceBackend => {
       }),
     restart: () =>
       Effect.gen(function* () {
-        const sdEnv = { XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}` };
+        const sdEnv = {
+          XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR ?? `/run/user/${currentUid()}`,
+        };
         const result = yield* runCommand("systemctl", ["--user", "restart", unitName], sdEnv);
         if (result.code !== 0) {
           return yield* Effect.fail(