Reap orphaned dev supervisors and stop exception storms

scripts/dev-launch.mjs

@@ -7,6 +7,11 @@ delete process.env.ELECTRON_RUN_AS_NODE;
 import { execSync, spawn } from "node:child_process";
 import { createRequire } from "node:module";
 import { resolveDevServerPort } from "./dev-server-port.mjs";
+import { sweepStaleSupervisors } from "./sweepStaleSupervisors.mjs";
+
+// Reap orphaned dev supervisors left behind by crashed / force-quit dev
+// instances before launching a new one. Best effort; never blocks launch.
+sweepStaleSupervisors();
 
 const env = {
   ...process.env,

scripts/sweepStaleSupervisors.mjs

@@ -0,0 +1,98 @@
+// Dev-only safety net: reap orphaned dev supervisors left behind when an
+// Electron main died without disposing its forked supervisor (crash,
+// force-quit, `kill -9`, electronmon restart). macOS/Linux have no Job
+// Object equivalent, so such supervisors are reparented to launchd/init and
+// keep running — sometimes at 100% CPU. The in-app orphan watchdog covers
+// most cases; this sweep catches supervisors that wedged before any of
+// their timers could run.
+//
+// Detection is precise: a legitimate dev supervisor always has a live
+// Electron parent (ppid !== 1), so concurrent worktree dev apps are never
+// touched. Packaged-app supervisors (app.asar paths) are skipped. Windows
+// is skipped because Job Objects take the tree down with the parent.
+
+import { spawnSync } from "node:child_process";
+
+const SUPERVISOR_COMMAND_PATTERN = /[/\\]dist[/\\]main[/\\]supervisor\.cjs\b/;
+const SIGKILL_FOLLOW_UP_MS = 2_000;
+
+function pidIsAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function parseSupervisorOrphans(psOutput) {
+  const orphans = [];
+  for (const line of psOutput.split("\n")) {
+    const match = line.trim().match(/^(\d+)\s+(\d+)\s+(.*)$/);
+    if (!match) {
+      continue;
+    }
+    const [, pidRaw, ppidRaw, command] = match;
+    // Only true orphans: reparented to init/launchd after the parent died.
+    if (ppidRaw !== "1") {
+      continue;
+    }
+    if (!SUPERVISOR_COMMAND_PATTERN.test(command)) {
+      continue;
+    }
+    // Never touch packaged-app supervisors.
+    if (command.includes("app.asar")) {
+      continue;
+    }
+    orphans.push({ pid: Number(pidRaw), command });
+  }
+  return orphans;
+}
+
+export function sweepStaleSupervisors({ log = console.log } = {}) {
+  if (process.platform === "win32") {
+    return;
+  }
+  let output;
+  try {
+    const result = spawnSync("ps", ["-axo", "pid=,ppid=,command="], {
+      encoding: "utf8",
+      timeout: 5_000,
+    });
+    if (result.status !== 0 || !result.stdout) {
+      return;
+    }
+    output = result.stdout;
+  } catch {
+    return;
+  }
+
+  const orphans = parseSupervisorOrphans(output);
+  if (orphans.length === 0) {
+    return;
+  }
+
+  for (const orphan of orphans) {
+    log(`[dev-launch] reaping stale supervisor pid=${orphan.pid}: ${orphan.command}`);
+    try {
+      process.kill(orphan.pid, "SIGTERM");
+    } catch {
+      // Already gone.
+    }
+  }
+
+  // Escalate survivors to SIGKILL; a wedged supervisor cannot handle signals.
+  const followUp = setTimeout(() => {
+    for (const orphan of orphans) {
+      if (!pidIsAlive(orphan.pid)) {
+        continue;
+      }
+      try {
+        process.kill(orphan.pid, "SIGKILL");
+      } catch {
+        // Already gone.
+      }
+    }
+  }, SIGKILL_FOLLOW_UP_MS);
+  followUp.unref?.();
+}

src/supervisor/devOrphanWatchdog.test.ts

@@ -0,0 +1,171 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { startDevOrphanWatchdog, type DevOrphanWatchdogOptions } from "./devOrphanWatchdog";
+
+type WatchdogContext = {
+  stop(): void;
+  requestShutdown: ReturnType<typeof vi.fn>;
+  exit: ReturnType<typeof vi.fn>;
+  isConnected: ReturnType<typeof vi.fn<() => boolean>>;
+  getParentPid: ReturnType<typeof vi.fn<() => number>>;
+  pidExists: ReturnType<typeof vi.fn<(pid: number) => boolean>>;
+};
+
+const POLL_MS = 1_000;
+const HARD_EXIT_MS = 500;
+
+describe("startDevOrphanWatchdog", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.restoreAllMocks();
+  });
+
+  function start(overrides: Partial<DevOrphanWatchdogOptions> = {}): WatchdogContext {
+    const requestShutdown = vi.fn<() => void>();
+    const exit = vi.fn<(code: number) => void>();
+    const isConnected = vi.fn<() => boolean>(() => true);
+    const getParentPid = vi.fn<() => number>(() => 4242);
+    const pidExists = vi.fn<(pid: number) => boolean>(() => true);
+    const handle = startDevOrphanWatchdog({
+      pollMs: POLL_MS,
+      confirmations: 2,
+      hardExitMs: HARD_EXIT_MS,
+      requestShutdown,
+      exit,
+      isConnected,
+      getParentPid,
+      pidExists,
+      ...overrides,
+    });
+    return {
+      stop: () => handle.stop(),
+      requestShutdown,
+      exit,
+      isConnected,
+      getParentPid,
+      pidExists,
+    };
+  }
+
+  it("does not fire while the parent is alive", () => {
+    const ctx = start();
+    vi.advanceTimersByTime(POLL_MS * 10);
+    expect(ctx.requestShutdown).not.toHaveBeenCalled();
+    expect(ctx.exit).not.toHaveBeenCalled();
+    ctx.stop();
+  });
+
+  it("shuts down after the parent disconnects on two consecutive polls", () => {
+    const ctx = start();
+    ctx.isConnected.mockReturnValue(false);
+
+    vi.advanceTimersByTime(POLL_MS);
+    expect(ctx.requestShutdown).not.toHaveBeenCalled();
+
+    vi.advanceTimersByTime(POLL_MS);
+    expect(ctx.requestShutdown).toHaveBeenCalledTimes(1);
+    expect(ctx.exit).not.toHaveBeenCalled();
+
+    vi.advanceTimersByTime(HARD_EXIT_MS);
+    expect(ctx.exit).toHaveBeenCalledWith(1);
+    expect(ctx.exit).toHaveBeenCalledTimes(1);
+    ctx.stop();
+  });
+
+  it("treats reparenting to pid 1 as orphaned", () => {
+    const ctx = start();
+    ctx.getParentPid.mockReturnValue(1);
+    vi.advanceTimersByTime(POLL_MS * 2);
+    expect(ctx.requestShutdown).toHaveBeenCalledTimes(1);
+    ctx.stop();
+  });
+
+  it("treats a missing parent pid as orphaned", () => {
+    const ctx = start();
+    ctx.pidExists.mockReturnValue(false);
+    vi.advanceTimersByTime(POLL_MS * 2);
+    expect(ctx.requestShutdown).toHaveBeenCalledTimes(1);
+    ctx.stop();
+  });
+
+  it("ignores a single transient miss", () => {
+    const ctx = start();
+    ctx.isConnected.mockReturnValue(false);
+    vi.advanceTimersByTime(POLL_MS);
+    ctx.isConnected.mockReturnValue(true);
+    vi.advanceTimersByTime(POLL_MS * 5);
+    expect(ctx.requestShutdown).not.toHaveBeenCalled();
+    expect(ctx.exit).not.toHaveBeenCalled();
+    ctx.stop();
+  });
+
+  it("fires only once and stops polling", () => {
+    const ctx = start();
+    ctx.isConnected.mockReturnValue(false);
+    vi.advanceTimersByTime(POLL_MS * 2 + HARD_EXIT_MS);
+    expect(ctx.requestShutdown).toHaveBeenCalledTimes(1);
+    expect(ctx.exit).toHaveBeenCalledTimes(1);
+
+    vi.advanceTimersByTime(POLL_MS * 10 + HARD_EXIT_MS * 10);
+    expect(ctx.requestShutdown).toHaveBeenCalledTimes(1);
+    expect(ctx.exit).toHaveBeenCalledTimes(1);
+    ctx.stop();
+  });
+
+  it("does not flag reparenting when launched under pid 1", () => {
+    const getParentPid = vi.fn<() => number>(() => 1);
+    const pidExists = vi.fn<(pid: number) => boolean>(() => true);
+    const requestShutdown = vi.fn<() => void>();
+    const handle = startDevOrphanWatchdog({
+      pollMs: POLL_MS,
+      confirmations: 1,
+      requestShutdown,
+      isConnected: () => true,
+      getParentPid,
+      pidExists,
+    });
+    vi.advanceTimersByTime(POLL_MS * 5);
+    expect(requestShutdown).not.toHaveBeenCalled();
+    handle.stop();
+  });
+
+  it("uses live ppid and pid probes by default", () => {
+    // Real probes against the live test runner: its parent exists, so the
+    // watchdog must stay quiet.
+    const requestShutdown = vi.fn<() => void>();
+    const handle = startDevOrphanWatchdog({
+      pollMs: POLL_MS,
+      confirmations: 1,
+      requestShutdown,
+      isConnected: () => true,
+    });
+    vi.advanceTimersByTime(POLL_MS * 3);
+    expect(requestShutdown).not.toHaveBeenCalled();
+    handle.stop();
+  });
+
+  it("treats EPERM from the default pid probe as alive", () => {
+    vi.spyOn(process, "kill").mockImplementation(((pid: number, signal?: number | string) => {
+      if (signal === 0 || signal === undefined) {
+        const error = new Error("EPERM: operation not permitted") as NodeJS.ErrnoException;
+        error.code = "EPERM";
+        throw error;
+      }
+      return true;
+    }) as typeof process.kill);
+
+    const requestShutdown = vi.fn<() => void>();
+    const handle = startDevOrphanWatchdog({
+      pollMs: POLL_MS,
+      confirmations: 1,
+      requestShutdown,
+      isConnected: () => true,
+    });
+    vi.advanceTimersByTime(POLL_MS * 3);
+    expect(requestShutdown).not.toHaveBeenCalled();
+    handle.stop();
+  });
+});

src/supervisor/devOrphanWatchdog.ts

@@ -0,0 +1,103 @@
+/**
+ * Dev-only guard: self-exit when the Electron main process that forked this
+ * supervisor disappears (crash, force-quit, `kill -9`, electronmon restart).
+ * macOS/Linux have no Job Object equivalent, so an orphaned supervisor is
+ * reparented to launchd/init and otherwise runs forever — sometimes at 100%
+ * CPU when the event loop is stuck in an exception storm.
+ *
+ * The watchdog polls cheap liveness signals and, after two consecutive
+ * confirmations, requests a graceful shutdown with a hard `process.exit`
+ * deadline so a wedged dispose can never block the exit. Packaged builds
+ * never install it.
+ */
+
+export interface DevOrphanWatchdogOptions {
+  pollMs?: number;
+  /** Consecutive orphan detections required before acting. */
+  confirmations?: number;
+  /** Grace period for a graceful shutdown before the hard exit fires. */
+  hardExitMs?: number;
+  requestShutdown(): void;
+  exit?(code: number): void;
+  isConnected?(): boolean;
+  getParentPid?(): number;
+  pidExists?(pid: number): boolean;
+}
+
+export interface DevOrphanWatchdogHandle {
+  stop(): void;
+}
+
+const DEFAULT_POLL_MS = 2_000;
+const DEFAULT_CONFIRMATIONS = 2;
+const DEFAULT_HARD_EXIT_MS = 2_000;
+
+function defaultPidExists(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM means the process exists but belongs to another user; only a
+    // delivery failure (ESRCH-style) proves it is gone.
+    return (error as NodeJS.ErrnoException).code === "EPERM";
+  }
+}
+
+export function startDevOrphanWatchdog(options: DevOrphanWatchdogOptions): DevOrphanWatchdogHandle {
+  const pollMs = options.pollMs ?? DEFAULT_POLL_MS;
+  const confirmations = Math.max(1, options.confirmations ?? DEFAULT_CONFIRMATIONS);
+  const hardExitMs = options.hardExitMs ?? DEFAULT_HARD_EXIT_MS;
+  const isConnected = options.isConnected ?? (() => Boolean(process.connected));
+  const getParentPid = options.getParentPid ?? (() => process.ppid);
+  const pidExists = options.pidExists ?? defaultPidExists;
+  const exit = options.exit ?? ((code: number) => process.exit(code));
+
+  const initialParentPid = getParentPid();
+  let consecutiveMisses = 0;
+  let fired = false;
+
+  const isParentGone = (): boolean => {
+    // IPC EOF is the strongest signal: the fork contract closes the channel
+    // even when the parent dies hard. PID reuse cannot fool it.
+    if (!isConnected()) {
+      return true;
+    }
+    // Reparented to init/launchd after the parent died. Skip the heuristic
+    // when we legitimately started under pid 1 (e.g. container inits).
+    if (initialParentPid !== 1 && getParentPid() === 1) {
+      return true;
+    }
+    return !pidExists(initialParentPid);
+  };
+
+  const timer = setInterval(() => {
+    if (fired) {
+      return;
+    }
+    if (!isParentGone()) {
+      consecutiveMisses = 0;
+      return;
+    }
+    consecutiveMisses += 1;
+    if (consecutiveMisses < confirmations) {
+      return;
+    }
+    fired = true;
+    clearInterval(timer);
+    console.error(
+      `[supervisor] dev orphan watchdog: parent pid ${initialParentPid} is gone; shutting down`,
+    );
+    // Hard deadline first: the graceful shutdown below must never be able to
+    // wedge the exit path.
+    const hardExit = setTimeout(() => exit(1), hardExitMs);
+    hardExit.unref?.();
+    options.requestShutdown();
+  }, pollMs);
+  timer.unref?.();
+
+  return {
+    stop() {
+      clearInterval(timer);
+    },
+  };
+}