diff --git a/scripts/dev-launch.mjs b/scripts/dev-launch.mjs index 3728f3d3..cdcd0684 100644 --- a/scripts/dev-launch.mjs +++ b/scripts/dev-launch.mjs @@ -7,6 +7,11 @@ delete process.env.ELECTRON_RUN_AS_NODE; import { execSync, spawn } from "node:child_process"; import { createRequire } from "node:module"; import { resolveDevServerPort } from "./dev-server-port.mjs"; +import { sweepStaleSupervisors } from "./sweepStaleSupervisors.mjs"; + +// Reap orphaned dev supervisors left behind by crashed / force-quit dev +// instances before launching a new one. Best effort; never blocks launch. +sweepStaleSupervisors(); const env = { ...process.env, diff --git a/scripts/sweepStaleSupervisors.mjs b/scripts/sweepStaleSupervisors.mjs new file mode 100644 index 00000000..2f85d73b --- /dev/null +++ b/scripts/sweepStaleSupervisors.mjs @@ -0,0 +1,98 @@ +// Dev-only safety net: reap orphaned dev supervisors left behind when an +// Electron main died without disposing its forked supervisor (crash, +// force-quit, `kill -9`, electronmon restart). macOS/Linux have no Job +// Object equivalent, so such supervisors are reparented to launchd/init and +// keep running — sometimes at 100% CPU. The in-app orphan watchdog covers +// most cases; this sweep catches supervisors that wedged before any of +// their timers could run. +// +// Detection is precise: a legitimate dev supervisor always has a live +// Electron parent (ppid !== 1), so concurrent worktree dev apps are never +// touched. Packaged-app supervisors (app.asar paths) are skipped. Windows +// is skipped because Job Objects take the tree down with the parent. + +import { spawnSync } from "node:child_process"; + +const SUPERVISOR_COMMAND_PATTERN = /[/\\]dist[/\\]main[/\\]supervisor\.cjs\b/; +const SIGKILL_FOLLOW_UP_MS = 2_000; + +function pidIsAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +export function parseSupervisorOrphans(psOutput) { + const orphans = []; + for (const line of psOutput.split("\n")) { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(.*)$/); + if (!match) { + continue; + } + const [, pidRaw, ppidRaw, command] = match; + // Only true orphans: reparented to init/launchd after the parent died. + if (ppidRaw !== "1") { + continue; + } + if (!SUPERVISOR_COMMAND_PATTERN.test(command)) { + continue; + } + // Never touch packaged-app supervisors. + if (command.includes("app.asar")) { + continue; + } + orphans.push({ pid: Number(pidRaw), command }); + } + return orphans; +} + +export function sweepStaleSupervisors({ log = console.log } = {}) { + if (process.platform === "win32") { + return; + } + let output; + try { + const result = spawnSync("ps", ["-axo", "pid=,ppid=,command="], { + encoding: "utf8", + timeout: 5_000, + }); + if (result.status !== 0 || !result.stdout) { + return; + } + output = result.stdout; + } catch { + return; + } + + const orphans = parseSupervisorOrphans(output); + if (orphans.length === 0) { + return; + } + + for (const orphan of orphans) { + log(`[dev-launch] reaping stale supervisor pid=${orphan.pid}: ${orphan.command}`); + try { + process.kill(orphan.pid, "SIGTERM"); + } catch { + // Already gone. + } + } + + // Escalate survivors to SIGKILL; a wedged supervisor cannot handle signals. + const followUp = setTimeout(() => { + for (const orphan of orphans) { + if (!pidIsAlive(orphan.pid)) { + continue; + } + try { + process.kill(orphan.pid, "SIGKILL"); + } catch { + // Already gone. + } + } + }, SIGKILL_FOLLOW_UP_MS); + followUp.unref?.(); +} diff --git a/src/supervisor/devOrphanWatchdog.test.ts b/src/supervisor/devOrphanWatchdog.test.ts new file mode 100644 index 00000000..c9a1e1c5 --- /dev/null +++ b/src/supervisor/devOrphanWatchdog.test.ts @@ -0,0 +1,171 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { startDevOrphanWatchdog, type DevOrphanWatchdogOptions } from "./devOrphanWatchdog"; + +type WatchdogContext = { + stop(): void; + requestShutdown: ReturnType; + exit: ReturnType; + isConnected: ReturnType boolean>>; + getParentPid: ReturnType number>>; + pidExists: ReturnType boolean>>; +}; + +const POLL_MS = 1_000; +const HARD_EXIT_MS = 500; + +describe("startDevOrphanWatchdog", () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + function start(overrides: Partial = {}): WatchdogContext { + const requestShutdown = vi.fn<() => void>(); + const exit = vi.fn<(code: number) => void>(); + const isConnected = vi.fn<() => boolean>(() => true); + const getParentPid = vi.fn<() => number>(() => 4242); + const pidExists = vi.fn<(pid: number) => boolean>(() => true); + const handle = startDevOrphanWatchdog({ + pollMs: POLL_MS, + confirmations: 2, + hardExitMs: HARD_EXIT_MS, + requestShutdown, + exit, + isConnected, + getParentPid, + pidExists, + ...overrides, + }); + return { + stop: () => handle.stop(), + requestShutdown, + exit, + isConnected, + getParentPid, + pidExists, + }; + } + + it("does not fire while the parent is alive", () => { + const ctx = start(); + vi.advanceTimersByTime(POLL_MS * 10); + expect(ctx.requestShutdown).not.toHaveBeenCalled(); + expect(ctx.exit).not.toHaveBeenCalled(); + ctx.stop(); + }); + + it("shuts down after the parent disconnects on two consecutive polls", () => { + const ctx = start(); + ctx.isConnected.mockReturnValue(false); + + vi.advanceTimersByTime(POLL_MS); + expect(ctx.requestShutdown).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(POLL_MS); + expect(ctx.requestShutdown).toHaveBeenCalledTimes(1); + expect(ctx.exit).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(HARD_EXIT_MS); + expect(ctx.exit).toHaveBeenCalledWith(1); + expect(ctx.exit).toHaveBeenCalledTimes(1); + ctx.stop(); + }); + + it("treats reparenting to pid 1 as orphaned", () => { + const ctx = start(); + ctx.getParentPid.mockReturnValue(1); + vi.advanceTimersByTime(POLL_MS * 2); + expect(ctx.requestShutdown).toHaveBeenCalledTimes(1); + ctx.stop(); + }); + + it("treats a missing parent pid as orphaned", () => { + const ctx = start(); + ctx.pidExists.mockReturnValue(false); + vi.advanceTimersByTime(POLL_MS * 2); + expect(ctx.requestShutdown).toHaveBeenCalledTimes(1); + ctx.stop(); + }); + + it("ignores a single transient miss", () => { + const ctx = start(); + ctx.isConnected.mockReturnValue(false); + vi.advanceTimersByTime(POLL_MS); + ctx.isConnected.mockReturnValue(true); + vi.advanceTimersByTime(POLL_MS * 5); + expect(ctx.requestShutdown).not.toHaveBeenCalled(); + expect(ctx.exit).not.toHaveBeenCalled(); + ctx.stop(); + }); + + it("fires only once and stops polling", () => { + const ctx = start(); + ctx.isConnected.mockReturnValue(false); + vi.advanceTimersByTime(POLL_MS * 2 + HARD_EXIT_MS); + expect(ctx.requestShutdown).toHaveBeenCalledTimes(1); + expect(ctx.exit).toHaveBeenCalledTimes(1); + + vi.advanceTimersByTime(POLL_MS * 10 + HARD_EXIT_MS * 10); + expect(ctx.requestShutdown).toHaveBeenCalledTimes(1); + expect(ctx.exit).toHaveBeenCalledTimes(1); + ctx.stop(); + }); + + it("does not flag reparenting when launched under pid 1", () => { + const getParentPid = vi.fn<() => number>(() => 1); + const pidExists = vi.fn<(pid: number) => boolean>(() => true); + const requestShutdown = vi.fn<() => void>(); + const handle = startDevOrphanWatchdog({ + pollMs: POLL_MS, + confirmations: 1, + requestShutdown, + isConnected: () => true, + getParentPid, + pidExists, + }); + vi.advanceTimersByTime(POLL_MS * 5); + expect(requestShutdown).not.toHaveBeenCalled(); + handle.stop(); + }); + + it("uses live ppid and pid probes by default", () => { + // Real probes against the live test runner: its parent exists, so the + // watchdog must stay quiet. + const requestShutdown = vi.fn<() => void>(); + const handle = startDevOrphanWatchdog({ + pollMs: POLL_MS, + confirmations: 1, + requestShutdown, + isConnected: () => true, + }); + vi.advanceTimersByTime(POLL_MS * 3); + expect(requestShutdown).not.toHaveBeenCalled(); + handle.stop(); + }); + + it("treats EPERM from the default pid probe as alive", () => { + vi.spyOn(process, "kill").mockImplementation(((pid: number, signal?: number | string) => { + if (signal === 0 || signal === undefined) { + const error = new Error("EPERM: operation not permitted") as NodeJS.ErrnoException; + error.code = "EPERM"; + throw error; + } + return true; + }) as typeof process.kill); + + const requestShutdown = vi.fn<() => void>(); + const handle = startDevOrphanWatchdog({ + pollMs: POLL_MS, + confirmations: 1, + requestShutdown, + isConnected: () => true, + }); + vi.advanceTimersByTime(POLL_MS * 3); + expect(requestShutdown).not.toHaveBeenCalled(); + handle.stop(); + }); +}); diff --git a/src/supervisor/devOrphanWatchdog.ts b/src/supervisor/devOrphanWatchdog.ts new file mode 100644 index 00000000..56975cfc --- /dev/null +++ b/src/supervisor/devOrphanWatchdog.ts @@ -0,0 +1,103 @@ +/** + * Dev-only guard: self-exit when the Electron main process that forked this + * supervisor disappears (crash, force-quit, `kill -9`, electronmon restart). + * macOS/Linux have no Job Object equivalent, so an orphaned supervisor is + * reparented to launchd/init and otherwise runs forever — sometimes at 100% + * CPU when the event loop is stuck in an exception storm. + * + * The watchdog polls cheap liveness signals and, after two consecutive + * confirmations, requests a graceful shutdown with a hard `process.exit` + * deadline so a wedged dispose can never block the exit. Packaged builds + * never install it. + */ + +export interface DevOrphanWatchdogOptions { + pollMs?: number; + /** Consecutive orphan detections required before acting. */ + confirmations?: number; + /** Grace period for a graceful shutdown before the hard exit fires. */ + hardExitMs?: number; + requestShutdown(): void; + exit?(code: number): void; + isConnected?(): boolean; + getParentPid?(): number; + pidExists?(pid: number): boolean; +} + +export interface DevOrphanWatchdogHandle { + stop(): void; +} + +const DEFAULT_POLL_MS = 2_000; +const DEFAULT_CONFIRMATIONS = 2; +const DEFAULT_HARD_EXIT_MS = 2_000; + +function defaultPidExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM means the process exists but belongs to another user; only a + // delivery failure (ESRCH-style) proves it is gone. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +export function startDevOrphanWatchdog(options: DevOrphanWatchdogOptions): DevOrphanWatchdogHandle { + const pollMs = options.pollMs ?? DEFAULT_POLL_MS; + const confirmations = Math.max(1, options.confirmations ?? DEFAULT_CONFIRMATIONS); + const hardExitMs = options.hardExitMs ?? DEFAULT_HARD_EXIT_MS; + const isConnected = options.isConnected ?? (() => Boolean(process.connected)); + const getParentPid = options.getParentPid ?? (() => process.ppid); + const pidExists = options.pidExists ?? defaultPidExists; + const exit = options.exit ?? ((code: number) => process.exit(code)); + + const initialParentPid = getParentPid(); + let consecutiveMisses = 0; + let fired = false; + + const isParentGone = (): boolean => { + // IPC EOF is the strongest signal: the fork contract closes the channel + // even when the parent dies hard. PID reuse cannot fool it. + if (!isConnected()) { + return true; + } + // Reparented to init/launchd after the parent died. Skip the heuristic + // when we legitimately started under pid 1 (e.g. container inits). + if (initialParentPid !== 1 && getParentPid() === 1) { + return true; + } + return !pidExists(initialParentPid); + }; + + const timer = setInterval(() => { + if (fired) { + return; + } + if (!isParentGone()) { + consecutiveMisses = 0; + return; + } + consecutiveMisses += 1; + if (consecutiveMisses < confirmations) { + return; + } + fired = true; + clearInterval(timer); + console.error( + `[supervisor] dev orphan watchdog: parent pid ${initialParentPid} is gone; shutting down`, + ); + // Hard deadline first: the graceful shutdown below must never be able to + // wedge the exit path. + const hardExit = setTimeout(() => exit(1), hardExitMs); + hardExit.unref?.(); + options.requestShutdown(); + }, pollMs); + timer.unref?.(); + + return { + stop() { + clearInterval(timer); + }, + }; +} diff --git a/src/supervisor/devUncaughtStorm.test.ts b/src/supervisor/devUncaughtStorm.test.ts new file mode 100644 index 00000000..235d06d7 --- /dev/null +++ b/src/supervisor/devUncaughtStorm.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import { createUncaughtStormDetector } from "./devUncaughtStorm"; + +describe("createUncaughtStormDetector", () => { + const options = { limit: 3, windowMs: 10_000 }; + + it("does not trip below the limit", () => { + const detector = createUncaughtStormDetector(options); + expect(detector.record(0)).toBe(false); + expect(detector.record(1)).toBe(false); + }); + + it("trips when the limit occurs inside the window", () => { + const detector = createUncaughtStormDetector(options); + detector.record(0); + detector.record(1); + expect(detector.record(2)).toBe(true); + }); + + it("does not trip when occurrences are spread beyond the window", () => { + const detector = createUncaughtStormDetector(options); + detector.record(0); + detector.record(20_000); + expect(detector.record(40_000)).toBe(false); + }); + + it("slides: old occurrences fall out and a fresh burst trips it", () => { + const detector = createUncaughtStormDetector(options); + detector.record(0); + detector.record(20_000); + detector.record(40_000); + expect(detector.record(40_100)).toBe(false); + expect(detector.record(40_200)).toBe(true); + }); + + it("keeps reporting while the storm continues", () => { + const detector = createUncaughtStormDetector(options); + detector.record(0); + detector.record(1); + expect(detector.record(2)).toBe(true); + expect(detector.record(3)).toBe(true); + }); +}); diff --git a/src/supervisor/devUncaughtStorm.ts b/src/supervisor/devUncaughtStorm.ts new file mode 100644 index 00000000..49a0c129 --- /dev/null +++ b/src/supervisor/devUncaughtStorm.ts @@ -0,0 +1,35 @@ +export interface UncaughtStormOptions { + /** How many occurrences inside the window constitute a storm. */ + limit: number; + windowMs: number; +} + +export interface UncaughtStormDetector { + /** Records an occurrence; returns true once the storm threshold is met. */ + record(now: number): boolean; +} + +/** + * Detects bursts of uncaught exceptions — e.g. something throwing on every + * `setImmediate` tick, which monopolizes the event loop and burns a full CPU + * core forever. Sporadic, isolated errors must not trip the detector so + * recoverable failures keep their existing resilient behavior. + */ +export function createUncaughtStormDetector(options: UncaughtStormOptions): UncaughtStormDetector { + const limit = Math.max(2, options.limit); + const timestamps: number[] = []; + return { + record(now: number): boolean { + timestamps.push(now); + if (timestamps.length > limit) { + timestamps.shift(); + } + if (timestamps.length < limit) { + return false; + } + const oldest = timestamps[0]; + const newest = timestamps[timestamps.length - 1]; + return oldest !== undefined && newest !== undefined && newest - oldest <= options.windowMs; + }, + }; +} diff --git a/src/supervisor/index.ts b/src/supervisor/index.ts index 10a6d5a3..f563ce78 100644 --- a/src/supervisor/index.ts +++ b/src/supervisor/index.ts @@ -4,14 +4,18 @@ import { flushSupervisorSentry, initializeSupervisorSentry, } from "./diagnostics/sentry"; +import { startDevOrphanWatchdog } from "./devOrphanWatchdog"; +import { createUncaughtStormDetector } from "./devUncaughtStorm"; import { handleSupervisorIpcFailure } from "./ipcFailure"; import { createSupervisorIpcHandlers } from "./ipcHandlers"; import { SupervisorRuntime } from "./supervisorRuntime"; import { configureSecretStorageKey } from "./secretStorage"; +const isDev = process.env.PORACODE_IS_DEV === "1" || Boolean(process.env.VITE_DEV_SERVER_URL); + initializeSupervisorSentry({ appVersion: process.env.PORACODE_APP_VERSION ?? process.env.npm_package_version ?? "dev", - isDev: process.env.PORACODE_IS_DEV === "1" || Boolean(process.env.VITE_DEV_SERVER_URL), + isDev, }); configureSecretStorageKey(process.env.PORACODE_SECRET_STORAGE_KEY); delete process.env.PORACODE_SECRET_STORAGE_KEY; @@ -24,9 +28,16 @@ const handlers = createSupervisorIpcHandlers(runtime); let isShuttingDown = false; const SUPERVISOR_SHUTDOWN_TIMEOUT_MS = 5_000; +const DEV_SHUTDOWN_REPEAT_FORCE_EXIT_MS = 250; async function shutdownSupervisor(exitCode = 0): Promise { if (isShuttingDown) { + if (isDev) { + // Dev-only: a repeated disconnect/signal means the first shutdown has + // not finished yet. Force the exit instead of no-opping so a soft kill + // can never look hung. + setTimeout(() => process.exit(exitCode), DEV_SHUTDOWN_REPEAT_FORCE_EXIT_MS).unref(); + } return; } isShuttingDown = true; @@ -72,9 +83,29 @@ process.on("SIGTERM", () => { void shutdownSupervisor(0); }); +if (isDev) { + startDevOrphanWatchdog({ + requestShutdown: () => { + void shutdownSupervisor(1); + }, + }); +} + +const devUncaughtStorm = createUncaughtStormDetector({ limit: 3, windowMs: 10_000 }); + process.on("uncaughtException", (error) => { console.error("[supervisor] uncaught exception:", error); captureSupervisorException(error, { "poracode.feature_area": "supervisor" }); + // Dev-only: a rapid burst of uncaught exceptions means the event loop is + // stuck re-throwing (observed wedging orphaned dev supervisors at 100% + // CPU). Exit instead of lingering; the main-process client restarts + // non-zero exits, surfacing the failure in the dev console. + if (isDev && devUncaughtStorm.record(Date.now())) { + console.error("[supervisor] uncaught exception storm detected; exiting"); + setTimeout(() => process.exit(1), 1_500).unref(); + void flushSupervisorSentry(750).finally(() => process.exit(1)); + return; + } void flushSupervisorSentry(); });