feat(incident): always-on App Run Journal (Phases 1–8 + run-identity unification)

src/main/incident/appRunIds.ts

@@ -0,0 +1,35 @@
+import { randomBytes } from 'node:crypto'
+
+export function createAppRunId(now = new Date(), pid = process.pid): string {
+  const safeStamp = now.toISOString().replace(/[:.]/g, '-')
+  const suffix = randomBytes(3).toString('hex')
+  return `${safeStamp}-main-${pid}-${suffix}`
+}
+
+// The canonical, process-wide app run identity.
+//
+// WHY a singleton: the whole thesis of the journal is "stop manually matching
+// timestamps between subsystems." That only works if every diagnostic — the
+// incident journal, the performance run, (later) heap snapshots and debug
+// bundles — stamps the SAME id. So the id is minted exactly once, lazily, the
+// first time anything asks, and cached for the life of the process. Whichever
+// subsystem starts first mints it (AppRunJournal in normal runs; PerformanceService
+// at module load when AGENT_CODE_PERF is on); everyone else gets the same value.
+// This replaces the previous state where the journal and PerformanceService each
+// minted their own unrelated run id — the single biggest correlation gap.
+let canonicalAppRunId: string | null = null
+
+export function getAppRunId(): string {
+  if (canonicalAppRunId === null) {
+    canonicalAppRunId = createAppRunId()
+  }
+  return canonicalAppRunId
+}
+
+// Incident ids only need to be unique WITHIN one app run (they're already
+// scoped by appRunId on disk), so a timestamp + short random suffix is enough.
+export function createIncidentId(now = new Date()): string {
+  const safeStamp = now.toISOString().replace(/[:.]/g, '-')
+  const suffix = randomBytes(3).toString('hex')
+  return `${safeStamp}-${suffix}`
+}

src/main/incident/installCrashHooks.ts

@@ -0,0 +1,99 @@
+import type { AppRunJournal } from '@main/incident/AppRunJournal.js'
+
+// Process-level crash hooks for the Electron MAIN process.
+//
+// WHY this is separate from the window hooks and from the journal itself:
+// crash hooks are easy to get subtly wrong (recursion if recording the crash
+// throws, async work that never drains before exit, swallowing a fatal that
+// should kill the process). Isolating them keeps that danger reviewable, and
+// keeps AppRunJournal a pure writer with no opinions about process lifecycle.
+//
+// These complement — they do not replace — electron's crashReporter/Crashpad
+// (started in index.ts). Crashpad catches NATIVE death (V8 aborts, SIGSEGV in a
+// native addon) that never reaches JS; these JS hooks catch the JS-level faults
+// (uncaught exceptions, unhandled rejections, process warnings) that Crashpad
+// can't see. You need both.
+
+export type CrashHookOptions = {
+  journal: AppRunJournal
+  // Synchronous lock release. A fatal uncaughtException exits the process
+  // immediately, bypassing the normal before-quit/will-quit handlers that
+  // release the state-process lock — so without this, a main crash would strand
+  // the lock and block the NEXT launch (acquireStateProcessLock sees a live
+  // pid). The journal does not own the lock, so the owner injects the release.
+  releaseLockSync: () => void
+}
+
+// Distinct unhandled-rejection messages we will journal before we stop, so a
+// rejection loop (same error thrown thousands of times) can't flood the disk.
+const MAX_DISTINCT_REJECTIONS = 50
+
+export function installProcessCrashHooks({ journal, releaseLockSync }: CrashHookOptions): void {
+  let handlingFatal = false
+
+  process.on('uncaughtException', (error) => {
+    // Recursion guard: if recording the crash itself throws (e.g. disk full),
+    // do not re-enter — just release the lock and die.
+    if (handlingFatal) {
+      try { releaseLockSync() } catch { /* already dying */ }
+      process.exit(1)
+      return
+    }
+    handlingFatal = true
+    // An uncaught exception leaves main in an undefined state; continuing is
+    // more dangerous than exiting. recordIncident() writes synchronously AND
+    // flushes the pending event breadcrumbs first, so the timeline survives.
+    // We deliberately do NOT mark a clean shutdown (this run WAS unclean — a
+    // future prior-run classifier must see it that way), but we DO release the
+    // lock so the relaunch isn't blocked.
+    try {
+      journal.recordIncident({
+        kind: 'main.uncaught_exception',
+        severity: 'fatal',
+        process: 'main',
+        error,
+      })
+    } catch { /* best-effort: we're already crashing */ }
+    try { releaseLockSync() } catch { /* best-effort */ }
+    process.exit(1)
+  })
+
+  // Unhandled rejections are non-fatal (Node keeps running), but a hot loop can
+  // emit the same one thousands of times. Coalesce by message: journal the
+  // first occurrence of each distinct rejection, and cap distinct keys so the
+  // map can't grow without bound.
+  const journaledRejections = new Set<string>()
+  process.on('unhandledRejection', (reason) => {
+    const key = reason instanceof Error ? reason.message : String(reason)
+    if (journaledRejections.has(key)) return
+    if (journaledRejections.size >= MAX_DISTINCT_REJECTIONS) return
+    journaledRejections.add(key)
+    journal.recordIncident({
+      kind: 'main.unhandled_rejection',
+      severity: 'error',
+      process: 'main',
+      error: reason,
+    })
+  })
+
+  // process warnings (deprecations, MaxListenersExceeded, etc.) are low-severity
+  // but occasionally the first sign of a leak. Coalesce exactly like rejections:
+  // recordIncident() is SYNCHRONOUS (flushSync + appendFileSync), so an uncapped
+  // warning storm (a repeating MaxListenersExceededWarning, say) would drive one
+  // blocking disk write per warning on the main thread — the journal adding the
+  // very pressure it exists to observe. Journal the first occurrence of each
+  // distinct name+message, capped.
+  const journaledWarnings = new Set<string>()
+  process.on('warning', (warning) => {
+    const key = `${warning.name}: ${warning.message}`
+    if (journaledWarnings.has(key)) return
+    if (journaledWarnings.size >= MAX_DISTINCT_REJECTIONS) return
+    journaledWarnings.add(key)
+    journal.recordIncident({
+      kind: 'main.warning',
+      severity: 'warn',
+      process: 'main',
+      error: warning,
+    })
+  })
+}

src/main/incident/installWindowIncidentHooks.ts

@@ -0,0 +1,101 @@
+import { app } from 'electron'
+
+import type { AppRunJournal } from '@main/incident/AppRunJournal.js'
+
+// Window / child-process incident hooks.
+//
+// WHY app-level + per-window-on-create instead of editing mainWindow.ts:
+// these signals must be captured for EVERY window/webContents, including any
+// future secondary windows, and main must report them on its own — the renderer
+// can't report its own crash once it's gone (a core invariant of the plan).
+// Attaching via the `browser-window-created` app event means we instrument
+// windows uniformly without coupling the journal to the window-creation code.
+
+export function installWindowIncidentHooks(journal: AppRunJournal): void {
+  // Renderer/GPU/utility process death. This is THE renderer-crash signal —
+  // main observes it even though the renderer is already gone.
+  app.on('render-process-gone', (_event, _webContents, details) => {
+    // Skip 'clean-exit' entirely — it's routine teardown (window closed / app
+    // quitting) and recording it would fill incidents.jsonl with steady noise
+    // (plan: ignore clean-exit unless it happens in an unexpected phase).
+    if (details.reason === 'clean-exit') return
+    // 'killed' frequently occurs during macOS quit teardown, so it's a warning,
+    // not a fatal — the genuinely fatal reasons are oom / crashed / abnormal-exit
+    // / launch-failed / integrity-failure.
+    const fatal = details.reason !== 'killed'
+    journal.recordIncident({
+      kind: 'window.render_process_gone',
+      severity: fatal ? 'fatal' : 'warn',
+      process: 'renderer',
+      reason: details.reason,
+      exitCode: details.exitCode,
+    })
+  })
+
+  // Electron child processes (GPU, utility, pepper plugin, etc.) dying.
+  app.on('child-process-gone', (_event, details) => {
+    const clean = details.reason === 'clean-exit'
+    journal.recordIncident({
+      kind: 'electron.child_process_gone',
+      severity: clean ? 'warn' : 'error',
+      process: 'child',
+      reason: details.reason,
+      exitCode: details.exitCode,
+      context: { type: details.type, name: details.name, serviceName: details.serviceName },
+    })
+  })
+
+  app.on('browser-window-created', (_event, window) => {
+    // Track when this window went unresponsive so 'responsive' can report how
+    // long it was frozen. Keyed by window id; cleared on recovery.
+    let unresponsiveSince: number | null = null
+
+    window.on('unresponsive', () => {
+      unresponsiveSince = Date.now()
+      journal.recordIncident({
+        kind: 'window.unresponsive',
+        severity: 'error',
+        process: 'renderer',
+        context: { windowId: window.id },
+      })
+    })
+
+    window.on('responsive', () => {
+      const frozenMs = unresponsiveSince === null ? undefined : Date.now() - unresponsiveSince
+      unresponsiveSince = null
+      journal.recordIncident({
+        kind: 'window.responsive',
+        severity: 'warn',
+        process: 'renderer',
+        context: { windowId: window.id, frozenMs },
+      })
+    })
+
+    window.webContents.on('preload-error', (_e, preloadPath, error) => {
+      // A failed preload means the renderer boots without its IPC bridge — the
+      // app is effectively broken even though the window "loaded".
+      journal.recordIncident({
+        kind: 'window.preload_error',
+        severity: 'error',
+        process: 'renderer',
+        error,
+        context: { preloadPath },
+      })
+    })
+
+    window.webContents.on('did-fail-load', (_e, errorCode, errorDescription, validatedURL, isMainFrame) => {
+      // ERR_ABORTED (-3) is the routine "navigation superseded / cancelled"
+      // case (and fires constantly for cancelled sub-resource loads). Ignoring
+      // it keeps did_fail_load meaningful — a main-frame failure is a real
+      // "the app didn't load" incident; a sub-frame one is a warning.
+      if (errorCode === -3) return
+      journal.recordIncident({
+        kind: 'window.did_fail_load',
+        severity: isMainFrame ? 'error' : 'warn',
+        process: 'renderer',
+        reason: errorDescription,
+        context: { errorCode, validatedURL, isMainFrame },
+      })
+    })
+  })
+}

src/main/incident/journalTypes.ts

@@ -0,0 +1,134 @@
+export type AppRunJournalSeverity = 'debug' | 'info' | 'warn' | 'error' | 'fatal'
+
+export type AppRunJournalIds = {
+  sessionId?: string
+  providerSessionId?: string
+  orchestrationRunId?: string
+  orchestrationRequestId?: string
+  mcpSessionId?: string
+  workspaceSaveSeq?: number
+}
+
+export type AppRunJournalEventInput = {
+  area: string
+  name: string
+  severity?: AppRunJournalSeverity
+  ids?: AppRunJournalIds
+  data?: Record<string, unknown>
+}
+
+export type AppRunJournalEvent = AppRunJournalEventInput & {
+  schemaVersion: 1
+  seq: number
+  ts: number
+  tsIso: string
+  monotonicMs: number
+  appRunId: string
+  severity: AppRunJournalSeverity
+}
+
+export type AppRunJournalManifest = {
+  schemaVersion: 1
+  appRunId: string
+  startedAt: number
+  startedAtIso: string
+  pid: number
+  platform: NodeJS.Platform
+  arch: string
+  node: string
+  electron: string | undefined
+  chrome: string | undefined
+  appVersion: string
+  stateDir: string
+  perfEnabled: boolean
+  lock: {
+    // WHY only the path, never the lock token: this manifest is an always-on,
+    // retained-for-50-runs forensic file. The process-lock token is the secret
+    // that gates lock REMOVAL (processLock compares it before deleting the
+    // lock), so persisting it in cleartext would leak a security-relevant
+    // credential to disk. The path is enough to correlate which lock this run
+    // held; the token stays in memory only.
+    path: string
+  }
+}
+
+// Incidents are higher-level failure facts (vs. events, which are routine
+// lifecycle breadcrumbs). An incident is something expected to matter AFTER a
+// restart: a crash, a freeze, a child-process death. They live in their own
+// incidents.jsonl so a triage scan never has to wade through the event stream.
+export type AppRunIncidentSeverity = 'warn' | 'error' | 'fatal'
+
+export type AppRunIncidentKind =
+  | 'app.prior_unclean_shutdown'
+  | 'app.startup_failed'
+  | 'main.uncaught_exception'
+  | 'main.unhandled_rejection'
+  | 'main.warning'
+  | 'window.render_process_gone'
+  | 'window.unresponsive'
+  | 'window.responsive'
+  | 'window.preload_error'
+  | 'window.did_fail_load'
+  | 'electron.child_process_gone'
+  | 'heap.pressure'
+  | 'renderer.error'
+  | 'renderer.unhandledrejection'
+  | 'session.input_write_failed'
+  | 'orchestration.request_timeout'
+  | 'orchestration.prompt_delivery_failed'
+  | 'mcp.host_start_failed'
+
+export type AppRunIncidentInput = {
+  kind: AppRunIncidentKind
+  severity: AppRunIncidentSeverity
+  process?: 'main' | 'renderer' | 'gpu' | 'utility' | 'child'
+  reason?: string
+  exitCode?: number
+  // Raw error from a hook; normalized to {name,message,stack} before persistence.
+  error?: unknown
+  context?: Record<string, unknown>
+}
+
+export type AppRunIncident = {
+  schemaVersion: 1
+  incidentId: string
+  appRunId: string
+  seq: number
+  ts: number
+  tsIso: string
+  kind: AppRunIncidentKind
+  severity: AppRunIncidentSeverity
+  process?: 'main' | 'renderer' | 'gpu' | 'utility' | 'child'
+  reason?: string
+  exitCode?: number
+  error?: { name?: string; message: string; stack?: string }
+  context?: Record<string, unknown>
+}
+
+export type AppRunHeartbeat = {
+  schemaVersion: 1
+  appRunId: string
+  seq: number
+  ts: number
+  tsIso: string
+  uptimeMs: number
+  pid: number
+  memory: {
+    rss: number
+    heapUsed: number
+    heapTotal: number
+    heapLimit: number
+    external: number
+    arrayBuffers: number
+  }
+  mainEventLoop: {
+    delayMeanMs: number
+    delayMaxMs: number
+    delayP99Ms: number
+  }
+  window: {
+    count: number
+    focused: boolean
+  }
+  lastEventSeq: number
+}