diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index 72b0ebb7993..6d712a2b08d 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -29,6 +29,7 @@ import { PLATFORM, PROTOCOL_SCHEME, } from "shared/constants"; +import { flushAnalytics } from "./lib/analytics"; import { initAppState } from "./lib/app-state"; import { requestAppleEventsAccess } from "./lib/apple-events-permission"; import { isUpdateReadyToInstall, setupAutoUpdater } from "./lib/auto-updater"; @@ -46,6 +47,7 @@ import { } from "./lib/persistence/persistence"; import { ensureProjectIconsDir, getProjectIconPath } from "./lib/project-icons"; import { runQuitCleanup } from "./lib/quit-sequence"; +import { startResourceDigest, stopResourceDigest } from "./lib/resource-digest"; import { initSentry } from "./lib/sentry"; import { prewarmTerminalRuntime, @@ -244,6 +246,7 @@ app.on("before-quit", async (event) => { } isQuitting = true; + stopResourceDigest(); await runQuitCleanup({ isDev, forceFullCleanup, @@ -253,6 +256,7 @@ app.on("before-quit", async (event) => { disposeTerminalHostClient, shutdownPersistence: shutdownTanstackDbPersistence, disposeTray, + flushAnalytics, forceExit: (code) => app.exit(code), }); }); @@ -503,6 +507,7 @@ if (!gotTheLock) { await makeAppSetup(() => MainWindow()); setupAutoUpdater(); + startResourceDigest(); initTray(); const coldStartUrl = findDeepLinkInArgv(process.argv); diff --git a/apps/desktop/src/main/lib/analytics/index.ts b/apps/desktop/src/main/lib/analytics/index.ts index 7e36d4bd312..ab3dfb84e63 100644 --- a/apps/desktop/src/main/lib/analytics/index.ts +++ b/apps/desktop/src/main/lib/analytics/index.ts @@ -37,6 +37,26 @@ export function setUserId(id: string | null): void { userId = id; } +/** + * Drain captures onto the wire before process exit. `flushAt: 1` starts the + * HTTP send at capture time, but `app.exit()` right after a quit-time capture + * kills the request mid-flight; awaiting this (time-bounded) closes that race. + * The client is unusable afterwards, so only call it from the quit path. + */ +export async function flushAnalytics(timeoutMs = 2_000): Promise { + if (!posthog) return; + try { + await Promise.race([ + posthog.shutdown(timeoutMs), + new Promise((resolve) => { + setTimeout(resolve, timeoutMs).unref?.(); + }), + ]); + } catch { + // Losing a final telemetry event must never block quit. + } +} + export function track( event: string, properties?: Record, diff --git a/apps/desktop/src/main/lib/quit-sequence.test.ts b/apps/desktop/src/main/lib/quit-sequence.test.ts index 1f23af0e15e..c87ae008174 100644 --- a/apps/desktop/src/main/lib/quit-sequence.test.ts +++ b/apps/desktop/src/main/lib/quit-sequence.test.ts @@ -105,6 +105,38 @@ describe("runQuitCleanup", () => { expect(h.forceExit).toHaveBeenCalledWith(0); }); + test("waits for the analytics drain before force-exiting", async () => { + let resolveDrain: () => void = () => {}; + const drain = new Promise((resolve) => { + resolveDrain = resolve; + }); + const h = createHarness({ flushAnalytics: () => drain }); + + let finished = false; + const run = runQuitCleanup(h.deps).then(() => { + finished = true; + }); + await Promise.resolve(); + expect(finished).toBe(false); + expect(h.forceExit).not.toHaveBeenCalled(); + + resolveDrain(); + await run; + expect(h.forceExit).toHaveBeenCalledWith(0); + }); + + test("force-exits even if the analytics drain rejects", async () => { + const h = createHarness({ + flushAnalytics: async () => { + throw new Error("network down"); + }, + }); + + await runQuitCleanup(h.deps); + + expect(h.forceExit).toHaveBeenCalledWith(0); + }); + test("force-exits even if cleanup throws", async () => { const h = createHarness({ stopHostServices: () => { diff --git a/apps/desktop/src/main/lib/quit-sequence.ts b/apps/desktop/src/main/lib/quit-sequence.ts index 0403a42dd2a..cb6ee594142 100644 --- a/apps/desktop/src/main/lib/quit-sequence.ts +++ b/apps/desktop/src/main/lib/quit-sequence.ts @@ -20,6 +20,8 @@ export interface QuitCleanupDeps { disposeTerminalHostClient: () => void; shutdownPersistence: () => void; disposeTray: () => void; + /** Time-bounded analytics drain so quit-time captures reach the wire. */ + flushAnalytics?: () => Promise; forceExit: (code: number) => void; scheduleTimer?: (callback: () => void, delayMs: number) => void; logError?: (message: string, error: unknown) => void; @@ -36,12 +38,17 @@ export async function runQuitCleanup(deps: QuitCleanupDeps): Promise { shutdownPersistence, disposeTray, forceExit, + flushAnalytics, scheduleTimer = (callback, delayMs) => { setTimeout(callback, delayMs); }, logError = (message, error) => console.error(message, error), } = deps; + // Start draining quit-time analytics captures while cleanup runs; awaited + // (already time-bounded by the implementation) before the forced exit. + const analyticsDrain = flushAnalytics?.().catch(() => {}); + try { stopHostServices(); if (isDev || forceFullCleanup) { @@ -67,5 +74,6 @@ export async function runQuitCleanup(deps: QuitCleanupDeps): Promise { return; } + await analyticsDrain; forceExit(0); } diff --git a/apps/desktop/src/main/lib/resource-digest/index.ts b/apps/desktop/src/main/lib/resource-digest/index.ts new file mode 100644 index 00000000000..4092f473e95 --- /dev/null +++ b/apps/desktop/src/main/lib/resource-digest/index.ts @@ -0,0 +1,20 @@ +export { + ALERT_COOLDOWN_MS, + ALERT_KINDS, + type AlertKind, + DIGEST_SCALAR_KEYS, + IDLE_SKIP_THRESHOLD_SECONDS, + isResourceDigestEnabled, + MIN_SAMPLES_TO_FLUSH_ON_QUIT, + RESOURCE_ALERT_EVENT, + RESOURCE_DIGEST_EVENT, + type ResourceDigestDeps, + type ResourceSample, + recordWindowUnresponsive, + SAMPLE_INTERVAL_MS, + SAMPLE_JITTER_MS, + SAMPLE_KEYS, + type SampleKey, + startResourceDigest, + stopResourceDigest, +} from "./resource-digest"; diff --git a/apps/desktop/src/main/lib/resource-digest/resource-digest.test.ts b/apps/desktop/src/main/lib/resource-digest/resource-digest.test.ts new file mode 100644 index 00000000000..2b46fcc2ac2 --- /dev/null +++ b/apps/desktop/src/main/lib/resource-digest/resource-digest.test.ts @@ -0,0 +1,241 @@ +import { afterEach, describe, expect, it, jest } from "bun:test"; +import { + ALERT_COOLDOWN_MS, + IDLE_SKIP_THRESHOLD_SECONDS, + MIN_SAMPLES_TO_FLUSH_ON_QUIT, + RESOURCE_ALERT_EVENT, + RESOURCE_DIGEST_EVENT, + type ResourceDigestDeps, + type ResourceSample, + recordWindowUnresponsive, + SAMPLE_INTERVAL_MS, + SAMPLE_KEYS, + startResourceDigest, + stopResourceDigest, +} from "./resource-digest"; + +type TrackCall = { event: string; properties: Record }; + +const DAY_MS = 24 * 60 * 60_000; +// Deps use random: () => 0, so every tick is scheduled at exactly the +// interval; advancing by exactly that never double-fires. +const TICK_MS = SAMPLE_INTERVAL_MS; + +function makeSample(overrides: Partial = {}): ResourceSample { + const sample = { uptime_seconds: 60 } as ResourceSample; + for (const key of SAMPLE_KEYS) sample[key] = 0; + return { ...sample, ...overrides }; +} + +function createDeps(overrides: Partial = {}): { + deps: ResourceDigestDeps; + calls: TrackCall[]; + clock: { epochMs: number }; +} { + const calls: TrackCall[] = []; + const clock = { epochMs: Date.UTC(2026, 0, 5, 12, 0, 0) }; + const deps: ResourceDigestDeps = { + getSample: async () => makeSample(), + getIdleSeconds: () => 0, + track: (event, properties) => calls.push({ event, properties }), + isEnabled: () => true, + now: () => clock.epochMs, + random: () => 0, + ...overrides, + }; + return { deps, calls, clock }; +} + +/** Fire one sampler tick and let the async sample settle. */ +async function tick( + clock: { epochMs: number }, + advanceDayMs = 0, +): Promise { + clock.epochMs += advanceDayMs; + jest.advanceTimersByTime(TICK_MS); + // takeSample is async; drain microtasks so the sample lands before asserting. + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +afterEach(() => { + stopResourceDigest(); + jest.useRealTimers(); +}); + +describe("daily digest", () => { + it("aggregates samples into p50/p90/max and flushes on day rollover", async () => { + jest.useFakeTimers(); + const values = [100, 200, 300, 400, 500]; + let index = 0; + const { deps, calls, clock } = createDeps({ + getSample: async () => + makeSample({ + electron_renderer_rss_bytes: values[index++ % values.length] ?? 0, + uptime_seconds: index * 60, + }), + }); + startResourceDigest(deps); + + for (let i = 0; i < values.length; i++) await tick(clock); + expect(calls).toHaveLength(0); + + // Crossing the UTC day boundary flushes the previous day. + await tick(clock, DAY_MS); + const digests = calls.filter((c) => c.event === RESOURCE_DIGEST_EVENT); + expect(digests).toHaveLength(1); + const props = digests[0]?.properties as Record; + expect(props.sample_count).toBe(values.length); + expect(props.electron_renderer_rss_bytes_p50).toBe(300); + expect(props.electron_renderer_rss_bytes_max).toBe(500); + expect(props.uptime_seconds_max).toBe(values.length * 60); + expect(props.flushed_on_quit).toBe(0); + expect(props.active_minutes).toBe(values.length * 5); + }); + + it("emits only finite numbers under allowlisted digest keys", async () => { + jest.useFakeTimers(); + const { deps, calls, clock } = createDeps({ + getSample: async () => + makeSample({ + electron_total_rss_bytes: Number.NaN, + children_rss_bytes: -5, + }), + }); + startResourceDigest(deps); + await tick(clock); + await tick(clock, DAY_MS); + + const props = calls[0]?.properties as Record; + expect(props).toBeDefined(); + for (const [key, value] of Object.entries(props)) { + expect(typeof value).toBe("number"); + expect(Number.isFinite(value)).toBe(true); + expect( + SAMPLE_KEYS.some((k) => key.startsWith(`${k}_`)) || + [ + "sample_count", + "active_minutes", + "uptime_seconds_max", + "unresponsive_count", + "flushed_on_quit", + ].includes(key), + ).toBe(true); + } + // NaN and negative inputs normalize to 0 rather than leaking through. + expect(props.electron_total_rss_bytes_max).toBe(0); + expect(props.children_rss_bytes_max).toBe(0); + }); + + it("skips samples while the machine is idle", async () => { + jest.useFakeTimers(); + let idle = 0; + let sampleCalls = 0; + const { deps, clock } = createDeps({ + getIdleSeconds: () => idle, + getSample: async () => { + sampleCalls += 1; + return makeSample(); + }, + }); + startResourceDigest(deps); + + await tick(clock); + idle = IDLE_SKIP_THRESHOLD_SECONDS + 1; + await tick(clock); + await tick(clock); + idle = 0; + await tick(clock); + + expect(sampleCalls).toBe(2); + }); + + it("flushes a partial day on stop, but drops digests below the quit minimum", async () => { + jest.useFakeTimers(); + const first = createDeps(); + startResourceDigest(first.deps); + for (let i = 0; i < MIN_SAMPLES_TO_FLUSH_ON_QUIT; i++) + await tick(first.clock); + stopResourceDigest(); + expect( + first.calls.filter((c) => c.event === RESOURCE_DIGEST_EVENT), + ).toHaveLength(1); + expect(first.calls[0]?.properties.flushed_on_quit).toBe(1); + + const second = createDeps(); + startResourceDigest(second.deps); + await tick(second.clock); + stopResourceDigest(); + expect( + second.calls.filter((c) => c.event === RESOURCE_DIGEST_EVENT), + ).toHaveLength(0); + }); + + it("does not start when disabled and never double-starts", async () => { + jest.useFakeTimers(); + const disabled = createDeps({ isEnabled: () => false }); + startResourceDigest(disabled.deps); + await tick(disabled.clock); + stopResourceDigest(); + expect(disabled.calls).toHaveLength(0); + + let sampleCalls = 0; + const { deps, clock } = createDeps({ + getSample: async () => { + sampleCalls += 1; + return makeSample(); + }, + }); + startResourceDigest(deps); + startResourceDigest(deps); + await tick(clock); + expect(sampleCalls).toBe(1); + }); +}); + +describe("alerts", () => { + it("emits a threshold alert with the sample payload, honoring the cooldown", async () => { + jest.useFakeTimers(); + const { deps, calls, clock } = createDeps({ + getSample: async () => makeSample({ electron_renderer_rss_bytes: 4e9 }), + }); + startResourceDigest(deps); + + await tick(clock); + await tick(clock); + let alerts = calls.filter((c) => c.event === RESOURCE_ALERT_EVENT); + expect(alerts).toHaveLength(1); + expect(alerts[0]?.properties.alert_kind).toBe("renderer_rss_high"); + expect(alerts[0]?.properties.electron_renderer_rss_bytes).toBe(4e9); + + // Past the cooldown the same condition may alert again. + await tick(clock, ALERT_COOLDOWN_MS); + alerts = calls.filter((c) => c.event === RESOURCE_ALERT_EVENT); + expect(alerts).toHaveLength(2); + }); + + it("counts unresponsive episodes into the digest and alerts once per cooldown", async () => { + jest.useFakeTimers(); + const { deps, calls, clock } = createDeps(); + startResourceDigest(deps); + await tick(clock); + + recordWindowUnresponsive(); + recordWindowUnresponsive(); + + const alerts = calls.filter((c) => c.event === RESOURCE_ALERT_EVENT); + expect(alerts).toHaveLength(1); + expect(alerts[0]?.properties.alert_kind).toBe("window_unresponsive"); + + await tick(clock, DAY_MS); + const digest = calls.find((c) => c.event === RESOURCE_DIGEST_EVENT); + expect(digest?.properties.unresponsive_count).toBe(2); + }); + + it("is a no-op for unresponsive episodes when not running", () => { + const { calls } = createDeps(); + recordWindowUnresponsive(); + expect(calls).toHaveLength(0); + }); +}); diff --git a/apps/desktop/src/main/lib/resource-digest/resource-digest.ts b/apps/desktop/src/main/lib/resource-digest/resource-digest.ts new file mode 100644 index 00000000000..f9401baea03 --- /dev/null +++ b/apps/desktop/src/main/lib/resource-digest/resource-digest.ts @@ -0,0 +1,412 @@ +// Namespace import (not named) so this module links even when a test replaces +// the `electron` mock with a shape missing some exports — the real values are +// only dereferenced lazily inside defaultDeps at runtime. + +import { monitorEventLoopDelay } from "node:perf_hooks"; +import * as electron from "electron"; +import { track } from "main/lib/analytics"; +import { DEFAULT_TELEMETRY_ENABLED } from "shared/constants"; + +/** + * Privacy-safe aggregate resource telemetry, digest edition (SUPER-1550). + * + * The original `resource_snapshot` sampler (#5777) uploaded every raw 5-minute + * sample and was removed in #5964 for doubling PostHog ingestion with + * machine-cadence events. This module keeps the same 5-minute sampling but + * aggregates locally and uploads: + * + * - one `resource_digest` event per machine per UTC day (p50/p90/max per + * metric), plus a partial digest on quit, and + * - rare `resource_alert` events when a sample crosses a severity threshold, + * rate-limited by a per-kind cooldown. + * + * Sampling contributes nothing while the machine has been idle for 30+ + * minutes, so a desktop idling over the weekend goes silent instead of + * emitting all weekend (the other #5964 objection). + * + * Hard privacy rules — payloads NEVER contain IDs, names, paths, commands, + * repo/terminal/user data, or anything derived from them. Digest properties + * are filtered through a numeric-key allowlist; the only string an alert may + * carry is its `alert_kind`, drawn from the fixed enum below. + */ +export const RESOURCE_DIGEST_EVENT = "resource_digest"; +export const RESOURCE_ALERT_EVENT = "resource_alert"; + +export const SAMPLE_INTERVAL_MS = 5 * 60_000; +// Up to 60s of extra delay per tick so the fleet never emits in lockstep. +export const SAMPLE_JITTER_MS = 60_000; +export const IDLE_SKIP_THRESHOLD_SECONDS = 30 * 60; +export const ALERT_COOLDOWN_MS = 6 * 60 * 60_000; +/** A quit-time partial digest below this many samples is noise; drop it. */ +export const MIN_SAMPLES_TO_FLUSH_ON_QUIT = 3; +/** Hard bound on retained samples per day (a full day is 288). */ +const MAX_SAMPLES_PER_DAY = 512; + +/** Per-sample metrics aggregated into p50/p90/max in the daily digest. */ +export const SAMPLE_KEYS = [ + "electron_total_rss_bytes", + "electron_main_rss_bytes", + "electron_renderer_rss_bytes", + "electron_other_rss_bytes", + "electron_process_count", + "window_count", + "web_contents_count", + "process_heap_used_bytes", + "process_external_bytes", + "children_rss_bytes", + "children_process_count", + "host_memory_used_percent", + "host_load_average_1m", + "event_loop_p99_ms", +] as const; + +export type SampleKey = (typeof SAMPLE_KEYS)[number]; +export type ResourceSample = Record & { + uptime_seconds: number; +}; + +/** Scalar digest properties emitted alongside the aggregated keys. */ +export const DIGEST_SCALAR_KEYS = [ + "sample_count", + "active_minutes", + "uptime_seconds_max", + "unresponsive_count", + "flushed_on_quit", +] as const; + +export const ALERT_KINDS = [ + "renderer_rss_high", + "total_rss_high", + "children_rss_high", + "host_memory_pressure", + "main_event_loop_stall", + "window_unresponsive", +] as const; +export type AlertKind = (typeof ALERT_KINDS)[number]; + +const ALERT_THRESHOLDS: Record< + Exclude, + { key: SampleKey; above: number } +> = { + renderer_rss_high: { key: "electron_renderer_rss_bytes", above: 3e9 }, + total_rss_high: { key: "electron_total_rss_bytes", above: 5e9 }, + children_rss_high: { key: "children_rss_bytes", above: 16e9 }, + host_memory_pressure: { key: "host_memory_used_percent", above: 92 }, + main_event_loop_stall: { key: "event_loop_p99_ms", above: 1_000 }, +}; + +export interface ResourceDigestDeps { + getSample: () => Promise; + getIdleSeconds: () => number; + track: (event: string, properties: Record) => void; + isEnabled: () => boolean; + now: () => number; + random: () => number; +} + +function normalizeNumber(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : 0; +} + +function isRendererProcessType(type: string): boolean { + const normalized = type.toLowerCase(); + return normalized === "renderer" || normalized === "tab"; +} + +/** + * Telemetry is off in test and dev so analytics only reflects real usage, and + * honors the global telemetry-enabled flag. `track()` additionally no-ops + * until a user is identified and a PostHog key is configured. + */ +export function isResourceDigestEnabled(): boolean { + const nodeEnv = process.env.NODE_ENV; + if (nodeEnv === "test" || nodeEnv === "development") return false; + return DEFAULT_TELEMETRY_ENABLED; +} + +// The event-loop-delay histogram is cheap (kernel timer wheel sampling) and +// lives for the process; each sample reads p99 and resets it. +const loopDelayHistogram = monitorEventLoopDelay({ resolution: 20 }); +loopDelayHistogram.enable(); + +async function collectDefaultSample(): Promise { + let mainRss = 0; + let rendererRss = 0; + let otherRss = 0; + let processCount = 0; + + // Electron reports `workingSetSize` in KB, hence the `* 1024`. + for (const proc of electron.app.getAppMetrics()) { + const rss = normalizeNumber(proc.memory?.workingSetSize) * 1024; + processCount += 1; + if (proc.type === "Browser") { + mainRss += rss; + } else if ( + typeof proc.type === "string" && + isRendererProcessType(proc.type) + ) { + rendererRss += rss; + } else { + otherRss += rss; + } + } + + // Child-process totals (host-service, pty trees, agents) come from the + // existing UI collector. Only workspace-level numeric totals are read; + // every identifying field stays behind in the snapshot. + let childrenRss = 0; + let childrenCount = 0; + let hostMemoryUsedPercent = 0; + let hostLoadAverage = 0; + try { + const { collectResourceMetrics } = await import( + "main/lib/resource-metrics" + ); + const snapshot = await collectResourceMetrics({ mode: "idle" }); + for (const workspace of snapshot.workspaces) { + childrenRss += normalizeNumber(workspace.memory); + childrenCount += workspace.sessions.length; + } + hostMemoryUsedPercent = normalizeNumber(snapshot.host.memoryUsagePercent); + hostLoadAverage = normalizeNumber(snapshot.host.loadAverage1m); + } catch { + // Child-tree sampling is best-effort; Electron-side numbers still count. + } + + const loopP99Ms = normalizeNumber(loopDelayHistogram.percentile(99) / 1e6); + loopDelayHistogram.reset(); + + const mem = process.memoryUsage(); + + return { + electron_total_rss_bytes: mainRss + rendererRss + otherRss, + electron_main_rss_bytes: mainRss, + electron_renderer_rss_bytes: rendererRss, + electron_other_rss_bytes: otherRss, + electron_process_count: processCount, + window_count: electron.BrowserWindow.getAllWindows().length, + web_contents_count: electron.webContents.getAllWebContents().length, + process_heap_used_bytes: normalizeNumber(mem.heapUsed), + process_external_bytes: normalizeNumber(mem.external), + children_rss_bytes: childrenRss, + children_process_count: childrenCount, + host_memory_used_percent: hostMemoryUsedPercent, + host_load_average_1m: hostLoadAverage, + event_loop_p99_ms: loopP99Ms, + uptime_seconds: Math.round(process.uptime()), + }; +} + +const defaultDeps: ResourceDigestDeps = { + getSample: collectDefaultSample, + getIdleSeconds: () => electron.powerMonitor.getSystemIdleTime(), + track, + isEnabled: isResourceDigestEnabled, + now: Date.now, + random: Math.random, +}; + +interface DayAggregate { + dayKey: string; + samples: Record; + sampleCount: number; + uptimeSecondsMax: number; + unresponsiveCount: number; +} + +function emptyAggregate(dayKey: string): DayAggregate { + const samples = {} as Record; + for (const key of SAMPLE_KEYS) samples[key] = []; + return { + dayKey, + samples, + sampleCount: 0, + uptimeSecondsMax: 0, + unresponsiveCount: 0, + }; +} + +function utcDayKey(epochMs: number): string { + return new Date(epochMs).toISOString().slice(0, 10); +} + +function quantile(sorted: number[], q: number): number { + if (sorted.length === 0) return 0; + const index = Math.min( + sorted.length - 1, + Math.floor(q * (sorted.length - 1) + 0.5), + ); + return sorted[index] ?? 0; +} + +let running = false; +let sampleTimer: ReturnType | null = null; +let aggregate: DayAggregate | null = null; +let activeDeps: ResourceDigestDeps = defaultDeps; +const lastAlertAt = new Map(); + +function buildDigestPayload( + agg: DayAggregate, + flushedOnQuit: boolean, +): Record { + const payload: Record = {}; + for (const key of SAMPLE_KEYS) { + const sorted = [...agg.samples[key]].sort((a, b) => a - b); + payload[`${key}_p50`] = quantile(sorted, 0.5); + payload[`${key}_p90`] = quantile(sorted, 0.9); + payload[`${key}_max`] = sorted[sorted.length - 1] ?? 0; + } + payload.sample_count = agg.sampleCount; + payload.active_minutes = Math.round( + (agg.sampleCount * SAMPLE_INTERVAL_MS) / 60_000, + ); + payload.uptime_seconds_max = agg.uptimeSecondsMax; + payload.unresponsive_count = agg.unresponsiveCount; + payload.flushed_on_quit = flushedOnQuit ? 1 : 0; + + // Belt-and-braces: every value must be a finite number or the key is + // dropped, so non-aggregate data can never ride along. + for (const [key, value] of Object.entries(payload)) { + if (typeof value !== "number" || !Number.isFinite(value)) { + delete payload[key]; + } + } + return payload; +} + +function flushAggregate( + deps: ResourceDigestDeps, + flushedOnQuit: boolean, +): void { + if (!aggregate || aggregate.sampleCount === 0) return; + if (flushedOnQuit && aggregate.sampleCount < MIN_SAMPLES_TO_FLUSH_ON_QUIT) { + aggregate = null; + return; + } + deps.track( + RESOURCE_DIGEST_EVENT, + buildDigestPayload(aggregate, flushedOnQuit), + ); + aggregate = null; +} + +function maybeEmitAlert( + deps: ResourceDigestDeps, + kind: AlertKind, + sample: ResourceSample | null, +): void { + const nowMs = deps.now(); + const last = lastAlertAt.get(kind); + if (last !== undefined && nowMs - last < ALERT_COOLDOWN_MS) return; + lastAlertAt.set(kind, nowMs); + + const payload: Record = { alert_kind: kind }; + if (sample) { + for (const key of SAMPLE_KEYS) { + const value = sample[key]; + if (typeof value === "number" && Number.isFinite(value)) { + payload[key] = value; + } + } + } + deps.track(RESOURCE_ALERT_EVENT, payload); +} + +function evaluateAlerts( + deps: ResourceDigestDeps, + sample: ResourceSample, +): void { + for (const [kind, threshold] of Object.entries(ALERT_THRESHOLDS) as Array< + [AlertKind, { key: SampleKey; above: number }] + >) { + if (sample[threshold.key] > threshold.above) { + maybeEmitAlert(deps, kind, sample); + } + } +} + +async function takeSample(deps: ResourceDigestDeps): Promise { + const nowKey = utcDayKey(deps.now()); + if (aggregate && aggregate.dayKey !== nowKey) { + flushAggregate(deps, false); + } + + // Idle machines still roll the day over (above) but contribute no samples. + if (deps.getIdleSeconds() > IDLE_SKIP_THRESHOLD_SECONDS) return; + + const sample = await deps.getSample(); + if (!aggregate) aggregate = emptyAggregate(nowKey); + if (aggregate.sampleCount >= MAX_SAMPLES_PER_DAY) return; + + for (const key of SAMPLE_KEYS) { + aggregate.samples[key].push(normalizeNumber(sample[key])); + } + aggregate.sampleCount += 1; + aggregate.uptimeSecondsMax = Math.max( + aggregate.uptimeSecondsMax, + normalizeNumber(sample.uptime_seconds), + ); + + evaluateAlerts(deps, sample); +} + +function scheduleNext(deps: ResourceDigestDeps): void { + const delay = + SAMPLE_INTERVAL_MS + Math.floor(deps.random() * SAMPLE_JITTER_MS); + sampleTimer = setTimeout(() => { + void takeSample(deps) + .catch(() => { + // Telemetry must never crash the app; drop the sample. + }) + .finally(() => { + if (running) scheduleNext(deps); + }); + }, delay); + // Never keep the process alive for a telemetry sample. + sampleTimer.unref?.(); +} + +/** + * Start the periodic sampler. Idempotent: a second call while already running + * is a no-op, so startup + HMR re-entry can't create duplicate timers. + */ +export function startResourceDigest( + overrides?: Partial, +): void { + if (running) return; + const deps = { ...defaultDeps, ...overrides }; + if (!deps.isEnabled()) return; + running = true; + activeDeps = deps; + scheduleNext(deps); +} + +/** + * Stop the sampler and flush the partial day (dropped when it holds fewer + * than MIN_SAMPLES_TO_FLUSH_ON_QUIT samples). Called from before-quit. + */ +export function stopResourceDigest(): void { + const wasRunning = running; + running = false; + if (sampleTimer) { + clearTimeout(sampleTimer); + sampleTimer = null; + } + if (wasRunning) flushAggregate(activeDeps, true); + aggregate = null; + lastAlertAt.clear(); +} + +/** + * Count a main-window unresponsive episode into the daily digest and emit a + * (cooldown-limited) alert. Wired from the window's `unresponsive` handler; + * safe to call when the sampler is disabled or not running. + */ +export function recordWindowUnresponsive(): void { + if (!running) return; + if (!aggregate) aggregate = emptyAggregate(utcDayKey(activeDeps.now())); + aggregate.unresponsiveCount += 1; + maybeEmitAlert(activeDeps, "window_unresponsive", null); +} diff --git a/apps/desktop/src/main/windows/main.ts b/apps/desktop/src/main/windows/main.ts index 96ad2afd342..8e14d0fbc8a 100644 --- a/apps/desktop/src/main/windows/main.ts +++ b/apps/desktop/src/main/windows/main.ts @@ -9,6 +9,7 @@ import { createWindow } from "lib/electron-app/factories/windows/create"; import { createAppRouter } from "lib/trpc/routers"; import { localDb } from "main/lib/local-db"; import { isExpectedRendererExit } from "main/lib/renderer-exit"; +import { recordWindowUnresponsive } from "main/lib/resource-digest"; import { NOTIFICATION_EVENTS, PLATFORM } from "shared/constants"; import { env, @@ -167,18 +168,27 @@ export async function MainWindow() { } }, ); + } - window.on("unresponsive", () => { - log.warn("[main-window] Renderer became unresponsive", { - url: window.webContents.getURL(), - }); + // A hung renderer produces no crash and no exception, so without this + // production hangs are invisible to both Sentry and the resource digest. + let unresponsiveSince: number | null = null; + window.on("unresponsive", () => { + unresponsiveSince = Date.now(); + log.warn("[main-window] Renderer became unresponsive", { + url: window.webContents.getURL(), }); - window.on("responsive", () => { - log.info("[main-window] Renderer became responsive", { - url: window.webContents.getURL(), - }); + recordWindowUnresponsive(); + Sentry.captureMessage("renderer unresponsive", { level: "error" }); + }); + window.on("responsive", () => { + const durationMs = unresponsiveSince ? Date.now() - unresponsiveSince : 0; + unresponsiveSince = null; + log.info("[main-window] Renderer became responsive", { + url: window.webContents.getURL(), + durationMs, }); - } + }); if (ipcHandler) { ipcHandler.attachWindow(window);