From ea30cf447cfd50939ec32ebae6acdc27cabab89a Mon Sep 17 00:00:00 2001 From: T3 Code PR Stack <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:33:06 +0200 Subject: [PATCH] =?UTF-8?q?fix(discord-bot):=20page=20long-running=20turns?= =?UTF-8?q?=20on=2015m=C3=972=E2=81=BF=20milestones?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stop re-alerting every 10 minutes for the same stuck turn. Alert at 0.25h, 0.5h, 1h, 2h, 4h, … while projection_turns stays running, which keeps multi-hour agent work (and sticky rows) from flooding #omegent-alerts. --- apps/discord-bot/src/features/Alerts.test.ts | 66 +++++++++++++++ apps/discord-bot/src/features/Alerts.ts | 84 +++++++++++++++++++- 2 files changed, 146 insertions(+), 4 deletions(-) diff --git a/apps/discord-bot/src/features/Alerts.test.ts b/apps/discord-bot/src/features/Alerts.test.ts index 3d9a2fbda64e..1378996eae2e 100644 --- a/apps/discord-bot/src/features/Alerts.test.ts +++ b/apps/discord-bot/src/features/Alerts.test.ts @@ -17,14 +17,19 @@ import { classifySessionLastError, fatalAlertDelivery, formatAlertCause, + formatLongRunningTurnMilestone, isExpectedSessionLastError, listSessionErrors, + longRunningTurnMilestoneMs, + nextLongRunningTurnMilestoneMs, selectSessionErrorsForAlert, sessionErrorAgeMs, sessionErrorAlertDelivery, sessionErrorAlertKey, SESSION_ERROR_MAX_AGE_MS, + shouldAlertLongRunningTurn, trackSustainedHotProcesses, + TURN_RUNNING_MIN_MS, type ProcInfo, type ProcSustainState, } from "./Alerts.ts"; @@ -286,6 +291,67 @@ describe("session last_error alert classification", () => { }); }); +describe("long-running turn alert milestones (0.25h × 2ⁿ)", () => { + const min = (n: number) => n * 60_000; + + it("stays quiet below the first 15m threshold", () => { + expect(longRunningTurnMilestoneMs(min(14))).toBeNull(); + expect(shouldAlertLongRunningTurn(min(14), undefined).alert).toBe(false); + }); + + it("pages at 15m, 30m, 1h, 2h, 4h and not between rungs", () => { + // First page at 15m. + expect(longRunningTurnMilestoneMs(min(15))).toBe(min(15)); + expect(shouldAlertLongRunningTurn(min(15), undefined)).toEqual({ + alert: true, + milestoneMs: min(15), + }); + // Still on the 15m rung until 30m — no re-page. + expect(longRunningTurnMilestoneMs(min(29))).toBe(min(15)); + expect(shouldAlertLongRunningTurn(min(29), min(15)).alert).toBe(false); + // 30m / 1h / 2h / 4h rungs. + expect(longRunningTurnMilestoneMs(min(30))).toBe(min(30)); + expect(longRunningTurnMilestoneMs(min(75))).toBe(min(60)); // the 75m spam case + expect(longRunningTurnMilestoneMs(min(125))).toBe(min(120)); + expect(longRunningTurnMilestoneMs(min(240))).toBe(min(240)); + + let last: number | undefined; + const ages = [15, 20, 25, 30, 45, 59, 60, 75, 90, 119, 120, 180, 240]; + const paged: number[] = []; + for (const ageMin of ages) { + const decision = shouldAlertLongRunningTurn(min(ageMin), last); + if (decision.alert) { + paged.push(ageMin); + last = decision.milestoneMs; + } + } + // Only when a new doubling rung is crossed — not every 10 minutes. + expect(paged).toEqual([15, 30, 60, 120, 240]); + }); + + it("re-pages at most once for the current rung after a restart", () => { + // Bot restart loses last milestone; age is already 75m → one page for 1h rung. + const decision = shouldAlertLongRunningTurn(min(75), undefined); + expect(decision).toEqual({ alert: true, milestoneMs: min(60) }); + expect(shouldAlertLongRunningTurn(min(75), min(60)).alert).toBe(false); + // Next page only when age reaches 2h. + expect(shouldAlertLongRunningTurn(min(119), min(60)).alert).toBe(false); + expect(shouldAlertLongRunningTurn(min(120), min(60))).toEqual({ + alert: true, + milestoneMs: min(120), + }); + }); + + it("formats milestone labels and next-rung helper", () => { + expect(formatLongRunningTurnMilestone(min(15))).toBe("15m"); + expect(formatLongRunningTurnMilestone(min(30))).toBe("30m"); + expect(formatLongRunningTurnMilestone(min(60))).toBe("1h"); + expect(formatLongRunningTurnMilestone(min(120))).toBe("2h"); + expect(nextLongRunningTurnMilestoneMs(TURN_RUNNING_MIN_MS)).toBe(min(30)); + expect(nextLongRunningTurnMilestoneMs(min(60))).toBe(min(120)); + }); +}); + describe("trackSustainedHotProcesses", () => { it("does not alert on a long-lived but idle process", () => { // The reported bug: a process with lots of cumulative CPU time that now barely diff --git a/apps/discord-bot/src/features/Alerts.ts b/apps/discord-bot/src/features/Alerts.ts index 86edc6b6030e..01ff23a021c2 100644 --- a/apps/discord-bot/src/features/Alerts.ts +++ b/apps/discord-bot/src/features/Alerts.ts @@ -4,7 +4,8 @@ * * - Host: load, CPU%, memory, disk free * - Runaways: legacy stdio Sentry MCP proliferation / high RSS → alert only (never kill) - * - T3: long-running turns; **real** session errors only (not orphan-restart recover text) + * - T3: long-running turns (page at 0.25h × 2ⁿ age milestones); **real** session errors only + * (not orphan-restart recover text) * - App: postFatalAlert() / postBridgeAlert() for hard + bridge failures */ import * as NodeChildProcess from "node:child_process"; @@ -66,7 +67,12 @@ const SENTRY_COUNT_ALERT = 2; const DEFAULT_PROCESS_CPU_PERCENT = 50; // percent of a single core, averaged over the tick gap const DEFAULT_PROCESS_RSS_ALERT_MB = 768; const DEFAULT_PROCESS_SUSTAINED_FOR_MS = 4 * POLL_MS; -const TURN_RUNNING_MIN_MS = 15 * 60 * 1000; +/** + * First long-turn page threshold and base of the doubling milestone ladder. + * Pages at 0.25h, 0.5h, 1h, 2h, 4h, … (15m × 2ⁿ) while the turn stays `running`, + * instead of re-paging every fixed cooldown (was 10m — spammy for multi-hour turns). + */ +export const TURN_RUNNING_MIN_MS = 15 * 60 * 1000; /** Paths to check for free space (guest rootfs is tiny; data volume is the real store). */ const DISK_PATHS = ["/", "/var/lib/t3"] as const; @@ -570,6 +576,60 @@ ${scriptBody} } } +/** + * Highest 0.25h × 2ⁿ milestone the turn age has reached, or `null` if still + * below the first page threshold. + * + * Examples (base 15m): 14m → null; 15–29m → 15m; 30–59m → 30m; 60–119m → 60m; + * 125m → 120m. + */ +export function longRunningTurnMilestoneMs( + ageMs: number, + baseMs: number = TURN_RUNNING_MIN_MS, +): number | null { + if (!(ageMs >= baseMs) || !(baseMs > 0)) return null; + const exp = Math.floor(Math.log2(ageMs / baseMs)); + if (!Number.isFinite(exp) || exp < 0) return null; + return baseMs * 2 ** exp; +} + +/** Next doubling milestone after the one we just paged (cap not applied). */ +export function nextLongRunningTurnMilestoneMs(milestoneMs: number): number { + return milestoneMs * 2; +} + +/** Human label for a milestone duration: `15m`, `30m`, `1h`, `2h`, … */ +export function formatLongRunningTurnMilestone(milestoneMs: number): string { + const minutes = milestoneMs / 60_000; + if (minutes < 60) return `${Math.round(minutes)}m`; + const hours = minutes / 60; + if (Number.isInteger(hours)) return `${hours}h`; + // Keep one decimal for non-integer hours (shouldn't happen on pure 2ⁿ ladder). + return `${hours}h`; +} + +/** + * Whether to page for this long-running turn given the last milestone already + * posted for its `turnId`. Milestone is derived from turn age (not wall-clock + * since last post), so bot restarts re-page at most once for the current rung. + */ +export function shouldAlertLongRunningTurn( + ageMs: number, + lastAlertedMilestoneMs: number | undefined, + baseMs: number = TURN_RUNNING_MIN_MS, +): + | { readonly alert: true; readonly milestoneMs: number } + | { readonly alert: false; readonly milestoneMs: number | null } { + const milestoneMs = longRunningTurnMilestoneMs(ageMs, baseMs); + if (milestoneMs === null) { + return { alert: false, milestoneMs: null }; + } + if (lastAlertedMilestoneMs !== undefined && milestoneMs <= lastAlertedMilestoneMs) { + return { alert: false, milestoneMs }; + } + return { alert: true, milestoneMs }; +} + function listLongRunningTurns( dbPath: string, minAgeMs: number, @@ -835,6 +895,8 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) => const discordConfig = yield* DiscordConfig.DiscordConfig; const alertProcessRules = loadAlertProcessRulesFromFileSync(botConfig.alertProcessRulesPath); const lastSent = new Map(); + /** Highest long-turn milestone already posted per `turnId` (0.25h × 2ⁿ ladder). */ + const lastTurnMilestones = new Map(); const postAlert: Poster = (key, content, cooldownMs = COOLDOWN_MS, files = []) => Effect.gen(function* () { @@ -1020,16 +1082,30 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) => ); } - // --- long T3 turns --- + // --- long T3 turns (page at 0.25h × 2ⁿ age milestones; drop finished turns) --- + const longTurnIds = new Set(snap.longTurns.map((t) => t.turnId)); + for (const turnId of lastTurnMilestones.keys()) { + if (!longTurnIds.has(turnId)) lastTurnMilestones.delete(turnId); + } for (const turn of snap.longTurns) { + const ageMs = turn.ageMin * 60_000; + const decision = shouldAlertLongRunningTurn(ageMs, lastTurnMilestones.get(turn.turnId)); + if (!decision.alert) continue; + lastTurnMilestones.set(turn.turnId, decision.milestoneMs); + const milestoneLabel = formatLongRunningTurnMilestone(decision.milestoneMs); + const nextLabel = formatLongRunningTurnMilestone( + nextLongRunningTurnMilestoneMs(decision.milestoneMs), + ); + // Cooldown 0: cadence is the age-milestone ladder, not wall-clock spacing. yield* postAlert( `turn:${turn.turnId}`, [ "**Long-running T3 turn**", `thread=\`${turn.threadId}\``, `turn=\`${turn.turnId}\``, - `age≈${turn.ageMin} min (alert after ${TURN_RUNNING_MIN_MS / 60_000} min)`, + `age≈${turn.ageMin} min · milestone ${milestoneLabel} (next ~${nextLabel}; ladder 15m×2ⁿ)`, ].join("\n"), + 0, ); }