fix(discord-bot): page long-running turns on 15m×2ⁿ milestones

apps/discord-bot/src/features/Alerts.test.ts

@@ -17,14 +17,19 @@ import {
   classifySessionLastError,
   fatalAlertDelivery,
   formatAlertCause,
+  formatLongRunningTurnMilestone,
   isExpectedSessionLastError,
   listSessionErrors,
+  longRunningTurnMilestoneMs,
+  nextLongRunningTurnMilestoneMs,
   selectSessionErrorsForAlert,
   sessionErrorAgeMs,
   sessionErrorAlertDelivery,
   sessionErrorAlertKey,
   SESSION_ERROR_MAX_AGE_MS,
+  shouldAlertLongRunningTurn,
   trackSustainedHotProcesses,
+  TURN_RUNNING_MIN_MS,
   type ProcInfo,
   type ProcSustainState,
 } from "./Alerts.ts";
@@ -286,6 +291,67 @@ describe("session last_error alert classification", () => {
   });
 });
 
+describe("long-running turn alert milestones (0.25h × 2ⁿ)", () => {
+  const min = (n: number) => n * 60_000;
+
+  it("stays quiet below the first 15m threshold", () => {
+    expect(longRunningTurnMilestoneMs(min(14))).toBeNull();
+    expect(shouldAlertLongRunningTurn(min(14), undefined).alert).toBe(false);
+  });
+
+  it("pages at 15m, 30m, 1h, 2h, 4h and not between rungs", () => {
+    // First page at 15m.
+    expect(longRunningTurnMilestoneMs(min(15))).toBe(min(15));
+    expect(shouldAlertLongRunningTurn(min(15), undefined)).toEqual({
+      alert: true,
+      milestoneMs: min(15),
+    });
+    // Still on the 15m rung until 30m — no re-page.
+    expect(longRunningTurnMilestoneMs(min(29))).toBe(min(15));
+    expect(shouldAlertLongRunningTurn(min(29), min(15)).alert).toBe(false);
+    // 30m / 1h / 2h / 4h rungs.
+    expect(longRunningTurnMilestoneMs(min(30))).toBe(min(30));
+    expect(longRunningTurnMilestoneMs(min(75))).toBe(min(60)); // the 75m spam case
+    expect(longRunningTurnMilestoneMs(min(125))).toBe(min(120));
+    expect(longRunningTurnMilestoneMs(min(240))).toBe(min(240));
+
+    let last: number | undefined;
+    const ages = [15, 20, 25, 30, 45, 59, 60, 75, 90, 119, 120, 180, 240];
+    const paged: number[] = [];
+    for (const ageMin of ages) {
+      const decision = shouldAlertLongRunningTurn(min(ageMin), last);
+      if (decision.alert) {
+        paged.push(ageMin);
+        last = decision.milestoneMs;
+      }
+    }
+    // Only when a new doubling rung is crossed — not every 10 minutes.
+    expect(paged).toEqual([15, 30, 60, 120, 240]);
+  });
+
+  it("re-pages at most once for the current rung after a restart", () => {
+    // Bot restart loses last milestone; age is already 75m → one page for 1h rung.
+    const decision = shouldAlertLongRunningTurn(min(75), undefined);
+    expect(decision).toEqual({ alert: true, milestoneMs: min(60) });
+    expect(shouldAlertLongRunningTurn(min(75), min(60)).alert).toBe(false);
+    // Next page only when age reaches 2h.
+    expect(shouldAlertLongRunningTurn(min(119), min(60)).alert).toBe(false);
+    expect(shouldAlertLongRunningTurn(min(120), min(60))).toEqual({
+      alert: true,
+      milestoneMs: min(120),
+    });
+  });
+
+  it("formats milestone labels and next-rung helper", () => {
+    expect(formatLongRunningTurnMilestone(min(15))).toBe("15m");
+    expect(formatLongRunningTurnMilestone(min(30))).toBe("30m");
+    expect(formatLongRunningTurnMilestone(min(60))).toBe("1h");
+    expect(formatLongRunningTurnMilestone(min(120))).toBe("2h");
+    expect(nextLongRunningTurnMilestoneMs(TURN_RUNNING_MIN_MS)).toBe(min(30));
+    expect(nextLongRunningTurnMilestoneMs(min(60))).toBe(min(120));
+  });
+});
+
 describe("trackSustainedHotProcesses", () => {
   it("does not alert on a long-lived but idle process", () => {
     // The reported bug: a process with lots of cumulative CPU time that now barely

apps/discord-bot/src/features/Alerts.ts

@@ -4,7 +4,8 @@
  *
  * - Host: load, CPU%, memory, disk free
  * - Runaways: legacy stdio Sentry MCP proliferation / high RSS → alert only (never kill)
- * - T3: long-running turns; **real** session errors only (not orphan-restart recover text)
+ * - T3: long-running turns (page at 0.25h × 2ⁿ age milestones); **real** session errors only
+ *   (not orphan-restart recover text)
  * - App: postFatalAlert() / postBridgeAlert() for hard + bridge failures
  */
 import * as NodeChildProcess from "node:child_process";
@@ -66,7 +67,12 @@ const SENTRY_COUNT_ALERT = 2;
 const DEFAULT_PROCESS_CPU_PERCENT = 50; // percent of a single core, averaged over the tick gap
 const DEFAULT_PROCESS_RSS_ALERT_MB = 768;
 const DEFAULT_PROCESS_SUSTAINED_FOR_MS = 4 * POLL_MS;
-const TURN_RUNNING_MIN_MS = 15 * 60 * 1000;
+/**
+ * First long-turn page threshold and base of the doubling milestone ladder.
+ * Pages at 0.25h, 0.5h, 1h, 2h, 4h, … (15m × 2ⁿ) while the turn stays `running`,
+ * instead of re-paging every fixed cooldown (was 10m — spammy for multi-hour turns).
+ */
+export const TURN_RUNNING_MIN_MS = 15 * 60 * 1000;
 
 /** Paths to check for free space (guest rootfs is tiny; data volume is the real store). */
 const DISK_PATHS = ["/", "/var/lib/t3"] as const;
@@ -570,6 +576,60 @@ ${scriptBody}
   }
 }
 
+/**
+ * Highest 0.25h × 2ⁿ milestone the turn age has reached, or `null` if still
+ * below the first page threshold.
+ *
+ * Examples (base 15m): 14m → null; 15–29m → 15m; 30–59m → 30m; 60–119m → 60m;
+ * 125m → 120m.
+ */
+export function longRunningTurnMilestoneMs(
+  ageMs: number,
+  baseMs: number = TURN_RUNNING_MIN_MS,
+): number | null {
+  if (!(ageMs >= baseMs) || !(baseMs > 0)) return null;
+  const exp = Math.floor(Math.log2(ageMs / baseMs));
+  if (!Number.isFinite(exp) || exp < 0) return null;
+  return baseMs * 2 ** exp;
+}
+
+/** Next doubling milestone after the one we just paged (cap not applied). */
+export function nextLongRunningTurnMilestoneMs(milestoneMs: number): number {
+  return milestoneMs * 2;
+}
+
+/** Human label for a milestone duration: `15m`, `30m`, `1h`, `2h`, … */
+export function formatLongRunningTurnMilestone(milestoneMs: number): string {
+  const minutes = milestoneMs / 60_000;
+  if (minutes < 60) return `${Math.round(minutes)}m`;
+  const hours = minutes / 60;
+  if (Number.isInteger(hours)) return `${hours}h`;
+  // Keep one decimal for non-integer hours (shouldn't happen on pure 2ⁿ ladder).
+  return `${hours}h`;
+}
+
+/**
+ * Whether to page for this long-running turn given the last milestone already
+ * posted for its `turnId`. Milestone is derived from turn age (not wall-clock
+ * since last post), so bot restarts re-page at most once for the current rung.
+ */
+export function shouldAlertLongRunningTurn(
+  ageMs: number,
+  lastAlertedMilestoneMs: number | undefined,
+  baseMs: number = TURN_RUNNING_MIN_MS,
+):
+  | { readonly alert: true; readonly milestoneMs: number }
+  | { readonly alert: false; readonly milestoneMs: number | null } {
+  const milestoneMs = longRunningTurnMilestoneMs(ageMs, baseMs);
+  if (milestoneMs === null) {
+    return { alert: false, milestoneMs: null };
+  }
+  if (lastAlertedMilestoneMs !== undefined && milestoneMs <= lastAlertedMilestoneMs) {
+    return { alert: false, milestoneMs };
+  }
+  return { alert: true, milestoneMs };
+}
+
 function listLongRunningTurns(
   dbPath: string,
   minAgeMs: number,
@@ -835,6 +895,8 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) =>
     const discordConfig = yield* DiscordConfig.DiscordConfig;
     const alertProcessRules = loadAlertProcessRulesFromFileSync(botConfig.alertProcessRulesPath);
     const lastSent = new Map<string, number>();
+    /** Highest long-turn milestone already posted per `turnId` (0.25h × 2ⁿ ladder). */
+    const lastTurnMilestones = new Map<string, number>();
 
     const postAlert: Poster = (key, content, cooldownMs = COOLDOWN_MS, files = []) =>
       Effect.gen(function* () {
@@ -1020,16 +1082,30 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) =>
           );
         }
 
-        // --- long T3 turns ---
+        // --- long T3 turns (page at 0.25h × 2ⁿ age milestones; drop finished turns) ---
+        const longTurnIds = new Set(snap.longTurns.map((t) => t.turnId));
+        for (const turnId of lastTurnMilestones.keys()) {
+          if (!longTurnIds.has(turnId)) lastTurnMilestones.delete(turnId);
+        }
         for (const turn of snap.longTurns) {
+          const ageMs = turn.ageMin * 60_000;
+          const decision = shouldAlertLongRunningTurn(ageMs, lastTurnMilestones.get(turn.turnId));
+          if (!decision.alert) continue;
+          lastTurnMilestones.set(turn.turnId, decision.milestoneMs);
+          const milestoneLabel = formatLongRunningTurnMilestone(decision.milestoneMs);
+          const nextLabel = formatLongRunningTurnMilestone(
+            nextLongRunningTurnMilestoneMs(decision.milestoneMs),
+          );
+          // Cooldown 0: cadence is the age-milestone ladder, not wall-clock spacing.
           yield* postAlert(
             `turn:${turn.turnId}`,
             [
               "**Long-running T3 turn**",
               `thread=\`${turn.threadId}\``,
               `turn=\`${turn.turnId}\``,
-              `age≈${turn.ageMin} min (alert after ${TURN_RUNNING_MIN_MS / 60_000} min)`,
+              `age≈${turn.ageMin} min · milestone ${milestoneLabel} (next ~${nextLabel}; ladder 15m×2ⁿ)`,
             ].join("\n"),
+            0,
           );
         }