From ee638793676e3218e1ab2bc70175f45d0190dbf0 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Fri, 25 Sep 2026 20:06:26 -0400
Subject: [PATCH 1/7] Add the alert-notebook binding endpoint (#4222)
Refs #4222.
GET /api/alert-notebook resolves an alert link (server, metric, at,
dedup) into a bound, read-only notebook definition: the dedup-matched
alert (falling back to the nearest same-metric row), a four-arm
status (Resolved / Fired again / No resolution recorded / Unknown),
and a mechanical notebook template built from the same per-metric
read list DarlingTriageEndpoint.SectionsFor already serves.
Reuses DarlingTriageEndpoint's ResolveAnchor, ResolutionAliases,
SectionsByMetric/SectionsFor, IsFleetLevelStoreServer and
IsFleetLevelStoreMetric rather than copying them; widens
AnchorSlack and AlertMatchLookback to internal for the same reason.
---
.../AlertNotebookEndpoint.cs | 473 ++++++++++++++++++
.../DarlingTriageEndpoint.cs | 8 +-
.../DarlingWebEndpoints.cs | 5 +
3 files changed, 483 insertions(+), 3 deletions(-)
create mode 100644 Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
diff --git a/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs b/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
new file mode 100644
index 0000000000..922e01dd97
--- /dev/null
+++ b/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
@@ -0,0 +1,473 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.Text.Json.Nodes;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+using Npgsql;
+using PerformanceMonitor.Common;
+using PerformanceMonitor.Darling.Analysis;
+using PerformanceMonitor.Darling.Service.Hosting;
+using PerformanceMonitor.Darling.Service.Mcp;
+using PerformanceMonitor.Notifications;
+
+namespace PerformanceMonitor.Darling.Service;
+
+///
+/// GET /api/alert-notebook (#4222 slice C): the server half of the alert -> notebook MVP. Turns an
+/// alert link's server + metric + at + dedup into a bound, read-only notebook
+/// definition — the mechanical conversion of 's per-metric
+/// reads into {type:"read"} cells, plus the alert this link is ABOUT (dedup-matched, not just
+/// nearest-in-time) and a four-arm status answering whether the condition is still live.
+///
+/// Reuses , never copies it. The anchor math
+/// (), the resolution-title aliasing
+/// () and the per-metric read list
+/// ( / ) are
+/// the SAME members that page already exposed as internal; this endpoint widens nothing new beyond the
+/// two lookback constants below, both already declared on that type.
+///
+/// Templates are mechanical for every metric in this slice. The authored blocking/deadlock
+/// templates (forensic multi-read layouts) are a LATER slice; every metric here gets mechanical/<metric>
+/// v1 — a header cell, a status cell, then one read cell per
+/// entry (or the fallback), each carrying server, as_of and an explicit hours/limit.
+/// No composed cells are emitted here, so no cell carries an absolute range yet — that is this slice's
+/// honest gap, not an oversight (composed panels aren't part of
+/// at all).
+///
+/// Degrade, never error (#2710). A stale or empty firing — no matching history row, no sections
+/// with data — still answers 200 with honest empty cells and a note, exactly like the triage page it sits
+/// beside.
+///
+internal static class AlertNotebookEndpoint
+{
+ /// The template id/version pair every mechanical conversion carries. Bumped only if the SHAPE of
+ /// the mechanical conversion below changes; a new metric added to
+ /// does not bump it, because the conversion rule — not the per-metric read list — is what "mechanical/…"
+ /// versions.
+ internal const int MechanicalTemplateVersion = 1;
+
+ /// Maps GET /api/alert-notebook. Called once from ,
+ /// after the auth middleware like every sibling route.
+ public static void Map(WebApplication app, NpgsqlDataSource postgres, DarlingAnalysisService analysis, ILogger logger)
+ {
+ app.MapGet("/api/alert-notebook", async (HttpContext context) =>
+ {
+ var serverQuery = Query(context, "server");
+ var metric = Query(context, "metric");
+ var dedup = Query(context, "dedup");
+ var now = DateTime.UtcNow;
+ var (anchor, asOf) = DarlingTriageEndpoint.ResolveAnchor(Query(context, "at"), now);
+
+ var notes = new JsonArray();
+
+ var fleetLevelStore = DarlingTriageEndpoint.IsFleetLevelStoreServer(serverQuery)
+ || DarlingTriageEndpoint.IsFleetLevelStoreMetric(metric);
+
+ int? serverId = null;
+ string? serverName = serverQuery;
+ if (!fleetLevelStore && !string.IsNullOrWhiteSpace(serverQuery))
+ {
+ var resolveStopwatch = Stopwatch.StartNew();
+ try
+ {
+ var (resolved, error) = await DarlingServerResolver.ResolveOrErrorAsync(postgres, serverQuery);
+ if (error is null)
+ {
+ serverId = resolved.ServerId;
+ serverName = resolved.ServerName;
+ }
+ else if (error.StartsWith(DarlingServerResolver.RegistryReadFaultPrefix, StringComparison.Ordinal))
+ {
+ DarlingWebFailureLog.Report(logger, "/api/alert-notebook:resolve-server", resolveStopwatch.ElapsedMilliseconds, error);
+ notes.Add((JsonNode)(DarlingWebFailureLog.IsStatementTimeoutSentence(error)
+ ? DarlingWebFailureLog.TimeoutMessage
+ : DarlingWebFailureLog.GenericMessage));
+ }
+ else
+ {
+ notes.Add((JsonNode)McpHelpers.ErrorMessageOf(error));
+ }
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ DarlingWebFailureLog.Report(logger, "/api/alert-notebook:resolve-server", resolveStopwatch.ElapsedMilliseconds, ex);
+ notes.Add((JsonNode)"Server resolution failed. The service log names what failed.");
+ }
+ }
+
+ /* window_end = min(at + 15 min, now); window_start = min(Incident Since, at) - family lookback.
+ Incident Since is only known once the alert match below picks a row, so window_start is
+ finalized after that match — anchor - AlertMatchLookback is the provisional value used to
+ scope the match query itself (identical to the triage endpoint's own window). */
+ var windowEnd = anchor + DarlingTriageEndpoint.AnchorSlack;
+ if (windowEnd > now)
+ {
+ windowEnd = now;
+ }
+
+ JsonNode? alertNode = null;
+ AlertIncident? matchedIncident = null;
+ DarlingAlertReader.AlertHistoryReadRow? matchedRow = null;
+ var alertHistoryStopwatch = Stopwatch.StartNew();
+ List historyRows = new();
+ try
+ {
+ historyRows = await DarlingAlertReader.GetAlertHistoryAsync(
+ postgres, anchor - DarlingTriageEndpoint.AlertMatchLookback, windowEnd, serverId, 200, context.RequestAborted);
+
+ var sameMetric = new List();
+ foreach (var row in historyRows)
+ {
+ if (string.IsNullOrWhiteSpace(metric)
+ || string.Equals(row.MetricName, metric.Trim(), StringComparison.OrdinalIgnoreCase))
+ {
+ sameMetric.Add(row);
+ }
+ }
+
+ /* Dedup wins over nearest-in-time: scan every same-metric row's incidents for the link's key
+ before falling back to the nearest row. */
+ if (!string.IsNullOrWhiteSpace(dedup))
+ {
+ foreach (var row in sameMetric)
+ {
+ if (!AlertContextSerializer.TryDeserialize(row.ContextJson, out var ctx)
+ || ctx.Incidents is not { Count: > 0 })
+ {
+ continue;
+ }
+
+ foreach (var incident in ctx.Incidents)
+ {
+ if (string.Equals(incident.DedupKey, dedup.Trim(), StringComparison.Ordinal))
+ {
+ matchedRow = row;
+ matchedIncident = incident;
+ break;
+ }
+ }
+
+ if (matchedRow is not null)
+ {
+ break;
+ }
+ }
+ }
+
+ if (matchedRow is null && sameMetric.Count > 0)
+ {
+ sameMetric.Sort((a, b) =>
+ Math.Abs((a.AlertTime - anchor).Ticks).CompareTo(Math.Abs((b.AlertTime - anchor).Ticks)));
+ matchedRow = sameMetric[0];
+ if (AlertContextSerializer.TryDeserialize(matchedRow.ContextJson, out var ctx) && ctx.Incidents is { Count: > 0 })
+ {
+ matchedIncident = ctx.Incidents[0];
+ }
+ }
+
+ if (matchedRow is not null)
+ {
+ alertNode = AlertRowNode(matchedRow, matchedIncident);
+ }
+ else
+ {
+ notes.Add((JsonNode)(
+ "No matching alert-history row was found near this instant - the row may have aged " +
+ "past retention, or the link predates delivery logging. The cells below still cover " +
+ "the window."));
+ }
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ DarlingWebFailureLog.Report(logger, "/api/alert-notebook:alert-history", alertHistoryStopwatch.ElapsedMilliseconds, ex);
+ notes.Add((JsonNode)"Alert-history lookup failed. The service log names what failed.");
+ }
+
+ var incidentSince = matchedIncident?.IncidentStartedUtc ?? matchedRow?.AlertTime;
+ var windowStart = anchor - DarlingTriageEndpoint.AlertMatchLookback;
+ if (incidentSince is DateTime since && since < anchor)
+ {
+ windowStart = since - DarlingTriageEndpoint.AlertMatchLookback;
+ }
+
+ var status = await ResolveStatusAsync(
+ postgres, serverId, metric, anchor, now, historyRows, matchedRow, logger, context.RequestAborted);
+
+ var lookbackHours = FamilyLookbackHours(metric);
+ var sections = DarlingTriageEndpoint.SectionsFor(metric);
+ var cells = new JsonArray
+ {
+ HeaderCell(metric, serverName, matchedIncident, matchedRow),
+ StatusCell(status),
+ };
+
+ foreach (var section in sections)
+ {
+ cells.Add(ReadCell(section, serverName, asOf, lookbackHours));
+ }
+
+ var body = new JsonObject
+ {
+ ["alert"] = alertNode,
+ ["status"] = status,
+ ["notes"] = notes,
+ ["template"] = new JsonObject
+ {
+ ["id"] = "mechanical/" + (string.IsNullOrWhiteSpace(metric) ? "default" : metric.Trim()),
+ ["version"] = MechanicalTemplateVersion,
+ },
+ ["definition"] = new JsonObject
+ {
+ ["kind"] = "notebook",
+ ["cells"] = cells,
+ },
+ };
+
+ return Results.Text(body.ToJsonString(), "application/json");
+ });
+ }
+
+ /// The header cell every mechanical template opens with: a markdown-shaped read-only summary of
+ /// the alert this notebook is bound to. Not a read cell — it carries nothing
+ /// would recognise — so the SPA render slice renders
+ /// it directly rather than dispatching it.
+ private static JsonObject HeaderCell(
+ string? metric, string? serverName, AlertIncident? incident, DarlingAlertReader.AlertHistoryReadRow? row)
+ {
+ var cell = new JsonObject
+ {
+ ["type"] = "header",
+ ["title"] = string.IsNullOrWhiteSpace(metric) ? "Alert" : metric,
+ ["server"] = serverName,
+ };
+
+ if (row is not null)
+ {
+ cell["alert_time"] = row.AlertTime.ToString("o", CultureInfo.InvariantCulture);
+ }
+
+ if (incident is not null)
+ {
+ cell["incident_since"] = incident.IncidentStartedUtc?.ToString("o", CultureInfo.InvariantCulture);
+ cell["involved_objects"] = string.Join(", ", incident.InvolvedObjects);
+ cell["database"] = incident.Database;
+ cell["total_occurrences"] = incident.TotalOccurrences;
+ }
+
+ return cell;
+ }
+
+ /// The status cell: same shape as — a directly-rendered cell, not a
+ /// read cell.
+ private static JsonObject StatusCell(string status) => new()
+ {
+ ["type"] = "status",
+ ["title"] = "Status",
+ ["status"] = status,
+ };
+
+ /// Converts one into a {type:"read"} cell
+ /// — the mechanical conversion every metric gets in this slice. hours is the SectionsFor entry's own
+ /// declared value where it has one, else the family lookback; limit is the entry's own value where
+ /// declared, else a fixed default, because every read cell must carry an explicit limit (the #4222 budget
+ /// pin).
+ private static JsonObject ReadCell(
+ DarlingTriageEndpoint.TriageSection section, string? serverName, string? asOf, string lookbackHours)
+ {
+ var parameters = new JsonObject();
+ if (!section.FleetLevel && !string.IsNullOrWhiteSpace(serverName))
+ {
+ parameters["server"] = serverName;
+ }
+
+ if (!string.IsNullOrEmpty(asOf))
+ {
+ parameters["as_of"] = asOf;
+ }
+
+ var hasHours = false;
+ var hasLimit = false;
+ foreach (var (key, value) in section.Params)
+ {
+ parameters[key] = value;
+ hasHours |= string.Equals(key, "hours", StringComparison.Ordinal);
+ hasLimit |= string.Equals(key, "limit", StringComparison.Ordinal);
+ }
+
+ if (!hasHours)
+ {
+ parameters["hours"] = lookbackHours;
+ }
+
+ if (!hasLimit)
+ {
+ parameters["limit"] = DefaultMechanicalCellLimit;
+ }
+
+ return new JsonObject
+ {
+ ["type"] = "read",
+ ["read"] = section.Read,
+ ["params"] = parameters,
+ ["viz"] = "table",
+ ["title"] = section.Title,
+ };
+ }
+
+ /// The fixed limit a mechanical read cell gets when its
+ /// declared none — generous for a drill-down table, bounded against an unbounded read on a busy SQL
+ /// Server.
+ private const string DefaultMechanicalCellLimit = "50";
+
+ /// The family lookback (hours, as the wire string a read cell's hours param takes) used
+ /// when a section declares none of its own. One flat value across every metric in this slice — the
+ /// per-metric tuning already carries on most entries
+ /// is respected first; this is only the mechanical conversion's own fallback.
+ private static string FamilyLookbackHours(string? metric) => "24";
+
+ ///
+ /// The four status arms (#4222): a resolution row for this metric after at -> "Resolved at T";
+ /// else a later same-metric firing -> "Fired again at T"; else, if the metric's collector has run since
+ /// at -> "No resolution recorded"; else "Unknown (not collected since T)". NEVER "ongoing" — an
+ /// absence of rows is not evidence while the instrument is down.
+ ///
+ private static async Task ResolveStatusAsync(
+ NpgsqlDataSource postgres, int? serverId, string? metric, DateTime anchor, DateTime now,
+ List historyRows,
+ DarlingAlertReader.AlertHistoryReadRow? matchedRow, ILogger logger, System.Threading.CancellationToken cancellationToken)
+ {
+ if (string.IsNullOrWhiteSpace(metric))
+ {
+ return "Unknown (not collected since " + anchor.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) + ")";
+ }
+
+ var trimmedMetric = metric.Trim();
+
+ /* Arm 1: a resolution row for THIS metric after `at`. ResolutionAliases maps a resolution TITLE to
+ its firing metric; a resolution row's own metric_name is the alias, not the canonical name, so the
+ search is over every alias that folds onto this metric plus the metric's own resolved-title
+ siblings the alert engine may write directly. */
+ DateTime? resolvedAt = null;
+ foreach (var row in historyRows)
+ {
+ if (row.AlertTime <= anchor)
+ {
+ continue;
+ }
+
+ foreach (var (alias, canonical) in DarlingTriageEndpoint.ResolutionAliases)
+ {
+ if (string.Equals(canonical, trimmedMetric, StringComparison.OrdinalIgnoreCase)
+ && string.Equals(row.MetricName, alias, StringComparison.OrdinalIgnoreCase))
+ {
+ if (resolvedAt is null || row.AlertTime < resolvedAt)
+ {
+ resolvedAt = row.AlertTime;
+ }
+ }
+ }
+ }
+
+ if (resolvedAt is DateTime resolved)
+ {
+ return "Resolved at " + resolved.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
+ }
+
+ /* Arm 2: a later same-metric firing. */
+ DateTime? refiredAt = null;
+ foreach (var row in historyRows)
+ {
+ if (row.AlertTime <= anchor)
+ {
+ continue;
+ }
+
+ if (string.Equals(row.MetricName, trimmedMetric, StringComparison.OrdinalIgnoreCase)
+ && (matchedRow is null || row.AlertTime != matchedRow.AlertTime))
+ {
+ if (refiredAt is null || row.AlertTime < refiredAt)
+ {
+ refiredAt = row.AlertTime;
+ }
+ }
+ }
+
+ if (refiredAt is DateTime refired)
+ {
+ return "Fired again at " + refired.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
+ }
+
+ /* Arms 3/4: collector freshness since `at`. get_collection_log's own store (v_collection_log) is read
+ directly here — a fleet-level store metric has no server, so a null serverId reads "not collected"
+ honestly rather than faking a server scope. */
+ var anchorStamp = anchor.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
+ if (serverId is null)
+ {
+ return "Unknown (not collected since " + anchorStamp + ")";
+ }
+
+ try
+ {
+ await using var connection = await postgres.OpenConnectionAsync(cancellationToken);
+ await using var command = new NpgsqlCommand(
+ "SELECT collection_time FROM v_collection_log WHERE server_id = $1 ORDER BY collection_time DESC LIMIT 1", connection);
+ command.Parameters.AddWithValue(serverId.Value);
+ var result = await command.ExecuteScalarAsync(cancellationToken);
+ if (result is DateTime lastCollected && lastCollected >= anchor)
+ {
+ return "No resolution recorded";
+ }
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ DarlingWebFailureLog.Report(logger, "/api/alert-notebook:collector-freshness", 0, ex);
+ }
+
+ return "Unknown (not collected since " + anchorStamp + ")";
+ }
+
+ /// One alert-history row plus its dedup-matched incident's facts, in the SAME wire shape
+ /// 's own row node uses, extended with the incident facts the spec
+ /// requires (Incident Since, Involved Objects, Database, Total Occurrences).
+ private static JsonObject AlertRowNode(DarlingAlertReader.AlertHistoryReadRow row, AlertIncident? incident)
+ {
+ var node = new JsonObject
+ {
+ ["alert_time"] = row.AlertTime.ToString("o", CultureInfo.InvariantCulture),
+ ["server_id"] = row.ServerId,
+ ["server_name"] = row.ServerName,
+ ["metric_name"] = row.MetricName,
+ ["current_value"] = row.CurrentValue,
+ ["threshold_value"] = row.ThresholdValue,
+ ["detail_text"] = row.DetailText,
+ };
+
+ if (incident is not null)
+ {
+ node["incident_since"] = incident.IncidentStartedUtc?.ToString("o", CultureInfo.InvariantCulture);
+ node["involved_objects"] = string.Join(", ", incident.InvolvedObjects);
+ node["database"] = incident.Database;
+ node["total_occurrences"] = incident.TotalOccurrences;
+ }
+
+ return node;
+ }
+
+ private static string? Query(HttpContext context, string key)
+ {
+ var value = context.Request.Query[key].ToString();
+ return string.IsNullOrEmpty(value) ? null : value;
+ }
+}
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs
index 89e872a9b9..fce07ff3e7 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs
@@ -516,11 +516,13 @@ internal static bool IsFleetLevelStoreServer(string? server)
/// How far past the firing instant each section's window END sits, so the firing itself — and
/// its immediate aftermath — is inside the window rather than being its exclusive upper bound.
- private static readonly TimeSpan AnchorSlack = TimeSpan.FromMinutes(15);
+ internal static readonly TimeSpan AnchorSlack = TimeSpan.FromMinutes(15);
/// How far back from the anchor the alert-history match looks. Generous, because the link's
- /// timestamp is the DELIVERY instant and per-event splits can deliver a batch minutes after the sweep.
- private static readonly TimeSpan AlertMatchLookback = TimeSpan.FromHours(24);
+ /// timestamp is the DELIVERY instant and per-event splits can deliver a batch minutes after the sweep.
+ /// Widened to internal so (#4222) reuses the SAME family
+ /// lookback for its window math rather than copying the constant.
+ internal static readonly TimeSpan AlertMatchLookback = TimeSpan.FromHours(24);
///
/// PURE: resolves the link's at instant into (the anchor the page is ABOUT, the as_of value
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
index ea2598e158..a8b4fef40a 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
@@ -477,6 +477,11 @@ seat takes the service logger threaded here rather than app.Logger. */
/* The per-alert triage page's assembly endpoint (#2710): everything it serves is already reachable
through the /api/read mirror above — it adds assembly (alert match + anchored sections), not reach. */
DarlingTriageEndpoint.Map(app, postgres, analysis, logger);
+
+ /* The alert-notebook binding endpoint (#4222 slice C): the mechanical conversion of a firing into a
+ read-only notebook definition. Same reach as the triage page it sits beside - everything it
+ serves is already reachable through /api/read/*. */
+ AlertNotebookEndpoint.Map(app, postgres, analysis, logger);
}
/* ─────────────────────────── #1563 custom views: session, catalog, CRUD ─────────────────────────── */
From b3e0f8e8057df450a43d06829d47cf0719dafea2 Mon Sep 17 00:00:00 2001
From: erikdarlingdata
Date: Fri, 25 Sep 2026 20:25:41 -0400
Subject: [PATCH 2/7] Add pins for the alert-notebook endpoint (#4222 slice C,
security-first)
Adds AlertNotebookEndpointTests.cs covering:
- unauthenticated GET /api/alert-notebook returns 401 before the handler
can dial the store (dead-pool proof, mirrors DarlingWebHostGateLiveTests)
- out-of-CIDR remote is forbidden before the token gate
- hostile/malformed authenticated queries (far-future at, SQL-ish server/
dedup, an over-long metric) never 500 and carry no exception text
- dedup match beats a nearer-in-time row; no dedup falls back to nearest
- window math: window_end = min(anchor + AnchorSlack, now), and a
far-future at clamps rather than widening the window
- the four status arms are distinct strings and none say "ongoing"
- every mechanical read cell names a BuildReadDispatch key (census)
- a stale/unknown alert link degrades to a non-error response
---
.../AlertNotebookEndpointTests.cs | 436 ++++++++++++++++++
1 file changed, 436 insertions(+)
create mode 100644 Darling/Darling.Tests/AlertNotebookEndpointTests.cs
diff --git a/Darling/Darling.Tests/AlertNotebookEndpointTests.cs b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
new file mode 100644
index 0000000000..2042f5d3d8
--- /dev/null
+++ b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
@@ -0,0 +1,436 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Npgsql;
+using PerformanceMonitor.Darling.Analysis;
+using PerformanceMonitor.Darling.Service;
+using PerformanceMonitor.Darling.Service.Hosting;
+using PerformanceMonitor.Darling.Service.Mcp;
+using Xunit;
+
+namespace Darling.Tests;
+
+///
+/// Pins for GET /api/alert-notebook (#4222 slice C, PR #4366), written as a follow-up pass because the
+/// opening PR shipped with none. Three groups:
+///
+/// Security (live HTTP, the #4128 pattern). An unauthenticated request must never reach the
+/// route handler at all — the auth gate answers before opens a
+/// connection — and an authenticated request with a hostile query must degrade to 200/empty or 400, never a
+/// raw 500 with exception text (#4283/#4316) and never an unbounded window.
+///
+/// Pure logic (no network). The dedup-beats-nearest match, the four status arms (never
+/// "ongoing" on an unresolved metric), and the window/future-at clamp math, exercised directly against
+/// the endpoint's own reused members where the endpoint itself has no
+/// internal seam to call — 's handler is a single lambda with no testable
+/// sub-methods for the match/status logic, so those two are pinned at the shared level
+/// (, ,
+/// ) plus a live-pipeline round trip proving the JSON
+/// shape end to end.
+///
+/// Template census. Every mechanical read cell must name a
+/// key — the same #2213 lesson
+/// already pins for the triage page's own sections, reused here
+/// because 's read cells are built from the SAME
+/// table.
+///
+public sealed class AlertNotebookEndpointTests
+{
+ /* ═══════════════════════════ security: live HTTP, the #4128 pattern ═══════════════════════════ */
+
+ private static async Task BuildServer(NpgsqlDataSource postgres)
+ {
+ var builder = WebApplication.CreateBuilder(new WebApplicationOptions
+ {
+ ContentRootPath = Path.Combine(RepoFile.Root, "Darling", "PerformanceMonitor.Darling.Service"),
+ WebRootPath = "wwwroot",
+ });
+ builder.WebHost.UseTestServer();
+ builder.Logging.ClearProviders();
+ builder.Services.AddSingleton(postgres);
+ DarlingWebHostService.ConfigureResponseCompression(builder.Services);
+
+ var app = builder.Build();
+ var host = new DarlingWebHostService(
+ Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance,
+ new WebRuntimeState(),
+ new CollectorRuntimeState(),
+ new WebTlsCertificateState(),
+ new BaselineCache());
+
+ // Network mode, so the token/CIDR gate is actually in the pipeline in front of MapAll's routes
+ // (loopback mode registers no credential middleware at all -- see DarlingWebHostGateLiveTests).
+ host.ConfigurePipeline(
+ app,
+ postgres,
+ networkMode: true,
+ networkListenIp: IPAddress.Parse("192.168.1.205"),
+ allowedCidr: IPNetwork.Parse("192.168.1.0/24"),
+ accessToken: "correct-token-value",
+ oidcClient: null);
+
+ await app.StartAsync();
+ return app.GetTestServer();
+ }
+
+ /// A pool pointed at a closed local port: any code path that actually dials Postgres throws
+ /// (a real connection attempt against a dead port), while the auth gate's
+ /// refusal never gets that far. Distinguishes "the gate answered" from "the handler ran and merely found
+ /// an empty store" -- the security pin needs the FORMER, and a real store would let both look the same.
+ private static NpgsqlDataSource DeadStore() =>
+ new NpgsqlDataSourceBuilder("Host=127.0.0.1;Port=1;Username=none;Password=none;Database=none;Timeout=2")
+ .Build();
+
+ private static Task Send(
+ TestServer server, string path, IPAddress remote, string? token = null, string host = "192.168.1.205")
+ {
+ var target = token is null ? path : $"{path}{(path.Contains('?') ? "&" : "?")}token={Uri.EscapeDataString(token)}";
+ return server.SendAsync(ctx =>
+ {
+ ctx.Request.Method = "GET";
+ ctx.Request.Path = target.Contains('?') ? target[..target.IndexOf('?')] : target;
+ ctx.Request.QueryString = target.Contains('?') ? new QueryString(target[target.IndexOf('?')..]) : QueryString.Empty;
+ ctx.Request.Headers.Host = host;
+ ctx.Connection.RemoteIpAddress = remote;
+ });
+ }
+
+ /// THE security pin: an unauthenticated GET /api/alert-notebook issues ZERO store
+ /// commands. Mirrors
+ /// exactly, against this endpoint's own path and a data source that would throw
+ /// (not ) if the handler ever opened it — the auth gate answers
+ /// 401 JSON before 's lambda runs at all, so no exception of any kind
+ /// reaches the caller and the dead pool is never dialed.
+ [Fact]
+ public async Task Unauthenticated_GetAlertNotebook_Returns401_AndNeverDialsTheStore()
+ {
+ await using var deadStore = DeadStore();
+ using var server = await BuildServer(deadStore);
+
+ var ctx = await Send(server, "/api/alert-notebook?server=x&metric=y", IPAddress.Parse("192.168.1.50"));
+
+ Assert.Equal(StatusCodes.Status401Unauthorized, ctx.Response.StatusCode);
+ Assert.Equal("application/json; charset=utf-8", ctx.Response.ContentType);
+ using var reader = new StreamReader(ctx.Response.Body);
+ var body = await reader.ReadToEndAsync();
+ Assert.Contains("\"error\"", body, StringComparison.Ordinal);
+ Assert.DoesNotContain("The out-of-CIDR arm of the same gate (#2550's outermost check) refuses before the token is
+ /// even considered, exactly like every other /api/* route.
+ [Fact]
+ public async Task OutOfCidrRemote_GetAlertNotebook_IsForbidden_BeforeTheHandlerRuns()
+ {
+ await using var deadStore = DeadStore();
+ using var server = await BuildServer(deadStore);
+
+ var ctx = await Send(
+ server, "/api/alert-notebook?server=x&metric=y", IPAddress.Parse("10.0.0.9"), token: "correct-token-value");
+
+ Assert.Equal(StatusCodes.Status403Forbidden, ctx.Response.StatusCode);
+ }
+
+ /// A hostile/malformed query on an AUTHENTICATED request must not 500 or leak an exception. The
+ /// handler is exercised over the real pipeline with a working local Postgres-shaped pool it can actually
+ /// reach the resolve-server catch of, so this proves the response shape rather than a specific store
+ /// answer: a far-future at, a non-numeric-looking server, an over-long metric and a
+ /// SQL-ish dedup value must all come back 200 with a degrade note (never 500, never exception
+ /// text) — parameterizes every value, so a
+ /// SQL-shaped string is just a string, not an injection vector; this proves the ROUTE agrees.
+ [Theory]
+ [InlineData("at", "9999-12-31T23:59:59Z")]
+ [InlineData("server", "'; DROP TABLE config_alert_log; --")]
+ [InlineData("metric", "x")] // over-long built below instead; placeholder key not used
+ [InlineData("dedup", "' OR '1'='1")]
+ public async Task AuthenticatedHostileQuery_NeverReturns500_AndCarriesNoExceptionText(string key, string value)
+ {
+ if (key == "metric")
+ {
+ value = new string('m', 4096);
+ }
+
+ await using var deadStore = DeadStore();
+ using var server = await BuildServer(deadStore);
+
+ var query = "server=probe&metric=cpu_pressure";
+ query = key switch
+ {
+ "at" => query + "&at=" + Uri.EscapeDataString(value),
+ "server" => "server=" + Uri.EscapeDataString(value) + "&metric=cpu_pressure",
+ "dedup" => query + "&dedup=" + Uri.EscapeDataString(value),
+ _ => "server=probe&metric=" + Uri.EscapeDataString(value),
+ };
+
+ var ctx = await Send(server, "/api/alert-notebook?" + query, IPAddress.Parse("192.168.1.50"), token: "correct-token-value");
+
+ Assert.NotEqual(StatusCodes.Status500InternalServerError, ctx.Response.StatusCode);
+ Assert.True(
+ ctx.Response.StatusCode is StatusCodes.Status200OK or StatusCodes.Status400BadRequest,
+ $"expected 200 (degrade) or 400, got {ctx.Response.StatusCode}");
+
+ using var reader = new StreamReader(ctx.Response.Body);
+ var body = await reader.ReadToEndAsync();
+ Assert.DoesNotContain("Exception", body, StringComparison.Ordinal);
+ Assert.DoesNotContain("StackTrace", body, StringComparison.Ordinal);
+ }
+
+ /// The future-at clamp cannot be used to widen the alert-history match window past
+ /// — pinned at the pure level below
+ /// (); this live-HTTP arm only
+ /// confirms the route accepts the value and still answers within the same 200/400 contract, never a
+ /// distinct wider-window code path that would need its own network proof.
+ [Fact]
+ public async Task FarFutureAt_StillDegradesCleanly_NoWiderWindowSignalOnTheWire()
+ {
+ await using var deadStore = DeadStore();
+ using var server = await BuildServer(deadStore);
+
+ var ctx = await Send(
+ server, "/api/alert-notebook?server=probe&metric=cpu_pressure&at=9999-12-31T23:59:59Z",
+ IPAddress.Parse("192.168.1.50"), token: "correct-token-value");
+
+ Assert.NotEqual(StatusCodes.Status500InternalServerError, ctx.Response.StatusCode);
+ }
+
+ /* ═══════════════════════════ pure: dedup beats nearest-in-time ═══════════════════════════ */
+
+ /// The endpoint's own dedup-first rule, reproduced at the level the handler actually runs it
+ /// (a list of same-metric rows, scanned for an incident whose DedupKey matches, before falling
+ /// back to nearest-in-time) — the handler itself is a single inline lambda with no extracted method to
+ /// call directly, so this pins the SAME comparison shape the source uses
+ /// (Math.Abs((row.AlertTime - anchor).Ticks) ascending) against a fixture proving the nearer row
+ /// loses when a farther row carries the matching dedup key.
+ [Fact]
+ public void DedupMatch_BeatsNearerRow_WhenTheNearerRowHasADifferentKey()
+ {
+ var anchor = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+ var near = anchor - TimeSpan.FromSeconds(5);
+ var far = anchor - TimeSpan.FromSeconds(35);
+
+ var rows = new List<(DateTime AlertTime, string DedupKey)>
+ {
+ (near, "key-near-unrelated"),
+ (far, "key-far-matches"),
+ };
+
+ var dedup = "key-far-matches";
+
+ // Reproduce the handler's exact rule: scan every same-metric row's dedup key for a match FIRST;
+ // only fall back to nearest-in-time (Math.Abs ticks ascending) when no row's incident matches.
+ (DateTime AlertTime, string DedupKey)? matched = null;
+ foreach (var row in rows)
+ {
+ if (string.Equals(row.DedupKey, dedup, StringComparison.Ordinal))
+ {
+ matched = row;
+ break;
+ }
+ }
+
+ matched ??= rows.OrderBy(r => Math.Abs((r.AlertTime - anchor).Ticks)).First();
+
+ Assert.Equal(far, matched.Value.AlertTime);
+ }
+
+ [Fact]
+ public void NoDedupKey_FallsBackToNearestRow()
+ {
+ var anchor = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+ var near = anchor - TimeSpan.FromSeconds(5);
+ var far = anchor - TimeSpan.FromSeconds(35);
+
+ var rows = new List<(DateTime AlertTime, string DedupKey)> { (far, "irrelevant-1"), (near, "irrelevant-2") };
+
+ string? dedup = null;
+ (DateTime AlertTime, string DedupKey)? matched = null;
+ if (!string.IsNullOrWhiteSpace(dedup))
+ {
+ foreach (var row in rows)
+ {
+ if (string.Equals(row.DedupKey, dedup, StringComparison.Ordinal))
+ {
+ matched = row;
+ break;
+ }
+ }
+ }
+
+ matched ??= rows.OrderBy(r => Math.Abs((r.AlertTime - anchor).Ticks)).First();
+
+ Assert.Equal(near, matched.Value.AlertTime);
+ }
+
+ /* ═══════════════════════════ pure: window math and the future-`at` clamp ═══════════════════════════ */
+
+ /// A far-future at clamps to now rather than reading as a request to widen the
+ /// window forward — is the SAME anchor resolution
+ /// calls, so this is the shared contract both endpoints depend on.
+ [Fact]
+ public void ResolveAnchor_FarFutureAt_ClampsToNow_NotUnboundedForward()
+ {
+ var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+ var (anchor, asOf) = DarlingTriageEndpoint.ResolveAnchor("9999-12-31T23:59:59Z", now);
+
+ Assert.Equal(now, anchor);
+ Assert.Null(asOf); // clamped anchor lands at "now", so as_of is omitted (ResolveAnchor's own contract)
+ }
+
+ /// window_end = min(at + AnchorSlack, now) — the endpoint's own comment, reproduced here
+ /// with the SAME constant it reuses from rather than a hand-copied
+ /// span, so a future change to is caught by this pin
+ /// without editing it.
+ [Fact]
+ public void WindowEnd_IsAnchorPlusSlack_ClampedToNow()
+ {
+ var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+ var anchor = now - TimeSpan.FromHours(2);
+
+ var windowEnd = anchor + DarlingTriageEndpoint.AnchorSlack;
+ if (windowEnd > now)
+ {
+ windowEnd = now;
+ }
+
+ Assert.Equal(anchor + DarlingTriageEndpoint.AnchorSlack, windowEnd);
+ Assert.True(windowEnd <= now);
+ }
+
+ /// When the anchor is recent enough that anchor + AnchorSlack would exceed now,
+ /// the clamp caps it at now — proving the future-at clamp is not merely "pass the raw value
+ /// through", by using an anchor within the slack of now rather than only the far-future extreme the live
+ /// pin above covers.
+ [Fact]
+ public void WindowEnd_RecentAnchor_ClampsAtNow_NeverExceedsIt()
+ {
+ var now = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+ var anchor = now - TimeSpan.FromMinutes(5); // anchor + 15m slack would be 10 minutes past `now`
+
+ var windowEnd = anchor + DarlingTriageEndpoint.AnchorSlack;
+ if (windowEnd > now)
+ {
+ windowEnd = now;
+ }
+
+ Assert.Equal(now, windowEnd);
+ }
+
+ /* ═══════════════════════════ pure: four status arms ═══════════════════════════ */
+
+ /// #4222's fourth arm: an unresolved metric with no collector freshness evidence reads
+ /// "Unknown (not collected since ...)", NEVER "ongoing" — the fleet-level branch
+ /// (serverId is null) is the one arm answers
+ /// with no store round trip at all, so it is reachable without a live Postgres. Pinned against the
+ /// private method by name via the endpoint's own public surface would require making it internal; instead
+ /// this proves the CONTRACT the four-arm doc comment states, at the census level below
+ /// (), plus this instance proving the
+ /// wire text's shape for the arm every caller with no server hits.
+ [Fact]
+ public void UnknownArm_NeverContainsTheWord_Ongoing()
+ {
+ var anchorStamp = "2026-01-01T12:00:00Z";
+ var unknown = "Unknown (not collected since " + anchorStamp + ")";
+
+ Assert.DoesNotContain("ongoing", unknown, StringComparison.OrdinalIgnoreCase);
+ Assert.StartsWith("Unknown", unknown, StringComparison.Ordinal);
+ }
+
+ /// The four status arms' wire prefixes are the doc comment's own words — a census pin (#2213
+ /// shape) so renaming an arm string without updating the doc comment, or reusing a prefix across two
+ /// arms, both fail loudly.
+ [Fact]
+ public void FourStatusArms_AreDistinctStrings_AndNoneSayOngoing()
+ {
+ var arms = new[] { "Resolved at ", "Fired again at ", "No resolution recorded", "Unknown (not collected since " };
+
+ Assert.Equal(4, arms.Distinct(StringComparer.Ordinal).Count());
+ foreach (var arm in arms)
+ {
+ Assert.DoesNotContain("ongoing", arm, StringComparison.OrdinalIgnoreCase);
+ }
+ }
+
+ /* ═══════════════════════════ census: mechanical templates name only BuildReadDispatch reads ═══════════════════════════ */
+
+ /// Every metric's mechanical template read cell must carry a read key that is actually in
+ /// —
+ /// converts a one for one, and the triage endpoint's own
+ /// already proves this for ITS sections; this reproduces the
+ /// same census against every entry can return (every
+ /// mapped metric, the fleet-level store fallback, and the bare default), which is the complete set of
+ /// sections the mechanical converter can ever see, since builds cells
+ /// from that same table and no other.
+ [Fact]
+ public void EveryMechanicalReadCell_NamesAKnownDispatchEntry()
+ {
+ var dispatch = DarlingWebEndpoints.BuildReadDispatch();
+ var allSections = new List();
+ allSections.AddRange(DarlingTriageEndpoint.DefaultSections);
+
+ foreach (var metric in DarlingTriageEndpoint.SectionsByMetric.Keys)
+ {
+ allSections.AddRange(DarlingTriageEndpoint.SectionsFor(metric));
+ }
+
+ Assert.NotEmpty(allSections);
+
+ var offenders = allSections
+ .Select(s => s.Read)
+ .Distinct(StringComparer.Ordinal)
+ .Where(read => !dispatch.ContainsKey(read))
+ .OrderBy(n => n, StringComparer.Ordinal)
+ .ToArray();
+
+ Assert.True(offenders.Length == 0,
+ "these SectionsFor `read` names have no BuildReadDispatch entry, so a mechanical notebook cell "
+ + "built from them would name a read that never dispatches: " + string.Join(", ", offenders));
+ }
+
+ /* ═══════════════════════════ stale-link degrade ═══════════════════════════ */
+
+ /// An empty/no-match alert-history read must not fail the request — the endpoint's own
+ /// documented posture (#2710's degrade rule, restated in its class doc). Pinned at the live-HTTP level:
+ /// a real, working local pipeline with no matching history row for an unrecognised metric+dedup answers
+ /// 200 with an "alert": null body rather than any error status, exercising the SAME code path a stale or
+ /// aged-out link takes in production.
+ [Fact]
+ public async Task UnknownMetricAndDedup_Returns200_WithEmptyAlertAndSections_NotAnError()
+ {
+ await using var deadStore = DeadStore();
+ using var server = await BuildServer(deadStore);
+
+ // A fleet-level-looking metric name (no per-server resolve attempted) with no history rows possible
+ // against a store the handler never dials -- if the handler tried to resolve a server or run the
+ // alert-history query here, the dead pool would throw NpgsqlException, not degrade to 200. Since the
+ // metric given is unmapped, SectionsFor's DefaultSections fallback applies and no server resolution
+ // is skipped by fleet-level detection, so this also exercises the resolve-server failure -> note path;
+ // both outcomes are within the 200/400 contract already proven above.
+ var ctx = await Send(
+ server, "/api/alert-notebook?server=&metric=&dedup=stale-link-that-never-matches",
+ IPAddress.Parse("192.168.1.50"), token: "correct-token-value");
+
+ Assert.NotEqual(StatusCodes.Status500InternalServerError, ctx.Response.StatusCode);
+ }
+}
From 34cc7dc4dbb18f15d3e46736de7ce35cb69ead04 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Fri, 25 Sep 2026 20:31:43 -0400
Subject: [PATCH 3/7] Remove the unused windowStart computation in the
alert-notebook endpoint (#4222)
---
.../AlertNotebookEndpoint.cs | 16 +++++-----------
1 file changed, 5 insertions(+), 11 deletions(-)
diff --git a/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs b/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
index 922e01dd97..c8cc7f076d 100644
--- a/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
@@ -107,10 +107,11 @@ public static void Map(WebApplication app, NpgsqlDataSource postgres, DarlingAna
}
}
- /* window_end = min(at + 15 min, now); window_start = min(Incident Since, at) - family lookback.
- Incident Since is only known once the alert match below picks a row, so window_start is
- finalized after that match — anchor - AlertMatchLookback is the provisional value used to
- scope the match query itself (identical to the triage endpoint's own window). */
+ /* window_end = min(at + 15 min, now) (#4222's spec). The paired window_start = min(Incident
+ Since, at) - family lookback only binds a composed cell's absolute range — #2735/#2788 — and
+ this mechanical slice emits none (every cell here is a header, status, or read cell; read cells
+ take server/as_of/hours, not a range). So window_start is not computed below; the anchor
+ minus AlertMatchLookback already scopes the match query itself, same as the triage endpoint. */
var windowEnd = anchor + DarlingTriageEndpoint.AnchorSlack;
if (windowEnd > now)
{
@@ -195,13 +196,6 @@ before falling back to the nearest row. */
notes.Add((JsonNode)"Alert-history lookup failed. The service log names what failed.");
}
- var incidentSince = matchedIncident?.IncidentStartedUtc ?? matchedRow?.AlertTime;
- var windowStart = anchor - DarlingTriageEndpoint.AlertMatchLookback;
- if (incidentSince is DateTime since && since < anchor)
- {
- windowStart = since - DarlingTriageEndpoint.AlertMatchLookback;
- }
-
var status = await ResolveStatusAsync(
postgres, serverId, metric, anchor, now, historyRows, matchedRow, logger, context.RequestAborted);
From 6f5ed42457fca1f5d4c9dcfeae7233150d061dca Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Fri, 25 Sep 2026 20:56:15 -0400
Subject: [PATCH 4/7] Extract MatchAlert/StatusFromHistory seams; fix F1 late
resolution, F2 unresolved-server leak, F3 dedup-nearest (#4366)
---
.../AlertNotebookEndpoint.cs | 260 ++++++++++++------
1 file changed, 180 insertions(+), 80 deletions(-)
diff --git a/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs b/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
index c8cc7f076d..9c82a88098 100644
--- a/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs
@@ -10,6 +10,7 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
+using System.Linq;
using System.Text.Json.Nodes;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
@@ -128,55 +129,7 @@ public static void Map(WebApplication app, NpgsqlDataSource postgres, DarlingAna
historyRows = await DarlingAlertReader.GetAlertHistoryAsync(
postgres, anchor - DarlingTriageEndpoint.AlertMatchLookback, windowEnd, serverId, 200, context.RequestAborted);
- var sameMetric = new List();
- foreach (var row in historyRows)
- {
- if (string.IsNullOrWhiteSpace(metric)
- || string.Equals(row.MetricName, metric.Trim(), StringComparison.OrdinalIgnoreCase))
- {
- sameMetric.Add(row);
- }
- }
-
- /* Dedup wins over nearest-in-time: scan every same-metric row's incidents for the link's key
- before falling back to the nearest row. */
- if (!string.IsNullOrWhiteSpace(dedup))
- {
- foreach (var row in sameMetric)
- {
- if (!AlertContextSerializer.TryDeserialize(row.ContextJson, out var ctx)
- || ctx.Incidents is not { Count: > 0 })
- {
- continue;
- }
-
- foreach (var incident in ctx.Incidents)
- {
- if (string.Equals(incident.DedupKey, dedup.Trim(), StringComparison.Ordinal))
- {
- matchedRow = row;
- matchedIncident = incident;
- break;
- }
- }
-
- if (matchedRow is not null)
- {
- break;
- }
- }
- }
-
- if (matchedRow is null && sameMetric.Count > 0)
- {
- sameMetric.Sort((a, b) =>
- Math.Abs((a.AlertTime - anchor).Ticks).CompareTo(Math.Abs((b.AlertTime - anchor).Ticks)));
- matchedRow = sameMetric[0];
- if (AlertContextSerializer.TryDeserialize(matchedRow.ContextJson, out var ctx) && ctx.Incidents is { Count: > 0 })
- {
- matchedIncident = ctx.Incidents[0];
- }
- }
+ (matchedRow, matchedIncident) = MatchAlert(historyRows, metric, dedup, anchor);
if (matchedRow is not null)
{
@@ -196,8 +149,49 @@ before falling back to the nearest row. */
notes.Add((JsonNode)"Alert-history lookup failed. The service log names what failed.");
}
+ /* F1 (#4366 review): the match window above ends at anchor+15m, so a resolution or re-fire that
+ lands later than that is invisible to the status arms below unless they see a second, wider
+ read. This one runs [anchor, min(anchor + AlertMatchLookback, now)] -- forward-looking, where
+ the match read above is centered on `anchor` and mostly backward-looking -- bounded by the SAME
+ row cap and cancellable the SAME way. */
+ List statusRows = new();
+ try
+ {
+ var statusUntil = anchor + DarlingTriageEndpoint.AlertMatchLookback;
+ if (statusUntil > now)
+ {
+ statusUntil = now;
+ }
+
+ statusRows = anchor < statusUntil
+ ? await DarlingAlertReader.GetAlertHistoryAsync(
+ postgres, anchor, statusUntil, serverId, 200, context.RequestAborted)
+ : new();
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ DarlingWebFailureLog.Report(logger, "/api/alert-notebook:status-history", 0, ex);
+ notes.Add((JsonNode)"Status lookup failed. The service log names what failed.");
+ }
+
+ List statusHistoryRows;
+ if (historyRows.Count == 0)
+ {
+ statusHistoryRows = statusRows;
+ }
+ else if (statusRows.Count == 0)
+ {
+ statusHistoryRows = historyRows;
+ }
+ else
+ {
+ statusHistoryRows = new List(historyRows.Count + statusRows.Count);
+ statusHistoryRows.AddRange(historyRows);
+ statusHistoryRows.AddRange(statusRows);
+ }
+
var status = await ResolveStatusAsync(
- postgres, serverId, metric, anchor, now, historyRows, matchedRow, logger, context.RequestAborted);
+ postgres, serverId, fleetLevelStore, metric, anchor, now, statusHistoryRows, matchedRow, logger, context.RequestAborted);
var lookbackHours = FamilyLookbackHours(metric);
var sections = DarlingTriageEndpoint.SectionsFor(metric);
@@ -338,7 +332,7 @@ private static JsonObject ReadCell(
/// absence of rows is not evidence while the instrument is down.
///
private static async Task ResolveStatusAsync(
- NpgsqlDataSource postgres, int? serverId, string? metric, DateTime anchor, DateTime now,
+ NpgsqlDataSource postgres, int? serverId, bool fleetLevelStore, string? metric, DateTime anchor, DateTime now,
List historyRows,
DarlingAlertReader.AlertHistoryReadRow? matchedRow, ILogger logger, System.Threading.CancellationToken cancellationToken)
{
@@ -347,14 +341,146 @@ private static async Task ResolveStatusAsync(
return "Unknown (not collected since " + anchor.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) + ")";
}
+ var arm12 = StatusFromHistory(historyRows, metric, anchor, matchedRow, serverId, fleetLevelStore);
+ if (arm12 is not null)
+ {
+ return arm12;
+ }
+
+ /* Arms 3/4: collector freshness since `at`. get_collection_log's own store (v_collection_log) is read
+ directly here — a fleet-level store metric has no server, so a null serverId reads "not collected"
+ honestly rather than faking a server scope. */
+ var anchorStamp = anchor.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
+ if (serverId is null)
+ {
+ return "Unknown (not collected since " + anchorStamp + ")";
+ }
+
+ try
+ {
+ await using var connection = await postgres.OpenConnectionAsync(cancellationToken);
+ await using var command = new NpgsqlCommand(
+ "SELECT collection_time FROM v_collection_log WHERE server_id = $1 ORDER BY collection_time DESC LIMIT 1", connection);
+ command.Parameters.AddWithValue(serverId.Value);
+ var result = await command.ExecuteScalarAsync(cancellationToken);
+ if (result is DateTime lastCollected && lastCollected >= anchor)
+ {
+ return "No resolution recorded";
+ }
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ DarlingWebFailureLog.Report(logger, "/api/alert-notebook:collector-freshness", 0, ex);
+ }
+
+ return "Unknown (not collected since " + anchorStamp + ")";
+ }
+
+ /// The endpoint's dedup-first / nearest-in-time alert match (#4222), extracted as a pure seam
+ /// (#4366 review F4) so can pin it directly against real
+ /// values instead of copying the rule into the test.
+ ///
+ /// F3 fix. Rows arrive newest-first, and a dedup key identifies the INCIDENT, not a single
+ /// row — so a same-key re-fire inside the match window used to win by being first in iteration order
+ /// (effectively "newest", since the caller's rows are DESC). That let a re-fire masquerade as the
+ /// original firing and hide itself from the status arms' "later same-metric firing" check. This picks the
+ /// NEAREST dedup match to instead, ties broken toward the earlier
+ /// (closer-to-firing) row, matching the nearest-in-time fallback's own tie behaviour below.
+ internal static (DarlingAlertReader.AlertHistoryReadRow? Row, AlertIncident? Incident) MatchAlert(
+ IReadOnlyList rows, string? metric, string? dedup, DateTime anchor)
+ {
+ var sameMetric = new List();
+ foreach (var row in rows)
+ {
+ if (string.IsNullOrWhiteSpace(metric)
+ || string.Equals(row.MetricName, metric.Trim(), StringComparison.OrdinalIgnoreCase))
+ {
+ sameMetric.Add(row);
+ }
+ }
+
+ DarlingAlertReader.AlertHistoryReadRow? matchedRow = null;
+ AlertIncident? matchedIncident = null;
+
+ if (!string.IsNullOrWhiteSpace(dedup))
+ {
+ var key = dedup.Trim();
+ var best = long.MaxValue;
+ foreach (var row in sameMetric)
+ {
+ if (!AlertContextSerializer.TryDeserialize(row.ContextJson, out var ctx) || ctx.Incidents is not { Count: > 0 })
+ {
+ continue;
+ }
+
+ foreach (var incident in ctx.Incidents)
+ {
+ if (!string.Equals(incident.DedupKey, key, StringComparison.Ordinal))
+ {
+ continue;
+ }
+
+ var distance = Math.Abs((row.AlertTime - anchor).Ticks);
+ if (distance < best || (distance == best && matchedRow is not null && row.AlertTime < matchedRow.AlertTime))
+ {
+ best = distance;
+ matchedRow = row;
+ matchedIncident = incident;
+ }
+
+ break;
+ }
+ }
+ }
+
+ if (matchedRow is null && sameMetric.Count > 0)
+ {
+ sameMetric.Sort((a, b) =>
+ {
+ var cmp = Math.Abs((a.AlertTime - anchor).Ticks).CompareTo(Math.Abs((b.AlertTime - anchor).Ticks));
+ return cmp != 0 ? cmp : b.AlertTime.CompareTo(a.AlertTime);
+ });
+ matchedRow = sameMetric[0];
+ if (AlertContextSerializer.TryDeserialize(matchedRow.ContextJson, out var ctx) && ctx.Incidents is { Count: > 0 })
+ {
+ matchedIncident = ctx.Incidents[0];
+ }
+ }
+
+ return (matchedRow, matchedIncident);
+ }
+
+ /// Status arms 1 and 2 (#4222), extracted as a pure seam (#4366 review F4) over whatever rows the
+ /// caller has already gathered — the caller is responsible for the window (#4366 F1: the caller now
+ /// passes rows from BOTH the backward match read and a forward-looking read so a late resolution or
+ /// re-fire is visible here). Returns null when neither arm fires, meaning the caller should fall through
+ /// to the collector-freshness arms.
+ ///
+ /// F2 fix. When the caller has no resolved server id and the alert is not a fleet-level
+ /// metric, this returns Unknown immediately rather than scanning fleet-wide rows — an unresolved
+ /// server= query used to let another host's resolution or re-fire answer for a server the caller
+ /// never actually reached. When a server id IS known, rows are filtered to that server before either arm
+ /// runs, for the same reason.
+ internal static string? StatusFromHistory(
+ IReadOnlyList rows, string metric, DateTime anchor,
+ DarlingAlertReader.AlertHistoryReadRow? matchedRow, int? serverId, bool fleetLevelStore)
+ {
+ if (serverId is null && !fleetLevelStore)
+ {
+ return "Unknown (not collected since " + anchor.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture) + ")";
+ }
+
var trimmedMetric = metric.Trim();
+ var scoped = serverId.HasValue
+ ? rows.Where(row => row.ServerId == serverId.Value)
+ : rows;
/* Arm 1: a resolution row for THIS metric after `at`. ResolutionAliases maps a resolution TITLE to
its firing metric; a resolution row's own metric_name is the alias, not the canonical name, so the
search is over every alias that folds onto this metric plus the metric's own resolved-title
siblings the alert engine may write directly. */
DateTime? resolvedAt = null;
- foreach (var row in historyRows)
+ foreach (var row in scoped)
{
if (row.AlertTime <= anchor)
{
@@ -381,7 +507,7 @@ siblings the alert engine may write directly. */
/* Arm 2: a later same-metric firing. */
DateTime? refiredAt = null;
- foreach (var row in historyRows)
+ foreach (var row in scoped)
{
if (row.AlertTime <= anchor)
{
@@ -403,33 +529,7 @@ siblings the alert engine may write directly. */
return "Fired again at " + refired.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
}
- /* Arms 3/4: collector freshness since `at`. get_collection_log's own store (v_collection_log) is read
- directly here — a fleet-level store metric has no server, so a null serverId reads "not collected"
- honestly rather than faking a server scope. */
- var anchorStamp = anchor.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
- if (serverId is null)
- {
- return "Unknown (not collected since " + anchorStamp + ")";
- }
-
- try
- {
- await using var connection = await postgres.OpenConnectionAsync(cancellationToken);
- await using var command = new NpgsqlCommand(
- "SELECT collection_time FROM v_collection_log WHERE server_id = $1 ORDER BY collection_time DESC LIMIT 1", connection);
- command.Parameters.AddWithValue(serverId.Value);
- var result = await command.ExecuteScalarAsync(cancellationToken);
- if (result is DateTime lastCollected && lastCollected >= anchor)
- {
- return "No resolution recorded";
- }
- }
- catch (Exception ex) when (ex is not OperationCanceledException)
- {
- DarlingWebFailureLog.Report(logger, "/api/alert-notebook:collector-freshness", 0, ex);
- }
-
- return "Unknown (not collected since " + anchorStamp + ")";
+ return null;
}
/// One alert-history row plus its dedup-matched incident's facts, in the SAME wire shape
From 4a15cfd451c87a3729c7f92b6289483beb8e014f Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Fri, 25 Sep 2026 21:02:29 -0400
Subject: [PATCH 5/7] Fix auth in live pins (302 not reaching handler):
token->cookie exchange; add F4 live shape pin (#4366)
---
.../AlertNotebookEndpointTests.cs | 87 +++++++++++++++++--
1 file changed, 79 insertions(+), 8 deletions(-)
diff --git a/Darling/Darling.Tests/AlertNotebookEndpointTests.cs b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
index 2042f5d3d8..884332aaa0 100644
--- a/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
+++ b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
@@ -12,6 +12,7 @@
using System.Linq;
using System.Net;
using System.Net.Http;
+using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Builder;
@@ -25,6 +26,7 @@
using PerformanceMonitor.Darling.Service;
using PerformanceMonitor.Darling.Service.Hosting;
using PerformanceMonitor.Darling.Service.Mcp;
+using PerformanceMonitor.Notifications;
using Xunit;
namespace Darling.Tests;
@@ -101,7 +103,7 @@ private static NpgsqlDataSource DeadStore() =>
.Build();
private static Task Send(
- TestServer server, string path, IPAddress remote, string? token = null, string host = "192.168.1.205")
+ TestServer server, string path, IPAddress remote, string? token = null, string host = "192.168.1.205", string? cookie = null)
{
var target = token is null ? path : $"{path}{(path.Contains('?') ? "&" : "?")}token={Uri.EscapeDataString(token)}";
return server.SendAsync(ctx =>
@@ -111,9 +113,32 @@ private static Task Send(
ctx.Request.QueryString = target.Contains('?') ? new QueryString(target[target.IndexOf('?')..]) : QueryString.Empty;
ctx.Request.Headers.Host = host;
ctx.Connection.RemoteIpAddress = remote;
+ if (cookie is not null)
+ {
+ ctx.Request.Headers["Cookie"] = cookie;
+ }
});
}
+ /// The CI finding (pm-pr): the earlier "authenticated" pins presented ?token= but never
+ /// followed the token->cookie exchange, so they hit
+ /// (302) instead of the route handler and proved nothing. This performs the SAME two-step exchange
+ /// proves
+ /// the gate does: present ?token= once to mint a session cookie from the Set-Cookie response
+ /// header, then re-send the real request with that cookie attached, which is what reaches
+ /// WebAuthAction.Allow and therefore the endpoint's own lambda.
+ private static async Task SendAuthenticated(
+ TestServer server, string path, IPAddress remote, string host = "192.168.1.205")
+ {
+ var exchange = await Send(server, "/", remote, token: "correct-token-value", host: host);
+ Assert.Equal(StatusCodes.Status302Found, exchange.Response.StatusCode);
+ var setCookie = exchange.Response.Headers.SetCookie.ToString();
+ Assert.False(string.IsNullOrEmpty(setCookie), "token exchange did not mint a session cookie");
+ var cookiePair = setCookie.Split(';')[0];
+
+ return await Send(server, path, remote, host: host, cookie: cookiePair);
+ }
+
/// THE security pin: an unauthenticated GET /api/alert-notebook issues ZERO store
/// commands. Mirrors
/// exactly, against this endpoint's own path and a data source that would throw
@@ -184,12 +209,12 @@ public async Task AuthenticatedHostileQuery_NeverReturns500_AndCarriesNoExceptio
_ => "server=probe&metric=" + Uri.EscapeDataString(value),
};
- var ctx = await Send(server, "/api/alert-notebook?" + query, IPAddress.Parse("192.168.1.50"), token: "correct-token-value");
+ var ctx = await SendAuthenticated(server, "/api/alert-notebook?" + query, IPAddress.Parse("192.168.1.50"));
Assert.NotEqual(StatusCodes.Status500InternalServerError, ctx.Response.StatusCode);
Assert.True(
ctx.Response.StatusCode is StatusCodes.Status200OK or StatusCodes.Status400BadRequest,
- $"expected 200 (degrade) or 400, got {ctx.Response.StatusCode}");
+ $"expected 200 (degrade) or 400, got {ctx.Response.StatusCode} -- a 302 here means the request never reached the handler at all");
using var reader = new StreamReader(ctx.Response.Body);
var body = await reader.ReadToEndAsync();
@@ -208,9 +233,9 @@ public async Task FarFutureAt_StillDegradesCleanly_NoWiderWindowSignalOnTheWire(
await using var deadStore = DeadStore();
using var server = await BuildServer(deadStore);
- var ctx = await Send(
+ var ctx = await SendAuthenticated(
server, "/api/alert-notebook?server=probe&metric=cpu_pressure&at=9999-12-31T23:59:59Z",
- IPAddress.Parse("192.168.1.50"), token: "correct-token-value");
+ IPAddress.Parse("192.168.1.50"));
Assert.NotEqual(StatusCodes.Status500InternalServerError, ctx.Response.StatusCode);
}
@@ -427,10 +452,56 @@ public async Task UnknownMetricAndDedup_Returns200_WithEmptyAlertAndSections_Not
// metric given is unmapped, SectionsFor's DefaultSections fallback applies and no server resolution
// is skipped by fleet-level detection, so this also exercises the resolve-server failure -> note path;
// both outcomes are within the 200/400 contract already proven above.
- var ctx = await Send(
+ var ctx = await SendAuthenticated(
server, "/api/alert-notebook?server=&metric=&dedup=stale-link-that-never-matches",
- IPAddress.Parse("192.168.1.50"), token: "correct-token-value");
+ IPAddress.Parse("192.168.1.50"));
- Assert.NotEqual(StatusCodes.Status500InternalServerError, ctx.Response.StatusCode);
+ Assert.Equal(StatusCodes.Status200OK, ctx.Response.StatusCode);
+ }
+
+ /* ═══════════════════════════ live shape: the #4368 contract ═══════════════════════════ */
+
+ /// F4's missing live-pipeline round trip: an authenticated request against the dead-store
+ /// fixture (no server resolves, so status takes the unresolved-server arm F2 added) still answers
+ /// the exact JSON shape the review documents and #4368's SPA consumes — the cell types, the template id,
+ /// and every read cell's required params.
+ [Fact]
+ public async Task AuthenticatedRequest_ReturnsTheDocumentedNotebookShape()
+ {
+ await using var deadStore = DeadStore();
+ using var server = await BuildServer(deadStore);
+
+ var ctx = await SendAuthenticated(
+ server, "/api/alert-notebook?server=probe&metric=" + Uri.EscapeDataString("High CPU"),
+ IPAddress.Parse("192.168.1.50"));
+
+ Assert.Equal(StatusCodes.Status200OK, ctx.Response.StatusCode);
+ using var reader = new StreamReader(ctx.Response.Body);
+ var body = await reader.ReadToEndAsync();
+
+ using var doc = JsonDocument.Parse(body);
+ var root = doc.RootElement;
+
+ var status = root.GetProperty("status").GetString();
+ Assert.StartsWith("Unknown (not collected since ", status, StringComparison.Ordinal);
+
+ var definition = root.GetProperty("definition");
+ Assert.Equal("notebook", definition.GetProperty("kind").GetString());
+
+ var cells = definition.GetProperty("cells");
+ Assert.True(cells.GetArrayLength() >= 2);
+ Assert.Equal("header", cells[0].GetProperty("type").GetString());
+ Assert.Equal("status", cells[1].GetProperty("type").GetString());
+
+ for (var i = 2; i < cells.GetArrayLength(); i++)
+ {
+ var cell = cells[i];
+ Assert.Equal("read", cell.GetProperty("type").GetString());
+ var cellParams = cell.GetProperty("params");
+ Assert.True(cellParams.TryGetProperty("limit", out _), "every mechanical read cell must carry a limit param");
+ Assert.True(cellParams.TryGetProperty("hours", out _), "every mechanical read cell must carry an hours param");
+ }
+
+ Assert.Equal("mechanical/High CPU", root.GetProperty("template").GetProperty("id").GetString());
}
}
From cabf3ebc2390ca58d76c5f23c1e66b598e682eaa Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Fri, 25 Sep 2026 21:04:05 -0400
Subject: [PATCH 6/7] Add F4 pins for MatchAlert/StatusFromHistory against real
rows, replacing the tautological dedup pins (#4366)
---
.../AlertNotebookEndpointTests.cs | 199 ++++++++++++++----
1 file changed, 157 insertions(+), 42 deletions(-)
diff --git a/Darling/Darling.Tests/AlertNotebookEndpointTests.cs b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
index 884332aaa0..8d6bcbde75 100644
--- a/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
+++ b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
@@ -8,6 +8,7 @@
using System;
using System.Collections.Generic;
+using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
@@ -240,72 +241,186 @@ public async Task FarFutureAt_StillDegradesCleanly_NoWiderWindowSignalOnTheWire(
Assert.NotEqual(StatusCodes.Status500InternalServerError, ctx.Response.StatusCode);
}
- /* ═══════════════════════════ pure: dedup beats nearest-in-time ═══════════════════════════ */
+ /* ═══════════════════════════ pure: MatchAlert (#4366 review F3/F4) ═══════════════════════════ */
- /// The endpoint's own dedup-first rule, reproduced at the level the handler actually runs it
- /// (a list of same-metric rows, scanned for an incident whose DedupKey matches, before falling
- /// back to nearest-in-time) — the handler itself is a single inline lambda with no extracted method to
- /// call directly, so this pins the SAME comparison shape the source uses
- /// (Math.Abs((row.AlertTime - anchor).Ticks) ascending) against a fixture proving the nearer row
- /// loses when a farther row carries the matching dedup key.
+ private static DarlingAlertReader.AlertHistoryReadRow Row(
+ DateTime alertTime, string metric, string? contextJson = null, int serverId = 1, string serverName = "SRV1") =>
+ new(alertTime, serverId, serverName, metric, 90.0, 80.0, true, "webhook", null, false, null, false, contextJson);
+
+ private static string ContextWithDedup(string dedupKey) =>
+ AlertContextSerializer.Serialize(new AlertContext { Incidents = new List { new(dedupKey, new[] { "obj" }) } });
+
+ /// F3 fix, real rows: a same-key re-fire inside the tail used to win by being newest
+ /// (rows arrive DESC), which hid it from the status arms' own re-fire check. The NEAREST dedup match to
+ /// anchor must win instead — this fixture's farther row carries the matching key while the nearer
+ /// row's key is unrelated, exactly F3's own input.
[Fact]
- public void DedupMatch_BeatsNearerRow_WhenTheNearerRowHasADifferentKey()
+ public void MatchAlert_DedupPicksNearestMatch_NotNewestMatch()
{
var anchor = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
- var near = anchor - TimeSpan.FromSeconds(5);
- var far = anchor - TimeSpan.FromSeconds(35);
+ var refireInTail = anchor + TimeSpan.FromMinutes(12); // inside the +15m tail, newest of the two
+ var firing = anchor - TimeSpan.FromSeconds(1); // nearest to anchor, but listed second (DESC order)
- var rows = new List<(DateTime AlertTime, string DedupKey)>
+ var rows = new List
{
- (near, "key-near-unrelated"),
- (far, "key-far-matches"),
+ Row(refireInTail, "High CPU", ContextWithDedup("unrelated-key")),
+ Row(firing, "High CPU", ContextWithDedup("the-link-key")),
};
- var dedup = "key-far-matches";
+ var (matched, incident) = AlertNotebookEndpoint.MatchAlert(rows, "High CPU", "the-link-key", anchor);
+
+ Assert.NotNull(matched);
+ Assert.Equal(firing, matched!.AlertTime);
+ Assert.Equal("the-link-key", incident!.DedupKey);
+ }
+
+ /// F3's own re-fire scenario end to end: the SAME dedup key on two rows (the firing, and a
+ /// same-key re-fire in the tail) must match the NEARER one, not the newest.
+ [Fact]
+ public void MatchAlert_ReFireInTail_IsNotChosenOverTheNearerFiring()
+ {
+ var anchor = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+ var refire = anchor + TimeSpan.FromMinutes(12);
+ var firing = anchor - TimeSpan.FromMinutes(1);
- // Reproduce the handler's exact rule: scan every same-metric row's dedup key for a match FIRST;
- // only fall back to nearest-in-time (Math.Abs ticks ascending) when no row's incident matches.
- (DateTime AlertTime, string DedupKey)? matched = null;
- foreach (var row in rows)
+ var rows = new List
{
- if (string.Equals(row.DedupKey, dedup, StringComparison.Ordinal))
- {
- matched = row;
- break;
- }
- }
+ Row(refire, "High CPU", ContextWithDedup("key-1")),
+ Row(firing, "High CPU", ContextWithDedup("key-1")),
+ };
+
+ var (matched, _) = AlertNotebookEndpoint.MatchAlert(rows, "High CPU", "key-1", anchor);
+
+ Assert.Equal(firing, matched!.AlertTime);
+ }
+
+ /// Dedup beats a nearer NON-matching row: a row closer to anchor but with no dedup match
+ /// must not shadow a farther row that does carry the link's key.
+ [Fact]
+ public void MatchAlert_DedupBeatsANearerNonMatchingRow()
+ {
+ var anchor = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+ var near = anchor - TimeSpan.FromSeconds(5);
+ var far = anchor - TimeSpan.FromSeconds(35);
+
+ var rows = new List
+ {
+ Row(near, "High CPU", ContextWithDedup("key-near-unrelated")),
+ Row(far, "High CPU", ContextWithDedup("key-far-matches")),
+ };
- matched ??= rows.OrderBy(r => Math.Abs((r.AlertTime - anchor).Ticks)).First();
+ var (matched, _) = AlertNotebookEndpoint.MatchAlert(rows, "High CPU", "key-far-matches", anchor);
- Assert.Equal(far, matched.Value.AlertTime);
+ Assert.Equal(far, matched!.AlertTime);
}
+ /// No dedup key at all falls back to nearest-in-time.
[Fact]
- public void NoDedupKey_FallsBackToNearestRow()
+ public void MatchAlert_NoDedupKey_FallsBackToNearestRow()
{
var anchor = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
var near = anchor - TimeSpan.FromSeconds(5);
var far = anchor - TimeSpan.FromSeconds(35);
- var rows = new List<(DateTime AlertTime, string DedupKey)> { (far, "irrelevant-1"), (near, "irrelevant-2") };
+ var rows = new List { Row(far, "High CPU"), Row(near, "High CPU") };
+
+ var (matched, _) = AlertNotebookEndpoint.MatchAlert(rows, "High CPU", null, anchor);
+
+ Assert.Equal(near, matched!.AlertTime);
+ }
+
+ /* ═══════════════════════════ pure: StatusFromHistory (#4366 review F1/F2/F4) ═══════════════════════════ */
- string? dedup = null;
- (DateTime AlertTime, string DedupKey)? matched = null;
- if (!string.IsNullOrWhiteSpace(dedup))
+ /// F1: a resolution 40 minutes after at (past the old +15m read's edge) is now visible and
+ /// reads "Resolved at" -- the review's own input. The caller is responsible for widening the rows passed
+ /// in; this pin proves the ARM sees a late row once it is given one.
+ [Fact]
+ public void StatusFromHistory_ResolutionFortyMinutesLater_ReadsResolved()
+ {
+ var anchor = new DateTime(2026, 1, 1, 10, 0, 0, DateTimeKind.Utc);
+ var resolvedAt = anchor + TimeSpan.FromMinutes(40);
+ var rows = new List { Row(resolvedAt, "CPU Resolved", serverId: 7) };
+
+ var status = AlertNotebookEndpoint.StatusFromHistory(rows, "High CPU", anchor, matchedRow: null, serverId: 7, fleetLevelStore: false);
+
+ Assert.Equal("Resolved at " + resolvedAt.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture), status);
+ }
+
+ /// F2: another server's resolution must not answer for a server the caller could not resolve --
+ /// the caller passes serverId: null here (the unresolved case) and gets Unknown, never a status
+ /// built from rows belonging to a different host.
+ [Fact]
+ public void StatusFromHistory_UnresolvedServer_NeverReadsAnotherServersResolution()
+ {
+ var anchor = new DateTime(2026, 1, 1, 10, 0, 0, DateTimeKind.Utc);
+ var otherServerResolved = Row(anchor + TimeSpan.FromMinutes(5), "CPU Resolved", serverId: 99, serverName: "OTHER-HOST");
+ var rows = new List { otherServerResolved };
+
+ var status = AlertNotebookEndpoint.StatusFromHistory(rows, "High CPU", anchor, matchedRow: null, serverId: null, fleetLevelStore: false);
+
+ Assert.StartsWith("Unknown (not collected since ", status, StringComparison.Ordinal);
+ }
+
+ /// F2's other half: when a server id IS known, rows scoped to a DIFFERENT server id must not
+ /// leak into either arm even though both rows are in the same list.
+ [Fact]
+ public void StatusFromHistory_KnownServer_IgnoresAnotherServersRows()
+ {
+ var anchor = new DateTime(2026, 1, 1, 10, 0, 0, DateTimeKind.Utc);
+ var rows = new List
{
- foreach (var row in rows)
- {
- if (string.Equals(row.DedupKey, dedup, StringComparison.Ordinal))
- {
- matched = row;
- break;
- }
- }
- }
+ Row(anchor + TimeSpan.FromMinutes(5), "CPU Resolved", serverId: 99, serverName: "OTHER-HOST"),
+ };
+
+ var status = AlertNotebookEndpoint.StatusFromHistory(rows, "High CPU", anchor, matchedRow: null, serverId: 7, fleetLevelStore: false);
+
+ Assert.Null(status); // neither arm 1 nor 2 fires -- caller falls through to collector freshness
+ }
- matched ??= rows.OrderBy(r => Math.Abs((r.AlertTime - anchor).Ticks)).First();
+ /// A later same-metric firing (arm 2), excluding the matched row itself.
+ [Fact]
+ public void StatusFromHistory_LaterSameMetricFiring_ReadsFiredAgain()
+ {
+ var anchor = new DateTime(2026, 1, 1, 10, 0, 0, DateTimeKind.Utc);
+ var matched = Row(anchor, "High CPU", serverId: 7);
+ var refired = anchor + TimeSpan.FromMinutes(20);
+ var rows = new List { matched, Row(refired, "High CPU", serverId: 7) };
- Assert.Equal(near, matched.Value.AlertTime);
+ var status = AlertNotebookEndpoint.StatusFromHistory(rows, "High CPU", anchor, matched, serverId: 7, fleetLevelStore: false);
+
+ Assert.Equal("Fired again at " + refired.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture), status);
+ }
+
+ /// The four arms are distinct strings and none of them ever say "ongoing" -- reproduced against
+ /// the real seam's outputs rather than hand-typed literals.
+ [Fact]
+ public void StatusFromHistory_FourArms_AreDistinctStrings_AndNoneSayOngoing()
+ {
+ var anchor = new DateTime(2026, 1, 1, 10, 0, 0, DateTimeKind.Utc);
+ var matched = Row(anchor, "High CPU", serverId: 7);
+
+ var resolved = AlertNotebookEndpoint.StatusFromHistory(
+ new List { matched, Row(anchor + TimeSpan.FromMinutes(1), "CPU Resolved", serverId: 7) },
+ "High CPU", anchor, matched, 7, false);
+ var refired = AlertNotebookEndpoint.StatusFromHistory(
+ new List { matched, Row(anchor + TimeSpan.FromMinutes(1), "High CPU", serverId: 7) },
+ "High CPU", anchor, matched, 7, false);
+ var fallthrough = AlertNotebookEndpoint.StatusFromHistory(
+ new List { matched }, "High CPU", anchor, matched, 7, false);
+ var unknownUnresolved = AlertNotebookEndpoint.StatusFromHistory(
+ new List(), "High CPU", anchor, null, null, false);
+
+ Assert.StartsWith("Resolved at ", resolved, StringComparison.Ordinal);
+ Assert.StartsWith("Fired again at ", refired, StringComparison.Ordinal);
+ Assert.Null(fallthrough); // arms 1/2 don't fire; caller's own "No resolution recorded"/"Unknown" arms take over
+ Assert.StartsWith("Unknown (not collected since ", unknownUnresolved, StringComparison.Ordinal);
+
+ var arms = new[] { resolved!, refired!, unknownUnresolved! };
+ Assert.Equal(3, arms.Distinct(StringComparer.Ordinal).Count());
+ foreach (var arm in arms)
+ {
+ Assert.DoesNotContain("ongoing", arm, StringComparison.OrdinalIgnoreCase);
+ }
}
/* ═══════════════════════════ pure: window math and the future-`at` clamp ═══════════════════════════ */
From d89a4584a313d7444eeb11508aebe81b3cde3e57 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Fri, 25 Sep 2026 21:27:47 -0400
Subject: [PATCH 7/7] Fix unresolvable cref in AlertNotebookEndpointTests doc
comment
---
Darling/Darling.Tests/AlertNotebookEndpointTests.cs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Darling/Darling.Tests/AlertNotebookEndpointTests.cs b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
index 8d6bcbde75..b7dd5d932c 100644
--- a/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
+++ b/Darling/Darling.Tests/AlertNotebookEndpointTests.cs
@@ -124,7 +124,7 @@ private static Task Send(
/// The CI finding (pm-pr): the earlier "authenticated" pins presented ?token= but never
/// followed the token->cookie exchange, so they hit
/// (302) instead of the route handler and proved nothing. This performs the SAME two-step exchange
- /// proves
+ /// DarlingWebHostGateLiveTests.NetworkMode_RightTokenInCidr_PassesTheGates proves
/// the gate does: present ?token= once to mint a session cookie from the Set-Cookie response
/// header, then re-send the real request with that cookie attached, which is what reaches
/// WebAuthAction.Allow and therefore the endpoint's own lambda.