Add authored Poison Wait alert-notebook template (#4223)

Darling/Darling.Tests/AlertNotebookAuthoredTemplateTests.cs

@@ -240,6 +240,10 @@ public void Budget_ReadCellsHaveLimitsExceptTheTrendRead_ComposedWindowsAreAtMos
                     $"authored read cell names a barred read: {read}");
 
                 var parameters = Assert.IsType<JsonObject>(cell["params"]);
+                /* Data-driven off the endpoint's OWN limitless-read set (#4223) rather than one hardcoded
+                   name, so a new bucketed/no-limit read (get_wait_trend, get_memory_grants,
+                   get_resource_semaphore) is covered the moment its family adds a row -- nothing here has
+                   to change. */
                 if (AlertNotebookEndpoint.s_authoredLimitlessTrendReads.Contains(read))
                 {
                     Assert.False(parameters.ContainsKey("limit"), $"{read} must not carry a limit param");

Darling/Darling.Tests/AlertNotebookTemplatePoisonWaitTests.cs

@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Globalization;
+using System.Linq;
+using System.Text.Json.Nodes;
+using PerformanceMonitor.Darling.Service;
+using PerformanceMonitor.Darling.Service.Mcp;
+using PerformanceMonitor.Notifications;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// <summary>
+/// Pins for #4223's authored Poison Wait alert-notebook template
+/// (<see cref="AlertNotebookEndpoint.AuthoredTemplate"/> routing "Poison Wait" to
+/// <c>authored/poison-wait</c> / <c>BuildPoisonWaitCells</c>). Follows the same shape as
+/// <c>AlertNotebookTemplateBlockingTests</c>/<c>Deadlocks</c>' shared theories, which already cover this
+/// family (<c>AlertNotebookAuthoredTemplateTests</c>' <c>AllAuthoredMetrics</c> enumerates
+/// <see cref="AlertNotebookEndpoint.s_authoredTemplates"/> itself); this file pins the shape only this
+/// family owns: the RESOURCE_SEMAPHORE gate on the two memory-grant reads, and the wait-type binding on
+/// <c>get_wait_trend</c>.
+/// </summary>
+public sealed class AlertNotebookTemplatePoisonWaitTests
+{
+    private static readonly DateTime WindowEnd = new(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
+    private static readonly DateTime WindowStart = WindowEnd - TimeSpan.FromHours(24);
+    private const string AsOf = "2026-01-01T12:00:00Z";
+
+    private static DarlingAlertReader.AlertHistoryReadRow RowWithWaitType(string? waitType) =>
+        new(WindowEnd, 1, "SRV1", "Poison Wait", 1, 1, true, "email", null, false, null, false,
+            ContextJson: waitType is null
+                ? null
+                : AlertContextSerializer.Serialize(new AlertContext { WaitType = waitType }));
+
+    /* ═══════════════════════════ routing ═══════════════════════════ */
+
+    [Fact]
+    public void AuthoredTemplate_PoisonWait_RoutesToItsOwnTemplate()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait");
+
+        Assert.NotNull(template);
+        Assert.Equal("authored/poison-wait", template!.Value.Id);
+        Assert.Equal(AlertNotebookEndpoint.PoisonWaitTemplateVersion, template.Value.Version);
+    }
+
+    /// <summary>RED on dev: before this PR, "Poison Wait" has no registered row, so
+    /// <see cref="AlertNotebookEndpoint.AuthoredTemplate"/> returns null and this assertion fails.</summary>
+    [Fact]
+    public void AuthoredTemplate_PoisonWait_IsNotNull()
+    {
+        Assert.NotNull(AlertNotebookEndpoint.AuthoredTemplate("Poison Wait"));
+    }
+
+    /* ═══════════════════════════ cell list ═══════════════════════════ */
+
+    [Fact]
+    public void BuildPoisonWaitCells_ResourceSemaphore_IncludesBothSemaphoreReads()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+        var row = RowWithWaitType("RESOURCE_SEMAPHORE");
+
+        var cells = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null, row, "Unknown");
+
+        var reads = cells.OfType<JsonObject>()
+            .Where(c => (string?)c["type"] == "read")
+            .Select(c => (string)c["read"]!)
+            .ToArray();
+
+        Assert.Contains("get_resource_semaphore", reads);
+        Assert.Contains("get_memory_grants", reads);
+        Assert.Contains("get_wait_stats", reads);
+        Assert.Contains("get_pg_wait_stats", reads);
+        Assert.Contains("get_waiting_tasks", reads);
+        Assert.Contains("get_wait_trend", reads);
+    }
+
+    /// <summary>Case-insensitive ordinal match.</summary>
+    [Fact]
+    public void BuildPoisonWaitCells_ResourceSemaphoreLowercase_StillIncludesBothSemaphoreReads()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+        var row = RowWithWaitType("resource_semaphore");
+
+        var cells = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null, row, "Unknown");
+
+        var reads = cells.OfType<JsonObject>()
+            .Where(c => (string?)c["type"] == "read")
+            .Select(c => (string)c["read"]!)
+            .ToArray();
+
+        Assert.Contains("get_resource_semaphore", reads);
+        Assert.Contains("get_memory_grants", reads);
+    }
+
+    [Theory]
+    [InlineData("THREADPOOL")]
+    [InlineData("RESOURCE_SEMAPHORE_QUERY_COMPILE")]
+    public void BuildPoisonWaitCells_OtherWaitType_ExcludesBothSemaphoreReads(string waitType)
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+        var row = RowWithWaitType(waitType);
+
+        var cells = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null, row, "Unknown");
+
+        var reads = cells.OfType<JsonObject>()
+            .Where(c => (string?)c["type"] == "read")
+            .Select(c => (string)c["read"]!)
+            .ToArray();
+
+        Assert.DoesNotContain("get_resource_semaphore", reads);
+        Assert.DoesNotContain("get_memory_grants", reads);
+    }
+
+    [Fact]
+    public void BuildPoisonWaitCells_WaitTrend_WaitTypeParamEqualsTheIncidentsExactly()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+        var row = RowWithWaitType("THREADPOOL");
+
+        var cells = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null, row, "Unknown");
+
+        var trend = cells.OfType<JsonObject>()
+            .Single(c => (string?)c["type"] == "read" && (string?)c["read"] == "get_wait_trend");
+        var parameters = Assert.IsType<JsonObject>(trend["params"]);
+
+        Assert.Equal("THREADPOOL", (string?)parameters["wait_type"]);
+    }
+
+    [Fact]
+    public void BuildPoisonWaitCells_NoWaitType_OmitsTrend_AddsANoteInstead()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+
+        var cells = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null, null, "Unknown");
+
+        var reads = cells.OfType<JsonObject>()
+            .Where(c => (string?)c["type"] == "read")
+            .Select(c => (string)c["read"]!)
+            .ToArray();
+
+        Assert.DoesNotContain("get_wait_trend", reads);
+        Assert.Contains(cells.OfType<JsonObject>(), c => (string?)c["type"] == "markdown");
+    }
+
+    /* ═══════════════════════════ degrade ═══════════════════════════ */
+
+    [Fact]
+    public void BuildPoisonWaitCells_NoMatchedIncidentOrRow_DegradesWithTheSameCellCount()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+
+        var withRow = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null,
+            RowWithWaitType("THREADPOOL"), "Unknown");
+        var withoutRow = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null, null, "Unknown");
+
+        /* Both degrade to "no wait type" cell counts identically -- the with-row case here carries a wait
+           type (adds the trend read), the without-row case adds the note markdown cell instead: same count,
+           different content. */
+        Assert.Equal(withRow.Count, withoutRow.Count);
+    }
+
+    /* ═══════════════════════════ validates ═══════════════════════════ */
+
+    [Fact]
+    public void BuildPoisonWaitCells_PassesValidateNotebookDefinition()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+        var cells = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null,
+            RowWithWaitType("RESOURCE_SEMAPHORE"), "Unknown");
+
+        var definition = new JsonObject { ["kind"] = "notebook", ["cells"] = cells };
+        var validation = DarlingWebEndpoints.ValidateNotebookDefinition(definition);
+
+        Assert.True(validation.IsValid, validation.Error);
+    }
+
+    [Fact]
+    public void BuildPoisonWaitCells_NoWaitType_AlsoPassesValidateNotebookDefinition()
+    {
+        var template = AlertNotebookEndpoint.AuthoredTemplate("Poison Wait")!.Value;
+        var cells = template.BuildCells(
+            "Poison Wait", "SRV1", AsOf, WindowStart, WindowEnd, null, null, "Unknown");
+
+        var definition = new JsonObject { ["kind"] = "notebook", ["cells"] = cells };
+        var validation = DarlingWebEndpoints.ValidateNotebookDefinition(definition);
+
+        Assert.True(validation.IsValid, validation.Error);
+    }
+
+    /* ═══════════════════════════ step 1: the context builder pin ═══════════════════════════ */
+
+    [Fact]
+    public void PoisonWaitContext_SerializesAndRehydratesTheWaitType()
+    {
+        var context = new AlertContext { WaitType = "RESOURCE_SEMAPHORE" };
+
+        var json = AlertContextSerializer.Serialize(context);
+
+        Assert.True(AlertContextSerializer.TryDeserialize(json, out var rehydrated));
+        Assert.Equal("RESOURCE_SEMAPHORE", rehydrated.WaitType);
+        Assert.Equal("RESOURCE_SEMAPHORE", AlertContextSerializer.TryReadWaitType(json));
+    }
+
+    [Fact]
+    public void PoisonWaitContext_LegacyJsonWithNoWaitTypeMember_RehydratesNull()
+    {
+        const string legacyJson = "{\"Details\":[]}";
+
+        Assert.Null(AlertContextSerializer.TryReadWaitType(legacyJson));
+    }
+}

Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.Templates.PoisonWait.cs

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Text.Json.Nodes;
+using PerformanceMonitor.Darling.Analysis;
+using PerformanceMonitor.Darling.Service.Mcp;
+using PerformanceMonitor.Notifications;
+
+namespace PerformanceMonitor.Darling.Service;
+
+internal static partial class AlertNotebookEndpoint
+{
+    /// <summary>Poison Wait template version (#4223). Bumped only if this template's SHAPE changes.</summary>
+    internal const int PoisonWaitTemplateVersion = 1;
+
+    /// <summary>The one wait type this alert fires on, RESOURCE_SEMAPHORE (ordinal, case-insensitive) — the
+    /// only type whose remedy is "raise a memory grant/worker ceiling", so its two supporting reads
+    /// (<see cref="BuildPoisonWaitCells"/>'s memory-grant cells) are wasted panels on every other poison
+    /// type and are gated on this string rather than shown unconditionally.</summary>
+    internal const string ResourceSemaphoreWaitType = "RESOURCE_SEMAPHORE";
+
+    /// <summary>Poison Wait (#4223, both engines): header, status, <c>get_wait_stats</c> /
+    /// <c>get_pg_wait_stats</c> (each engine's own top-waits read, titled by engine so the wrong-engine read
+    /// answers "not applicable" on its own — the mechanical map's #2719 convention), <c>get_waiting_tasks</c>,
+    /// <c>get_resource_semaphore</c> + <c>get_memory_grants</c> ONLY when the firing wait type is
+    /// RESOURCE_SEMAPHORE, then <c>get_wait_trend</c> for the wait type the alert names — or, when no
+    /// incident/row carries a wait type, a note cell in its place rather than a trend call that would 400 on
+    /// a missing required <c>wait_type</c>.
+    ///
+    /// <para>The wait type rides the alert's context as structured data (#4223 step 1):
+    /// <see cref="AlertContext.WaitType"/>, set at both fire sites
+    /// (<c>AlertEngine.CheckPoisonWaitsAsync</c>'s worst-graded <see cref="PoisonWaitEvaluator.SqlServerFinding.WaitType"/>,
+    /// <c>DarlingWorker.EvaluatePgPoisonWaitAsync</c>'s <c>finding.Subject</c>) — the same value each engine's
+    /// mute context already keys on (<c>AlertMuteContext.WaitType</c>), so this template reuses the field
+    /// rather than re-deriving it from prose.</para>
+    /// </summary>
+    private static JsonArray BuildPoisonWaitCells(
+        string? metric, string? serverName, string? asOf, DateTime windowStart, DateTime windowEnd,
+        AlertIncident? incident, DarlingAlertReader.AlertHistoryReadRow? row, string status)
+    {
+        /* The wait type rides the ROW's persisted context (AlertContext.WaitType), not the incident —
+           AlertIncident carries no wait-type member; the two engines' Poison Wait fire sites set the
+           context-level field directly (see this method's doc comment). */
+        var waitType = row?.ContextJson is string contextJson
+            ? AlertContextSerializer.TryReadWaitType(contextJson)
+            : null;
+
+        var cells = new JsonArray
+        {
+            HeaderCell(metric, serverName, incident, row),
+            StatusCell(status),
+            AuthoredReadCell("get_wait_stats", "Top waits (SQL Server)", serverName, asOf,
+                ("hours", "24"), ("limit", "20")),
+            AuthoredReadCell("get_pg_wait_stats", "Top waits (PostgreSQL)", serverName, asOf,
+                ("hours", "24"), ("limit", "20")),
+            AuthoredReadCell("get_waiting_tasks", "Waiting tasks", serverName, asOf,
+                ("hours", "1"), ("limit", "30")),
+        };
+
+        if (string.Equals(waitType, ResourceSemaphoreWaitType, StringComparison.OrdinalIgnoreCase))
+        {
+            cells.Add(AuthoredReadCell("get_resource_semaphore", "Resource semaphore", serverName, asOf,
+                ("hours", "24")));
+            cells.Add(AuthoredReadCell("get_memory_grants", "Memory grants", serverName, asOf,
+                ("hours", "1")));
+        }
+
+        /* A timeline on wait_stats/wait_time_ms, filtered to the firing wait type
+           when one is known -- built inline (not through TimelinePanel, which only knows a database_name
+           filter) rather than widening that shared helper for one family's dimension. */
+        var wallPanel = new JsonObject
+        {
+            ["type"] = "panel",
+            ["title"] = "Wait time over time",
+            ["source"] = "wait_stats",
+            ["measure"] = "wait_time_ms",
+            ["aggregate"] = "sum",
+            ["viz"] = "line",
+            ["timeBucket"] = "hour",
+            ["range"] = new JsonObject
+            {
+                ["windowStart"] = windowStart.ToString("o", System.Globalization.CultureInfo.InvariantCulture),
+                ["windowEnd"] = windowEnd.ToString("o", System.Globalization.CultureInfo.InvariantCulture),
+            },
+        };
+        if (!string.IsNullOrWhiteSpace(waitType))
+        {
+            wallPanel["filters"] = new JsonArray
+            {
+                new JsonObject
+                {
+                    ["dimension"] = "wait_type",
+                    ["op"] = "eq",
+                    ["value"] = waitType,
+                },
+            };
+        }
+        cells.Add(wallPanel);
+
+        if (string.IsNullOrWhiteSpace(waitType))
+        {
+            /* get_wait_trend REQUIRES wait_type (PReqText) — emitting the cell with none would 400. A
+               markdown cell is the notebook's own prose slot (ValidateNotebookDefinition's "markdown" arm),
+               so the degrade reads as a note rather than a missing panel. */
+            cells.Add(new JsonObject
+            {
+                ["type"] = "markdown",
+                ["text"] = "No matched incident carries a wait type, so the wait trend (which requires one) is omitted.",
+            });
+        }
+        else
+        {
+            cells.Add(AuthoredReadCell("get_wait_trend", "Wait trend", serverName, asOf,
+                ("hours", "24"), ("wait_type", waitType)));
+        }
+
+        return cells;
+    }
+}

Darling/PerformanceMonitor.Darling.Service/AlertNotebookEndpoint.cs

@@ -396,6 +396,8 @@ internal static readonly (string[] Metrics, AuthoredTemplateEntry Entry)[] s_aut
             new AuthoredTemplateEntry("authored/cpu", CpuTemplateVersion, BuildCpuCells)),
         (new[] { "Long-Running Query" },
             new AuthoredTemplateEntry("authored/long-running-query", LongRunningQueryTemplateVersion, BuildLongRunningQueryCells)),
+        (new[] { "Poison Wait" },
+            new AuthoredTemplateEntry("authored/poison-wait", PoisonWaitTemplateVersion, BuildPoisonWaitCells)),
         (new[] { "PostgreSQL Replication Slot Retention" },
             new AuthoredTemplateEntry("authored/pg-replication-slot", PgReplicationSlotTemplateVersion, BuildPgReplicationSlotCells)),
         (new[] { "PostgreSQL Vacuum Horizon Blocked" },
@@ -455,6 +457,12 @@ internal static bool IsDeclaredNoNotebook(string? metric) =>
     internal static readonly IReadOnlySet<string> s_authoredLimitlessTrendReads = new HashSet<string>(StringComparer.Ordinal)
     {
         "get_deadlock_trend",
+        /* #4223 Poison Wait: get_wait_trend has a bucket-count budget, not a row cap (PReqText("wait_type")
+           with no PLimit). get_memory_grants and get_resource_semaphore declare no limit param at all
+           (DarlingWebEndpoints.CatalogDescriptors). */
+        "get_wait_trend",
+        "get_memory_grants",
+        "get_resource_semaphore",
         /* #4223: get_top_queries_by_cpu / get_top_procedures_by_cpu / get_cpu_scheduler_pressure declare no
            'limit' param at all -- the first two cap with 'top' (carried explicitly below), the last has no
            row cap to carry. */

Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs

@@ -6211,6 +6211,10 @@ reasoning as the Tier 0 delivery loop above. */
                             {
                                 new(finding.Subject, new[] { finding.CurrentValue }),
                             },
+                            /* #4223: the wait type as structured data, the same string the mute context
+                               below already keys on (WaitType = finding.Subject) — an alert-notebook reader
+                               can branch on it without parsing the subject back out of the incident. */
+                            WaitType = finding.Subject,
                         },
                         DetailText: null,
                         finding.NumericCurrentValue,