Lite investigation requests journal a quiet join and wait for every metric to resolve (#6010)

Darling/Darling.Tests/AlertDelivererInventoryPinTests.cs

@@ -70,7 +70,7 @@ public sealed class AlertDelivererInventoryPinTests
     /// text only and are not entry points.
     /// </summary>
     internal static readonly Regex ChannelCall = new(
-        @"[.]\s*(?:TrySendAlertEmailAsync|TrySendAsync|TrySendWebhookAlertsAsync|PostWebhookAsync|RecordAlertAsync)\s*\(\s*(?!(?:DarlingSelfAlertEvaluator[.])?BuildResolutionRecord)",
+        @"[.]\s*(?:TrySendAlertEmailAsync|TrySendAsync|TrySendWebhookAlertsAsync|PostWebhookAsync|RecordAlertAsync)\s*\(\s*(?!(?:(?:DarlingSelfAlertEvaluator|AlertResolutionRecords)[.])?BuildResolutionRecord)",
         RegexOptions.CultureInvariant, TimeSpan.FromSeconds(5));
 
     private static IEnumerable<(string Relative, string Text)> Sources()

Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs

@@ -928,7 +928,7 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
            catch - the delivery self-alert's report is a parameter, with the count read counted in DarlingWorker. 20th exempt since
            #5867: EvaluateComposeHoursStuckAsync's wrapper catch - the compose rollup self-alert's failure record is a parameter,
            with the read exempt in DarlingWorker (it holds the alert on a failed read). */
-        (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 13, 21),
+        (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 13, 20),
     };
 
     /// <summary>
@@ -1057,7 +1057,6 @@ holds the standing alert on a failed read. */
         ["The deadlock rollup failed for {Server}: {Message}"] = "#6005: the daily signature rollup, a write (and its own bookkeeping read) after the deadlock read it counts, which is counted where it is made; the alert is judged on that read, not on this",
         ["Could not persist the {Metric} persistence gate"] = "a write (#3282; one save path for every gated built-in since #3653 A5 put tempdb Space behind the same gate as High CPU); the gate has already decided the observation from its in-memory record, so a dropped save costs the streak across a restart and never an alert - the seeding LOADs beside it are the reads, and they are counted",
         ["Alert resolution callback failed"] = "the delivery path",
-        ["A resolution context that is not JSON is replaced"] = "#6003: parses the caller's context text for the Resolves member, not a store read",
         ["Connection-change self-alert delivery failed"] = "the delivery path",
         ["Store disk-pressure self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker",
         ["Custom-alert rule-health self-alert failed"] = "handed its evidence (the report) as a parameter; the report-building read is in CustomAlertEvaluator, outside this census",
@@ -1291,10 +1290,9 @@ WebTlsCertificateState report the web host publishes - there is no store read to
            maintenance write no alert is judged on, redone by the next pass. 40th since #5867: the Compose Rollup Hours Stuck
            self-alert's wrapper catch, handed its failure record as a parameter. 41st since #5867: the worker's read of that
            record, which holds the standing alert on a failure. 42nd since #5910: the engine's threshold-override
-           fallback, an in-memory lookup and not a read. 43rd since #6003: the resolution context parse, the caller's text and
-           not a store read. 44th since #6005: the deadlock rollup's catch, a write after the deadlock read, which counts its own
+           fallback, an in-memory lookup and not a read. 43rd since #6005: the deadlock rollup's catch, a write after the deadlock read, which counts its own
            failure. */
-        Assert.Equal(44, totalExempt);
+        Assert.Equal(43, totalExempt);
 
         /* Every exemption in the table is actually used. An exemption for a message that no longer exists
            is a hole this pin would otherwise keep open indefinitely — the shape that lets a real new catch

Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs

@@ -8789,43 +8789,15 @@ FROM ag_database_replica_states
     /// of resolutions. The pairing this method exists for is unaffected: the row is still written, still
     /// carries the resolution title, and still reads as resolved.</para>
     /// </summary>
-    public static AlertHistoryRecord BuildResolutionRecord(AlertResolution resolution, string? contextJson = null) => new(
-        resolution.ServerKey, resolution.ServerName, resolution.Title,
-        CurrentValueText: "resolved", ThresholdValueText: "",
-        NumericCurrentValue: null, NumericThresholdValue: null,
-        Delivery: AlertDelivery.NoChannelApplies(),
-        Muted: false, DetailText: SensitiveStatements.Text(resolution.Message),
-        ContextJson: ResolutionContextJson(contextJson, resolution.MetricName));
+    public static AlertHistoryRecord BuildResolutionRecord(AlertResolution resolution, string? contextJson = null) =>
+        AlertResolutionRecords.BuildResolutionRecord(resolution, contextJson);
 
-    /// <summary>The member of a resolution row's <c>context_json</c> that names the metric the row resolves (#6003).</summary>
-    public const string ResolvesMember = "Resolves";
+    /// <summary>The member of a resolution row's <c>context_json</c> that names the metric the row resolves (#6003); shared with Lite (#6010).</summary>
+    public const string ResolvesMember = AlertResolutionRecords.ResolvesMember;
 
-    /// <summary>
-    /// A resolution row's <c>context_json</c> (#6003, D7): the caller's context, when it gave one, plus <see cref="ResolvesMember"/>
-    /// naming the FIRING metric. The row's <c>metric_name</c> is the resolution's title ("CPU Resolved"), not the metric that
-    /// fired ("High CPU"), so without this member nothing in the store ties a clear back to the alert it ends, and the
-    /// investigation <c>resolved</c> detector reads exactly this. A context that is not a JSON object is replaced: no writer
-    /// passes one.
-    /// </summary>
-    public static string ResolutionContextJson(string? contextJson, string metricName)
-    {
-        System.Text.Json.Nodes.JsonObject node;
-        try
-        {
-            node = string.IsNullOrWhiteSpace(contextJson)
-                ? new System.Text.Json.Nodes.JsonObject()
-                : System.Text.Json.Nodes.JsonNode.Parse(contextJson) as System.Text.Json.Nodes.JsonObject
-                  ?? new System.Text.Json.Nodes.JsonObject();
-        }
-        catch (JsonException)
-        {
-            /* A resolution context that is not JSON is replaced: the caller's text, not a store read. */
-            node = new System.Text.Json.Nodes.JsonObject();
-        }
-
-        node[ResolvesMember] = metricName;
-        return node.ToJsonString();
-    }
+    /// <summary>A resolution row's <c>context_json</c> (#6003, D7); the one implementation is <see cref="AlertResolutionRecords.ResolutionContextJson"/>, shared with Lite.</summary>
+    public static string ResolutionContextJson(string? contextJson, string metricName) =>
+        AlertResolutionRecords.ResolutionContextJson(contextJson, metricName);
 
     /// <summary>
     /// The explicit "this metric has no measurement" value for the history stores' NOT NULL

Lite.Tests/DuckDbInvestigationStoreTests.cs

@@ -116,7 +116,8 @@ public async Task AlertsGroup_WithinTheWindow_AndAKnownKeyPastItOpensAnother()
         Assert.Equal("2|3", Scalar($@"
 SELECT concat_ws('|', alert_count, (SELECT count(*) FROM investigation_request_alerts WHERE request_id = '{first.RequestId}'))
 FROM investigation_requests WHERE request_id = '{first.RequestId}'"));
-        Assert.Equal("2", Scalar("SELECT count(*) FROM investigation_journal WHERE event = 'joined'"));
+        /* #6010: every alert here is the same metric at the default severity, so both joins are quiet. */
+        Assert.Equal("2", Scalar("SELECT count(*) FROM investigation_journal WHERE event = 'joined_quiet'"));
         /* Every alert row has its own id: three on the first request, one each on the other three opens and the last join. */
         Assert.Equal("6|6", Scalar("SELECT concat_ws('|', count(*), count(DISTINCT request_alert_id)) FROM investigation_request_alerts"));
     }

Lite.Tests/LiteAlertForwardingTests.ResolutionRow.cs

@@ -0,0 +1,200 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Globalization;
+using System.IO;
+using System.Text.Json.Nodes;
+using System.Threading;
+using System.Threading.Tasks;
+using PerformanceMonitor.Alerting;
+using PerformanceMonitor.Common;
+using PerformanceMonitor.Notifications;
+using PerformanceMonitorLite.Database;
+using PerformanceMonitorLite.Services;
+using Xunit;
+
+namespace PerformanceMonitorLite.Tests;
+
+/// <summary>
+/// #6010 part 2: Lite records a resolution history row, in Darling's shape, when the engine reports an alert cleared, so the
+/// investigation sweep can journal <c>resolved</c>. Every reader of <c>config_alert_log</c> is checked here against that row.
+/// </summary>
+public partial class LiteAlertForwardingTests
+{
+    private static string TempDb(out DuckDbInitializer db)
+    {
+        var dir = Path.Combine(Path.GetTempPath(), "LiteResolutionRow_" + Guid.NewGuid().ToString("N")[..8]);
+        Directory.CreateDirectory(dir);
+        db = new DuckDbInitializer(Path.Combine(dir, "lite.duckdb"));
+        db.InitializeAsync().GetAwaiter().GetResult();
+        return dir;
+    }
+
+    private static string? Scalar(DuckDbInitializer db, string sql)
+    {
+        using var connection = db.CreateConnection();
+        connection.Open();
+        using var command = connection.CreateCommand();
+        command.CommandText = sql;
+        var value = command.ExecuteScalar();
+        return value is null or DBNull ? null : Convert.ToString(value, CultureInfo.InvariantCulture);
+    }
+
+    private static void TearDown(DuckDbInitializer db, string dir)
+    {
+        db.Dispose();
+        try { Directory.Delete(dir, recursive: true); } catch (IOException) { /* best effort */ }
+    }
+
+    [Fact]
+    public async Task TheResolutionCallback_WritesOneHistoryRow_NamingTheMetricItResolves()
+    {
+        var dir = TempDb(out var db);
+        try
+        {
+            var store = new DuckDbAlertHistoryStore(db);
+            var resolution = new AlertResolution(Key, Name, "High CPU", "CPU Resolved", "SRV-A: Total CPU back to 12%");
+            var after = 0;
+            var callback = LiteAlertResolutionRecorder.Callback(store, (_, _) => { after++; return Task.CompletedTask; });
+
+            await callback(resolution, CancellationToken.None);
+
+            Assert.Equal(1, after);
+            Assert.Equal("1", Scalar(db, "SELECT count(*) FROM config_alert_log"));
+            Assert.Equal("CPU Resolved", Scalar(db, "SELECT metric_name FROM config_alert_log"));
+            Assert.Equal("none", Scalar(db, "SELECT notification_type FROM config_alert_log"));
+            Assert.Equal("false", Scalar(db, "SELECT lower(CAST(alert_sent AS VARCHAR)) FROM config_alert_log"));
+            Assert.Equal("false", Scalar(db, "SELECT lower(CAST(muted AS VARCHAR)) FROM config_alert_log"));
+            Assert.Equal("SRV-A: Total CPU back to 12%", Scalar(db, "SELECT detail_text FROM config_alert_log"));
+            var context = JsonNode.Parse(Scalar(db, "SELECT context_json FROM config_alert_log")!)!;
+            Assert.Equal("High CPU", (string)context[AlertResolutionRecords.ResolvesMember]!);
+        }
+        finally { TearDown(db, dir); }
+    }
+
+    [Fact]
+    public async Task AFailedHistoryWrite_NeverCostsTheToastHalf()
+    {
+        var after = 0;
+        var callback = LiteAlertResolutionRecorder.Callback(new ThrowingHistoryStore(), (_, _) => { after++; return Task.CompletedTask; });
+
+        await callback(new AlertResolution(Key, Name, "High CPU", "CPU Resolved", "m"), CancellationToken.None);
+
+        Assert.Equal(1, after);
+    }
+
+    private sealed class ThrowingHistoryStore : IAlertHistoryStore
+    {
+        public Task RecordAlertAsync(AlertHistoryRecord record) => throw new InvalidOperationException("store down");
+        public Task<DateTime?> GetLastEmailSentUtcAsync(string serverId, string metricName, string? dedupKey = null) => Task.FromResult<DateTime?>(null);
+        public Task<DateTime?> GetLastWebhookSentUtcAsync(string serverId, string metricName, string? dedupKey = null) => Task.FromResult<DateTime?>(null);
+        public Task<DateTime?> GetLastAlertTimeAsync(string serverId, string metricName, string? dedupKey = null) => Task.FromResult<DateTime?>(null);
+        public Task<DateTime?> GetLastDeliveredPageUtcAsync(string serverId, string metricName) => Task.FromResult<DateTime?>(null);
+    }
+
+    [Fact]
+    public async Task EndToEnd_AnAlertOpensARequest_ItClears_AndTheNextSweepJournalsResolvedOnce()
+    {
+        DisableAllChecks();
+        App.AlertCpuEnabled = true;
+        var dir = TempDb(out var db);
+        try
+        {
+            var history = new DuckDbAlertHistoryStore(db);
+            var investigations = new DuckDbInvestigationStore(db, () => DateTime.UtcNow, DuckDbInvestigationStoreTests.TestLockBudget);
+            var h = new Harness { ResolutionHook = LiteAlertResolutionRecorder.Callback(history, (_, _) => Task.CompletedTask) };
+            var engine = h.Build();
+
+            var at = await DriveCpuAsync(engine, sqlCpu: 70, totalCpu: 92, samples: AlertEngine.CpuBreachSamples, from: Harness.SampleBase);
+            Assert.Single(h.Deliverer.Outcomes);
+
+            /* The fire opens a request for the same server and metric the engine named. */
+            var alert = new InvestigationAlert(1, int.Parse(Key, CultureInfo.InvariantCulture), Name, "resources", "High CPU",
+                DateTime.UtcNow.AddMinutes(-5), new[] { "k-cpu" }, null, 20, 100, LeaseMinutes: 15, ExpiryMinutes: 240);
+            var opened = await investigations.OpenOrJoinAsync(alert, TestContext.Current.CancellationToken);
+            Assert.Equal(InvestigationOpenOutcome.Opened, opened.Outcome);
+
+            Assert.Equal(0, (await investigations.SweepAsync(TestContext.Current.CancellationToken)).Resolved);
+
+            h.Now = h.Now.AddMinutes(6);
+            await DriveCpuAsync(engine, sqlCpu: 10, totalCpu: 12, samples: AlertEngine.CpuClearSamples, from: at);
+            Assert.Single(h.Resolutions);
+
+            Assert.Equal(1, (await investigations.SweepAsync(TestContext.Current.CancellationToken)).Resolved);
+            Assert.Equal(0, (await investigations.SweepAsync(TestContext.Current.CancellationToken)).Resolved);
+            Assert.Equal("1", Scalar(db, "SELECT count(*) FROM investigation_journal WHERE event = 'resolved'"));
+        }
+        finally { TearDown(db, dir); }
+    }
+
+    [Fact]
+    public async Task EveryReaderOfTheAlertLog_ReadsAResolutionRowAsAClearAndNotAsAFire()
+    {
+        var dir = TempDb(out var db);
+        try
+        {
+            var history = new DuckDbAlertHistoryStore(db);
+            var since = DateTime.UtcNow.AddHours(-1);
+
+            /* A critical fire, then the clear: the clear is written by the real recorder, so it is the row the app writes. */
+            Exec(db, $"INSERT INTO config_alert_log (alert_time, server_id, server_name, metric_name, current_value, threshold_value, alert_sent, notification_type, muted, context_json) VALUES ('{DateTime.UtcNow.AddMinutes(-10):yyyy-MM-dd HH:mm:ss.ffffff}', {Key}, '{Name}', 'High CPU', 97, 80, true, 'email', false, '{{\"Severity\":\"Critical\"}}')");
+            Assert.Equal(AlertSeverityLevel.Critical, await history.GetLastFiredTierAsync(Key, "High CPU", "CPU Resolved", since));
+
+            await LiteAlertResolutionRecorder.RecordAsync(history, new AlertResolution(Key, Name, "High CPU", "CPU Resolved", "back to 12%"));
+
+            /* Escalation seed: the clear closes the fire, as it does in Darling's store. */
+            Assert.Null(await history.GetLastFiredTierAsync(Key, "High CPU", "CPU Resolved", since));
+            /* Cooldown and delivered-page seeds: no channel applies, so the clear is neither a send nor a delivered page under its own name. */
+            Assert.Null(await history.GetLastEmailSentUtcAsync(Key, "CPU Resolved"));
+            Assert.Null(await history.GetLastWebhookSentUtcAsync(Key, "CPU Resolved"));
+            Assert.Null(await history.GetLastDeliveredPageUtcAsync(Key, "CPU Resolved"));
+            Assert.NotNull(await history.GetLastEmailSentUtcAsync(Key, "High CPU"));
+
+            /* History tab and get_alert_history: the row is shown, and reads as resolved with no delivery claim. */
+            var rows = await new LocalDataService(db).GetAlertHistoryAsync(hoursBack: 24, limit: 50);
+            Assert.Equal(2, rows.Count);
+            var resolved = Assert.Single(rows, r => r.MetricName == "CPU Resolved");
+            Assert.True(resolved.IsResolved);
+            Assert.False(resolved.AlertSent);
+            Assert.Equal(AlertDeliveryStatus.NoChannel, resolved.StatusDisplay);
+        }
+        finally { TearDown(db, dir); }
+    }
+
+    [Fact]
+    public async Task TheDailySummaryAlertCount_CountsTheFire_AndNotItsResolutionRow()
+    {
+        var dir = TempDb(out var db);
+        try
+        {
+            /* A fixed past day, so the pin holds at any hour. */
+            var day = new DateTime(2026, 6, 15, 0, 0, 0, DateTimeKind.Utc);
+            var history = new DuckDbAlertHistoryStore(db);
+            Exec(db, $"INSERT INTO config_alert_log (alert_time, server_id, server_name, metric_name, current_value, threshold_value, alert_sent, notification_type, muted) VALUES ('{day.AddHours(3):yyyy-MM-dd HH:mm:ss.ffffff}', {Key}, '{Name}', 'High CPU', 97, 80, true, 'email', false)");
+            await history.RecordAlertAsync(AlertResolutionRecords.BuildResolutionRecord(
+                new AlertResolution(Key, Name, "High CPU", "CPU Resolved", "back")) with { AlertTimeUtc = day.AddHours(4) });
+            Assert.Equal("2", Scalar(db, "SELECT count(*) FROM config_alert_log"));
+
+            var summary = await new LocalDataService(db).GetDailySummaryAsync(int.Parse(Key, CultureInfo.InvariantCulture), day);
+
+            Assert.NotNull(summary);
+            Assert.Equal(1L, summary!.AlertCount);
+        }
+        finally { TearDown(db, dir); }
+    }
+
+    private static void Exec(DuckDbInitializer db, string sql)
+    {
+        using var connection = db.CreateConnection();
+        connection.Open();
+        using var command = connection.CreateCommand();
+        command.CommandText = sql;
+        command.ExecuteNonQuery();
+    }
+}

Lite.Tests/LiteAlertForwardingTests.cs

@@ -278,6 +278,8 @@ private sealed class Harness
         public InMemoryStateStore StateStore { get; } = new();
         public RecordingDeliverer Deliverer { get; } = new();
         public List<AlertResolution> Resolutions { get; } = new();
+        /// <summary>#6010 part 2: runs after a resolution is noted, so a test can wire Lite's real history-row callback.</summary>
+        public Func<AlertResolution, CancellationToken, Task>? ResolutionHook { get; set; }
         public List<FailedJobInfo> FailedJobs { get; } = new();
         public bool Muted { get; set; }
         public DateTime Now { get; set; } = new(2026, 7, 1, 12, 0, 0, DateTimeKind.Utc);
@@ -291,7 +293,7 @@ private sealed class Harness
             Adapter, StateStore, deliverer,
             isAlertMuted: _ => Muted,
             failedJobsFetcher: (_, _, _) => Task.FromResult(new List<FailedJobInfo>(FailedJobs)),
-            resolutionCallback: (r, _) => { Resolutions.Add(r); return Task.CompletedTask; },
+            resolutionCallback: (r, ct) => { Resolutions.Add(r); return ResolutionHook?.Invoke(r, ct) ?? Task.CompletedTask; },
             logger: null,
             utcNow: () => Now);