Record web and composed-panel read durations into hourly latency histograms in the store (#4442)

Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs

@@ -965,7 +965,7 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
     /* 11th since #4215: ReadRejectedManagedConfSettingNamesAsync's catch — COUNTED,
        not exempt, because a RejectedValue row is judgeable evidence for the alert, not context for its text. */
     private const int WorkerCountedSites = 11;
-    private const int WorkerExemptSites = 10;
+    private const int WorkerExemptSites = 11;
 
     /// <summary>
     /// Counted sites tree-wide. ONE numeral with several readers rather than the same number written out at
@@ -1052,6 +1052,7 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
         ["CONVERTS the fault into the unreadable count"] = "a parse arm, not a read: the fleet-sweep rollup's store read is counted above it, and a document that does not parse becomes the rollup's own reportable unreadable count - the fault is evidence, not a swallow",
         ["Could not resolve Agent job names"] = "reads the monitored server's msdb through the host resolver, not the store - the Recently-failed-job precedent one seam over; the card degrades to the unresolved form whose raw marker keeps the gap visible, and the page still delivers",
         ["delivery stamp could not be written"] = "a write (#3580): the daily document was already delivered and process memory already gates it; the dropped stamp costs one re-announcement at the next restart and never a delivery - the stamp READ beside it is the read, and it is counted",
+        ["Read-latency flush failed"] = "the read-latency histogram flush writes instrumentation; losing an hour's rows can't hide an alert condition",
     };
 
     /// <summary>
@@ -1232,8 +1233,9 @@ WebTlsCertificateState report the web host publishes - there is no store read to
            ReadRejectedManagedConfSettingNamesAsync's catch is NOT here: a RejectedValue row is
            judgeable evidence, so it moved to the counted census above instead. 31st since #4299: the
            Raw Purge Over Horizon self-alert's catch, whose evidence (the readings) is handed in as a
-           parameter, with the read counted in DarlingWorker. */
-        Assert.Equal(31, totalExempt);
+           parameter, with the read counted in DarlingWorker. 32nd since #4442: the read-latency histogram
+           flush's catch, a telemetry write whose loss costs an hour's histogram rows and never an alert. */
+        Assert.Equal(32, totalExempt);
 
         /* Every exemption in the table is actually used. An exemption for a message that no longer exists
            is a hole this pin would otherwise keep open indefinitely — the shape that lets a real new catch

Darling/Darling.Tests/ComposeStatementTimeoutV147MigrationLiveTests.cs

@@ -25,6 +25,9 @@ namespace Darling.Tests;
 /// the same statement <see cref="DarlingManagedRoles.ReassertComposeStatementTimeoutAsync"/> runs on a
 /// control-plane reload), <c>pg_db_role_setting</c> shows the new ceiling on both <c>viewer</c> and <c>mcp</c>
 /// after the upgrade.
+///
+/// <para>This file's "I am the top rung" claim moved on to <c>ReadLatencyFlushLiveTests</c> (V148, #4442
+/// scope 2) the moment that rung landed.</para>
 /// </summary>
 /* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each fact mints its own scratch database
    through ScratchPostgres and never touches the shared store's tables, so it cannot race the live collection
@@ -34,63 +37,75 @@ public sealed class ComposeStatementTimeoutV147MigrationLiveTests
     private const int RungVersion = 147;
     private const int PreviousVersion = 146;
 
-    /// <summary>This rung's sentinel ordinal in the viewer probe — the newest, so the last argument.</summary>
+    /// <summary>This rung's sentinel ordinal in the viewer probe. No longer the last argument — V148
+    /// appended its own — so the invariant that outlives the handoff is that the ordinal is FIXED: a later
+    /// rung appends after it and never shifts it.</summary>
     private const int ProbeOrdinal = 122;
 
     private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");
 
     /// <summary>
-    /// The rung is registered and is the new top of the ladder — the claim this class takes over from
-    /// <c>ManagedConfVerdictsRungTests</c> (V146) now that V147 has landed.
+    /// The rung is registered — the claim this class took over from
+    /// <c>ManagedConfVerdictsRungTests</c> (V146) when V147 landed, and handed on to
+    /// <c>ReadLatencyFlushLiveTests</c> (V148) when that one did.
     /// </summary>
     [Fact]
-    public void TheRungIsRegisteredAtTheTopOfADenseLadder()
+    public void TheRungIsRegisteredInADenseLadder()
     {
         var versions = PgMigrations.Scripts.Select(s => s.Version).ToList();
 
         Assert.Equal("compose-statement-timeout-sixty", PgMigrations.Scripts.Single(s => s.Version == RungVersion).Name);
         Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version);
         Assert.Equal(StorageVersion.SchemaVersion, versions.Max());
-        Assert.Equal(RungVersion, StorageVersion.SchemaVersion);
+        /* One below the top since V148 landed; the "RungVersion == StorageVersion.SchemaVersion" half of
+           the top-arm claim moved to ReadLatencyFlushLiveTests with the top. */
+        Assert.True(RungVersion < StorageVersion.SchemaVersion, "V147 is expected to sit below the ladder's top now that V148 has landed");
         Assert.Equal(versions.Distinct().OrderBy(v => v), versions);
     }
 
     /// <summary>
-    /// The viewer probe's sentinel carries this rung, and the map treats it as the TOP arm: a missing top arm
-    /// maps a fully-migrated store one rung short, permanently, because <see cref="ViewerDataService.RequiredStoreSchemaVersion"/>
-    /// is <see cref="StorageVersion.SchemaVersion"/>.
+    /// The viewer probe's sentinel carries this rung, and the map's arm for it returns 147 — a position
+    /// within the signature, not its end, now that V148 has appended its own.
     /// </summary>
     [Fact]
-    public void TheProbeMapsAFullyMigratedStoreToThisTopRung()
+    public void TheProbeMapsAStoreStoppedHereToThisRung()
     {
         Assert.Contains(
             "column_name = 'compose_statement_timeout_seconds' AND column_default = '60'",
             ViewerDataService.StoreSchemaProbeSql.Replace("\r\n", "\n", StringComparison.Ordinal), StringComparison.Ordinal);
 
         var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs");
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal);
-        Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal);
 
         Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion);
 
         var method = typeof(ViewerDataService).GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!;
         var arity = method.GetParameters().Length;
-        Assert.Equal(ProbeOrdinal, arity - 1);
+
+        /* A position within the signature, not its end: "ProbeOrdinal == arity - 1" asserted this rung was
+           the NEWEST sentinel, which stopped being true the moment V148 appended its own. */
+        Assert.True(ProbeOrdinal < arity - 1);
         Assert.Equal("hasComposeTimeoutSixty", method.GetParameters()[ProbeOrdinal].Name);
 
-        var all = Enumerable.Repeat((object)true, arity).ToArray();
-        Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!);
+        var atThisRung = Enumerable.Range(0, arity).Select(i => (object)(i <= ProbeOrdinal)).ToArray();
+        Assert.Equal(RungVersion, (int)method.Invoke(null, atThisRung)!);
 
-        var behind = (object[])all.Clone();
+        var behind = (object[])atThisRung.Clone();
         behind[ProbeOrdinal] = false;
         Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!);
 
+        /* In the source, this rung's arm sits ABOVE the previous rung's and BELOW V148's, and returns this
+           rung's own version. */
         var thisArm = viewer.IndexOf("if (hasComposeTimeoutSixty)", StringComparison.Ordinal);
         var previousArm = viewer.IndexOf("if (hasManagedConfVerdicts)", StringComparison.Ordinal);
+        var nextArm = viewer.IndexOf("if (hasReadLatency)", StringComparison.Ordinal);
         Assert.True(thisArm >= 0, "the viewer has no V147 sentinel arm — a fully-migrated store would map one rung low");
+        Assert.True(previousArm >= 0, "the previous rung's arm is gone, so this pin is comparing against nothing");
+        Assert.True(nextArm >= 0, "the V148 arm is gone, so the handoff this file claims never happened");
         Assert.True(thisArm < previousArm, "the V147 arm sits below V146's, so a current store maps one rung low");
+        Assert.True(nextArm < thisArm, "the V148 arm sits below the V147 arm, so a current V148 store maps one rung low");
         Assert.Contains(
-            "return " + StorageVersion.SchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture) + ";",
+            "return " + RungVersion.ToString(System.Globalization.CultureInfo.InvariantCulture) + ";",
             viewer[thisArm..previousArm], StringComparison.Ordinal);
     }
 

Darling/Darling.Tests/FleetSweepWebFeedTests.cs

@@ -627,7 +627,7 @@ public void TheSweepRoutes_LogThroughTheServiceLogger_NotTheProviderlessAppFacto
             "Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingWebHostService.cs");
         Assert.Contains("builder.Logging.ClearProviders();", host, StringComparison.Ordinal);
         Assert.Contains(
-            "DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig);",
+            "DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig, _readLatency);",
             host, StringComparison.Ordinal);
     }
 

Darling/Darling.Tests/ReadLatencyAccumulatorTests.cs

@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System.Linq;
+using System.Threading.Tasks;
+using PerformanceMonitor.Darling.Service;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// <summary>#4442 scope 2: the read-latency histogram accumulator — the <c>CollectorCostAccumulator</c> shape
+/// applied to <c>(surface, route, outcome)</c> keys, unit-tested the same way that accumulator is: concurrency,
+/// drain-resets, and bucket-edge placement.</summary>
+public sealed class ReadLatencyAccumulatorTests
+{
+    [Fact]
+    public async Task Record_ManyConcurrentThreads_DrainTotalsMatch()
+    {
+        var accumulator = new ReadLatencyAccumulator();
+        const int threads = 16;
+        const int perThread = 200;
+
+        var tasks = Enumerable.Range(0, threads).Select(t => Task.Run(() =>
+        {
+            for (var i = 0; i < perThread; i++)
+            {
+                accumulator.Record(ReadSurface.Web, "get_wait_stats", ReadOutcome.Ok, 50 + (t + i) % 40);
+            }
+        }));
+
+        await Task.WhenAll(tasks);
+
+        var drained = accumulator.Drain();
+        var row = Assert.Single(drained);
+        Assert.Equal(threads * perThread, row.Count);
+        Assert.Equal(threads * perThread, row.BucketCounts.Sum());
+    }
+
+    [Fact]
+    public void Drain_ResetsTheWindow_SoASecondDrainIsEmpty()
+    {
+        var accumulator = new ReadLatencyAccumulator();
+        accumulator.Record(ReadSurface.Web, "get_blocking", ReadOutcome.Ok, 42);
+
+        var first = accumulator.Drain();
+        Assert.Single(first);
+
+        var second = accumulator.Drain();
+        Assert.Empty(second);
+    }
+
+    [Fact]
+    public void Record_QuietWindow_DrainsNothing()
+    {
+        var accumulator = new ReadLatencyAccumulator();
+        Assert.Empty(accumulator.Drain());
+    }
+
+    [Fact]
+    public void BucketIndexFor_ExactlyOnABound_LandsInThatBoundsBucket_NotTheNextOneUp()
+    {
+        var tenMsIndex = System.Array.IndexOf(ReadLatencyAccumulator.BucketUpperBoundsMs, 10);
+        Assert.Equal(tenMsIndex, ReadLatencyAccumulator.BucketIndexFor(10));
+
+        var oneSecondIndex = System.Array.IndexOf(ReadLatencyAccumulator.BucketUpperBoundsMs, 1_000);
+        Assert.Equal(oneSecondIndex, ReadLatencyAccumulator.BucketIndexFor(1_000));
+    }
+
+    [Fact]
+    public void BucketIndexFor_AboveTheLastBound_LandsInTheOverflowBucket()
+    {
+        var lastBound = ReadLatencyAccumulator.BucketUpperBoundsMs[^1];
+        var overflowIndex = ReadLatencyAccumulator.BucketUpperBoundsMs.Length;
+
+        Assert.Equal(overflowIndex, ReadLatencyAccumulator.BucketIndexFor(lastBound + 1));
+        Assert.Equal(overflowIndex, ReadLatencyAccumulator.BucketIndexFor(lastBound * 100));
+    }
+
+    [Fact]
+    public void Record_TracksCountTotalAndMax()
+    {
+        var accumulator = new ReadLatencyAccumulator();
+        accumulator.Record(ReadSurface.Compose, "compose:wait_stats", ReadOutcome.Ok, 10);
+        accumulator.Record(ReadSurface.Compose, "compose:wait_stats", ReadOutcome.Ok, 30);
+        accumulator.Record(ReadSurface.Compose, "compose:wait_stats", ReadOutcome.Ok, 5);
+
+        var row = Assert.Single(accumulator.Drain());
+        Assert.Equal(3, row.Count);
+        Assert.Equal(45, row.TotalMs);
+        Assert.Equal(30, row.MaxMs);
+    }
+
+    [Fact]
+    public void Record_DifferentOutcomes_AreSeparateBuckets()
+    {
+        var accumulator = new ReadLatencyAccumulator();
+        accumulator.Record(ReadSurface.Web, "get_plan_capture_readiness", ReadOutcome.Ok, 20);
+        accumulator.Record(ReadSurface.Web, "get_plan_capture_readiness", ReadOutcome.Timeout, 60_000);
+
+        var drained = accumulator.Drain();
+        Assert.Equal(2, drained.Count);
+        Assert.Contains(drained, r => r.Outcome == ReadOutcome.Ok && r.Count == 1);
+        Assert.Contains(drained, r => r.Outcome == ReadOutcome.Timeout && r.Count == 1);
+    }
+}