From ad7420039eaa36b58f3f81f4d4ef8ce7967d0ce1 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 26 Sep 2026 17:33:12 -0400
Subject: [PATCH 1/6] Add a read-latency histogram accumulator, percentile
helper, and outcome classifier, and record web and compose reads (#4442)
---
.../Darling.Tests/FleetSweepWebFeedTests.cs | 2 +-
.../ReadLatencyAccumulatorTests.cs | 110 ++++++++++++
.../ReadLatencyPercentilesTests.cs | 120 +++++++++++++
.../ReadOutcomeClassifierTests.cs | 82 +++++++++
.../Darling.Tests/SharedBaselineCacheTests.cs | 2 +-
.../DarlingWebEndpoints.cs | 109 +++++++++++-
.../Mcp/DarlingWebHostService.cs | 10 +-
.../Program.cs | 5 +
.../ReadLatencyAccumulator.cs | 159 ++++++++++++++++++
.../ReadLatencyPercentiles.cs | 79 +++++++++
.../ReadOutcomeClassifier.cs | 68 ++++++++
11 files changed, 741 insertions(+), 5 deletions(-)
create mode 100644 Darling/Darling.Tests/ReadLatencyAccumulatorTests.cs
create mode 100644 Darling/Darling.Tests/ReadLatencyPercentilesTests.cs
create mode 100644 Darling/Darling.Tests/ReadOutcomeClassifierTests.cs
create mode 100644 Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs
create mode 100644 Darling/PerformanceMonitor.Darling.Service/ReadLatencyPercentiles.cs
create mode 100644 Darling/PerformanceMonitor.Darling.Service/ReadOutcomeClassifier.cs
diff --git a/Darling/Darling.Tests/FleetSweepWebFeedTests.cs b/Darling/Darling.Tests/FleetSweepWebFeedTests.cs
index 651347816a..efc91dc58a 100644
--- a/Darling/Darling.Tests/FleetSweepWebFeedTests.cs
+++ b/Darling/Darling.Tests/FleetSweepWebFeedTests.cs
@@ -627,7 +627,7 @@ public void TheSweepRoutes_LogThroughTheServiceLogger_NotTheProviderlessAppFacto
"Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingWebHostService.cs");
Assert.Contains("builder.Logging.ClearProviders();", host, StringComparison.Ordinal);
Assert.Contains(
- "DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig);",
+ "DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig, _readLatency);",
host, StringComparison.Ordinal);
}
diff --git a/Darling/Darling.Tests/ReadLatencyAccumulatorTests.cs b/Darling/Darling.Tests/ReadLatencyAccumulatorTests.cs
new file mode 100644
index 0000000000..1d91ae52ba
--- /dev/null
+++ b/Darling/Darling.Tests/ReadLatencyAccumulatorTests.cs
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System.Linq;
+using System.Threading.Tasks;
+using PerformanceMonitor.Darling.Service;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// #4442 scope 2: the read-latency histogram accumulator — the CollectorCostAccumulator shape
+/// applied to (surface, route, outcome) keys, unit-tested the same way that accumulator is: concurrency,
+/// drain-resets, and bucket-edge placement.
+public sealed class ReadLatencyAccumulatorTests
+{
+ [Fact]
+ public async Task Record_ManyConcurrentThreads_DrainTotalsMatch()
+ {
+ var accumulator = new ReadLatencyAccumulator();
+ const int threads = 16;
+ const int perThread = 200;
+
+ var tasks = Enumerable.Range(0, threads).Select(t => Task.Run(() =>
+ {
+ for (var i = 0; i < perThread; i++)
+ {
+ accumulator.Record(ReadSurface.Web, "get_wait_stats", ReadOutcome.Ok, 50 + (t + i) % 40);
+ }
+ }));
+
+ await Task.WhenAll(tasks);
+
+ var drained = accumulator.Drain();
+ var row = Assert.Single(drained);
+ Assert.Equal(threads * perThread, row.Count);
+ Assert.Equal(threads * perThread, row.BucketCounts.Sum());
+ }
+
+ [Fact]
+ public void Drain_ResetsTheWindow_SoASecondDrainIsEmpty()
+ {
+ var accumulator = new ReadLatencyAccumulator();
+ accumulator.Record(ReadSurface.Web, "get_blocking", ReadOutcome.Ok, 42);
+
+ var first = accumulator.Drain();
+ Assert.Single(first);
+
+ var second = accumulator.Drain();
+ Assert.Empty(second);
+ }
+
+ [Fact]
+ public void Record_QuietWindow_DrainsNothing()
+ {
+ var accumulator = new ReadLatencyAccumulator();
+ Assert.Empty(accumulator.Drain());
+ }
+
+ [Fact]
+ public void BucketIndexFor_ExactlyOnABound_LandsInThatBoundsBucket_NotTheNextOneUp()
+ {
+ var tenMsIndex = System.Array.IndexOf(ReadLatencyAccumulator.BucketUpperBoundsMs, 10);
+ Assert.Equal(tenMsIndex, ReadLatencyAccumulator.BucketIndexFor(10));
+
+ var oneSecondIndex = System.Array.IndexOf(ReadLatencyAccumulator.BucketUpperBoundsMs, 1_000);
+ Assert.Equal(oneSecondIndex, ReadLatencyAccumulator.BucketIndexFor(1_000));
+ }
+
+ [Fact]
+ public void BucketIndexFor_AboveTheLastBound_LandsInTheOverflowBucket()
+ {
+ var lastBound = ReadLatencyAccumulator.BucketUpperBoundsMs[^1];
+ var overflowIndex = ReadLatencyAccumulator.BucketUpperBoundsMs.Length;
+
+ Assert.Equal(overflowIndex, ReadLatencyAccumulator.BucketIndexFor(lastBound + 1));
+ Assert.Equal(overflowIndex, ReadLatencyAccumulator.BucketIndexFor(lastBound * 100));
+ }
+
+ [Fact]
+ public void Record_TracksCountTotalAndMax()
+ {
+ var accumulator = new ReadLatencyAccumulator();
+ accumulator.Record(ReadSurface.Compose, "compose:wait_stats", ReadOutcome.Ok, 10);
+ accumulator.Record(ReadSurface.Compose, "compose:wait_stats", ReadOutcome.Ok, 30);
+ accumulator.Record(ReadSurface.Compose, "compose:wait_stats", ReadOutcome.Ok, 5);
+
+ var row = Assert.Single(accumulator.Drain());
+ Assert.Equal(3, row.Count);
+ Assert.Equal(45, row.TotalMs);
+ Assert.Equal(30, row.MaxMs);
+ }
+
+ [Fact]
+ public void Record_DifferentOutcomes_AreSeparateBuckets()
+ {
+ var accumulator = new ReadLatencyAccumulator();
+ accumulator.Record(ReadSurface.Web, "get_plan_capture_readiness", ReadOutcome.Ok, 20);
+ accumulator.Record(ReadSurface.Web, "get_plan_capture_readiness", ReadOutcome.Timeout, 60_000);
+
+ var drained = accumulator.Drain();
+ Assert.Equal(2, drained.Count);
+ Assert.Contains(drained, r => r.Outcome == ReadOutcome.Ok && r.Count == 1);
+ Assert.Contains(drained, r => r.Outcome == ReadOutcome.Timeout && r.Count == 1);
+ }
+}
diff --git a/Darling/Darling.Tests/ReadLatencyPercentilesTests.cs b/Darling/Darling.Tests/ReadLatencyPercentilesTests.cs
new file mode 100644
index 0000000000..950474b2d7
--- /dev/null
+++ b/Darling/Darling.Tests/ReadLatencyPercentilesTests.cs
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using PerformanceMonitor.Darling.Service;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// #4442 scope 2: pure percentile math over 's fixed buckets.
+/// is the field-shaped fixture named
+/// in the PR body: 1,000 samples, 90% fast (50-200 ms), 9% slow (1-5 s), 1% very slow (20-60 s).
+public sealed class ReadLatencyPercentilesTests
+{
+ private static readonly int[] s_bounds = ReadLatencyAccumulator.BucketUpperBoundsMs;
+
+ private static long[] NewCounts() => new long[s_bounds.Length + 1];
+
+ private static void Add(long[] counts, long elapsedMs, int times = 1)
+ {
+ counts[ReadLatencyAccumulator.BucketIndexFor(elapsedMs)] += times;
+ }
+
+ [Fact]
+ public void FromBuckets_AllZero_ReturnsNull()
+ {
+ var counts = NewCounts();
+ Assert.Null(ReadLatencyPercentiles.FromBuckets(counts, s_bounds, 0.5));
+ }
+
+ [Fact]
+ public void FromBuckets_MismatchedLengths_Throws()
+ {
+ var counts = new long[s_bounds.Length]; // one short of the required overflow slot
+ Assert.Throws(() => ReadLatencyPercentiles.FromBuckets(counts, s_bounds, 0.5));
+ }
+
+ [Theory]
+ [InlineData(0.0)]
+ [InlineData(1.0)]
+ [InlineData(-0.1)]
+ [InlineData(1.5)]
+ public void FromBuckets_POutOfRange_Throws(double p)
+ {
+ var counts = NewCounts();
+ Add(counts, 50);
+ Assert.Throws(() => ReadLatencyPercentiles.FromBuckets(counts, s_bounds, p));
+ }
+
+ [Fact]
+ public void FromBuckets_OverflowBucket_IsFlaggedAtLeast()
+ {
+ var counts = NewCounts();
+ Add(counts, s_bounds[^1] + 5_000); // above every finite bound
+
+ var p50 = ReadLatencyPercentiles.FromBuckets(counts, s_bounds, 0.5)!.Value;
+ Assert.True(p50.IsAtLeast);
+ Assert.Equal(s_bounds[^1], p50.UpperBoundMs);
+ }
+
+ [Fact]
+ public void FromBuckets_AllInOneFiniteBucket_IsNotFlaggedAtLeast()
+ {
+ var counts = NewCounts();
+ Add(counts, 90);
+
+ var p50 = ReadLatencyPercentiles.FromBuckets(counts, s_bounds, 0.5)!.Value;
+ Assert.False(p50.IsAtLeast);
+ Assert.Equal(100, p50.UpperBoundMs);
+ }
+
+ /// The field-shaped fixture (named in the PR body): 1,000 samples — 900 uniformly spread 50-200
+ /// ms (fast reads), 90 uniformly spread 1,000-5,000 ms (the slow tail), 10 uniformly spread 20,000-60,000
+ /// ms (the rare very-slow reads a statement_timeout would eventually catch). p50 must land in the fast
+ /// band's buckets, p95 must have crossed into the slow band, and p99 must have crossed into the very-slow
+ /// band — each an upper-bound estimate, never lower than the true percentile.
+ [Fact]
+ public void Percentiles_OverAFieldShapedFixture_LandInTheExpectedBuckets()
+ {
+ var counts = NewCounts();
+ var rng = new Random(20260926);
+
+ for (var i = 0; i < 900; i++)
+ {
+ Add(counts, 50 + rng.Next(0, 151)); // 50-200 ms inclusive
+ }
+
+ for (var i = 0; i < 90; i++)
+ {
+ Add(counts, 1_000 + rng.Next(0, 4_001)); // 1,000-5,000 ms inclusive
+ }
+
+ for (var i = 0; i < 10; i++)
+ {
+ Add(counts, 20_000 + rng.Next(0, 40_001)); // 20,000-60,000 ms inclusive
+ }
+
+ var p50 = ReadLatencyPercentiles.FromBuckets(counts, s_bounds, 0.50)!.Value;
+ var p95 = ReadLatencyPercentiles.FromBuckets(counts, s_bounds, 0.95)!.Value;
+ var p99 = ReadLatencyPercentiles.FromBuckets(counts, s_bounds, 0.99)!.Value;
+
+ // p50 sits inside the 90% fast band: at or under 200 ms, and never flagged AtLeast (there is plenty
+ // of finite headroom above it).
+ Assert.False(p50.IsAtLeast);
+ Assert.True(p50.UpperBoundMs <= 200, $"p50 upper bound was {p50.UpperBoundMs} ms, expected <= 200 ms.");
+
+ // p95 (the 90th-to-99th count) has crossed out of the fast band into the slow tail: strictly above
+ // 200 ms, at or under 5,000 ms.
+ Assert.True(p95.UpperBoundMs > 200, $"p95 upper bound was {p95.UpperBoundMs} ms, expected > 200 ms.");
+ Assert.True(p95.UpperBoundMs <= 5_000, $"p95 upper bound was {p95.UpperBoundMs} ms, expected <= 5,000 ms.");
+
+ // p99 (the last 1%) has crossed into the very-slow band: strictly above 5,000 ms.
+ Assert.True(p99.UpperBoundMs > 5_000, $"p99 upper bound was {p99.UpperBoundMs} ms, expected > 5,000 ms.");
+ }
+}
diff --git a/Darling/Darling.Tests/ReadOutcomeClassifierTests.cs b/Darling/Darling.Tests/ReadOutcomeClassifierTests.cs
new file mode 100644
index 0000000000..b4fca7984f
--- /dev/null
+++ b/Darling/Darling.Tests/ReadOutcomeClassifierTests.cs
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Threading;
+using Npgsql;
+using PerformanceMonitor.Darling.Service;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// #4442 scope 2: reuses 's
+/// own SQLSTATE-plus-wording rule rather than re-deriving it — these pins are the same shapes
+/// CollectorFaultCancelOriginTests pins for that type, read through the read-latency classifier.
+public sealed class ReadOutcomeClassifierTests
+{
+ [Fact]
+ public void Classify_57014WithStatementTimeoutWording_IsTimeout()
+ {
+ var ex = new PostgresException("canceling statement due to statement timeout", "ERROR", "ERROR", "57014");
+ Assert.Equal(ReadOutcome.Timeout, ReadOutcomeClassifier.Classify(ex, CancellationToken.None));
+ }
+
+ [Fact]
+ public void Classify_57014WithUserRequestWording_IsCancelled()
+ {
+ var ex = new PostgresException("canceling statement due to user request", "ERROR", "ERROR", "57014");
+ Assert.Equal(ReadOutcome.Cancelled, ReadOutcomeClassifier.Classify(ex, CancellationToken.None));
+ }
+
+ [Fact]
+ public void Classify_RequestTokenCancelled_IsCancelled_EvenWithAnUnrelatedException()
+ {
+ using var cts = new CancellationTokenSource();
+ cts.Cancel();
+ var ex = new InvalidOperationException("boom");
+
+ Assert.Equal(ReadOutcome.Cancelled, ReadOutcomeClassifier.Classify(ex, cts.Token));
+ }
+
+ [Fact]
+ public void Classify_AnotherException_IsError()
+ {
+ var ex = new InvalidOperationException("boom");
+ Assert.Equal(ReadOutcome.Error, ReadOutcomeClassifier.Classify(ex, CancellationToken.None));
+ }
+
+ [Fact]
+ public void Classify_OtherPostgresException_IsError()
+ {
+ var ex = new PostgresException("relation \"x\" does not exist", "ERROR", "ERROR", "42P01");
+ Assert.Equal(ReadOutcome.Error, ReadOutcomeClassifier.Classify(ex, CancellationToken.None));
+ }
+
+ [Fact]
+ public void ClassifySentence_StatementTimeoutSentence_IsTimeout()
+ {
+ var sentence = "Error during get_wait_stats: 57014: canceling statement due to statement timeout";
+ Assert.Equal(ReadOutcome.Timeout, ReadOutcomeClassifier.ClassifySentence(sentence, CancellationToken.None));
+ }
+
+ [Fact]
+ public void ClassifySentence_RequestTokenCancelled_IsCancelled()
+ {
+ using var cts = new CancellationTokenSource();
+ cts.Cancel();
+ var sentence = "Error during get_wait_stats: 57014: canceling statement due to statement timeout";
+ Assert.Equal(ReadOutcome.Cancelled, ReadOutcomeClassifier.ClassifySentence(sentence, cts.Token));
+ }
+
+ [Fact]
+ public void ClassifySentence_OtherSentence_IsError()
+ {
+ var sentence = "Error during get_wait_stats: 42P01: relation \"x\" does not exist";
+ Assert.Equal(ReadOutcome.Error, ReadOutcomeClassifier.ClassifySentence(sentence, CancellationToken.None));
+ }
+}
diff --git a/Darling/Darling.Tests/SharedBaselineCacheTests.cs b/Darling/Darling.Tests/SharedBaselineCacheTests.cs
index bcd5f10526..e5968aeb84 100644
--- a/Darling/Darling.Tests/SharedBaselineCacheTests.cs
+++ b/Darling/Darling.Tests/SharedBaselineCacheTests.cs
@@ -142,7 +142,7 @@ public void TheService_HandsOneBaselineCacheToEveryHostsAnalysis()
Assert.Contains("new DarlingAnalysisService(postgres, planFetcher, _logger, _baselineCache)", mcp, StringComparison.Ordinal);
var web = Code("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingWebHostService.cs");
- Assert.Contains("DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig);", web, StringComparison.Ordinal);
+ Assert.Contains("DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig, _readLatency);", web, StringComparison.Ordinal);
var endpoints = Code("Darling", "PerformanceMonitor.Darling.Service", "DarlingWebEndpoints.cs");
Assert.Contains("new DarlingAnalysisService(postgres, logger: logger, baselineCache: baselineCache)", endpoints, StringComparison.Ordinal);
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
index e3049b3830..4102453812 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
@@ -67,6 +67,18 @@ namespace PerformanceMonitor.Darling.Service;
///
public static class DarlingWebEndpoints
{
+ /// #4442 scope 2: the process-lifetime read-latency accumulator, set once from 's
+ /// DI singleton. Static because is shared, unchanged, with the MCP
+ /// run_custom_view_panel tool -- a static field is the seam that lets this recording land without adding a
+ /// parameter to that shared, static method's signature (and touching its MCP call site, which is a later,
+ /// separate change). Null in any context that never calls (a unit test exercising
+ /// RunComposedPanelAsync directly), so recording is always optional, never required.
+ private static ReadLatencyAccumulator? s_readLatency;
+
+ /// The logger recording failures are reported through, at Debug -- never at a level an operator
+ /// would see, since a recording failure is never a request failure.
+ private static ILogger? s_readLatencyLogger;
+
/// The tool names deliberately absent from the /api/read/* 1:1 read surface. analyze_server
/// makes a live monitored-server connection; mute_analysis_finding writes; the analyze_*_plan family
/// is the compute-heavy plan-analysis phase-2 work; the Custom Views tools (#1599) are served by their OWN
@@ -261,8 +273,18 @@ on a state it has not reasoned about. */
/// and the MCP host's analysis fill — so compare_analysis' banding here reads a series the store was already asked
/// for this analysis hour from memory. Null keeps the analysis service's baselines private to it.
///
- public static void MapAll(WebApplication app, NpgsqlDataSource postgres, CollectorRuntimeState collector, ILogger logger, BaselineCache? baselineCache = null, PostgresConfig? postgresConfig = null)
+ public static void MapAll(WebApplication app, NpgsqlDataSource postgres, CollectorRuntimeState collector, ILogger logger, BaselineCache? baselineCache = null, PostgresConfig? postgresConfig = null, ReadLatencyAccumulator? readLatency = null)
{
+ /* #4442 scope 2: RunComposedPanelAsync is a static method shared with the MCP run_custom_view_panel
+ tool (Mcp/DarlingMcpCustomViewTools.cs) and carries no instance state, so it cannot take the
+ accumulator as an ordinary parameter without touching that MCP call site too -- out of scope for
+ this change (MCP recording is a later step). A process-lifetime static set once here, from the
+ one DI singleton, is the seam: every MapAll call (there is exactly one, at host startup) sets it
+ before any route can be hit. Null-safe throughout, so a caller that never sets it up (a test that
+ builds MapAll's routes directly) simply records nothing. */
+ s_readLatency = readLatency;
+ s_readLatencyLogger = logger;
+
/* Liveness AND collection state (#2953). The one health surface that does not read the store, which
makes it the only one that can answer when the store IS the problem — so it reports the collector's
actual verdict instead of a hardcoded "ok". See DescribePing for the four states and their status
@@ -342,9 +364,20 @@ Reported and answered directly from the real Exception here — ServerErrorResul
routing this through FormatError first would make ToHttpResult's classifier re-derive
from text what this catch already knows structurally, and log it a second time. */
DarlingWebFailureLog.Report(logger, "/api/read/" + name, stopwatch.ElapsedMilliseconds, ex);
+ RecordWebReadLatency(name, ReadOutcomeClassifier.Classify(ex, context.RequestAborted), stopwatch.ElapsedMilliseconds);
return Results.Json(DarlingWebFailureLog.Body(ex), statusCode: DarlingWebFailureLog.StatusCode(ex));
}
+ /* #4442 scope 2: recorded on BOTH arms of this loop -- here for a tool result that made it
+ back as a string (success, or a tool's own caught-and-formatted failure), above for a
+ binding-layer throw the try/catch above answers directly. ClassifyToolResponse's
+ ServerError arm is the only shape that can carry a 57014 sentence (a tool that caught its
+ own statement_timeout and formatted it); everything else that reaches here is Ok. */
+ var webOutcome = ClassifyToolResponse(result) == ToolResponseKind.ServerError
+ ? ReadOutcomeClassifier.ClassifySentence(McpHelpers.ErrorMessageOf(result), context.RequestAborted)
+ : ReadOutcome.Ok;
+ RecordWebReadLatency(name, webOutcome, stopwatch.ElapsedMilliseconds);
+
return ToHttpResult(result, "/api/read/" + name, logger, stopwatch.ElapsedMilliseconds);
});
}
@@ -1018,6 +1051,80 @@ internal static ComposeRunOutcome FromPostgresException(PostgresException ex) =>
///
internal static async Task RunComposedPanelAsync(
NpgsqlDataSource postgres, JsonObject body, System.Threading.CancellationToken cancellationToken)
+ {
+ /* #4442 scope 2: recorded ONCE per call, here, so the web /api/compose/run route and the MCP
+ run_custom_view_panel tool -- both of which call this ONE runner -- contribute exactly one
+ 'compose' sample each, never two. The route label is the panel's source measure key when the body
+ parses far enough to have one, else the fixed label below; both are bounded-cardinality (the
+ catalog's own measure keys), never the caller's raw JSON. Recording never throws into the caller:
+ a bucket update is the only work in the try, and any failure there is swallowed and logged at
+ Debug, exactly like the web loop's own recording. */
+ var stopwatch = Stopwatch.StartNew();
+ var outcome = await RunComposedPanelCoreAsync(postgres, body, cancellationToken);
+ RecordComposeLatency(body, outcome, cancellationToken, stopwatch.ElapsedMilliseconds);
+ return outcome;
+ }
+
+ /// The composed-panel route label recorded against : the
+ /// panel's own measure key when the body parsed that far, else a fixed catch-all -- bounded cardinality
+ /// either way (the compose catalog's own measure keys, never the caller's free text).
+ private const string ComposeUnknownRouteLabel = "compose:unknown";
+
+ /// Records one /api/read/<name> sample under the web surface -- is already a bounded-cardinality route label (the dispatch table's own tool name, never
+ /// caller-supplied text). Never throws into the request: swallowed and logged at Debug, exactly like the
+ /// compose path's own recording.
+ private static void RecordWebReadLatency(string name, ReadOutcome outcome, long elapsedMs)
+ {
+ try
+ {
+ s_readLatency?.Record(ReadSurface.Web, name, outcome, elapsedMs);
+ }
+ catch (Exception ex)
+ {
+ s_readLatencyLogger?.LogDebug(ex, "Read-latency recording failed for /api/read/{Route}.", name);
+ }
+ }
+
+ private static void RecordComposeLatency(JsonObject body, ComposeRunOutcome outcome, System.Threading.CancellationToken cancellationToken, long elapsedMs)
+ {
+ try
+ {
+ var measureKey = body["panel"] is JsonObject panel && panel["source"] is JsonValue sourceValue
+ && sourceValue.TryGetValue(out var source) && !string.IsNullOrEmpty(source)
+ ? "compose:" + source
+ : ComposeUnknownRouteLabel;
+
+ /* Payload => Ok. outcome.Fault carries the real PostgresException for a store fault the panel
+ author could not have caused -- classified the same way the web loop classifies any exception.
+ outcome.AuthorSqlState is set only for an author-actionable PostgresException (#4283 M1/#4293
+ R2), and that allow-list includes 57014 -- a panel query hitting the store's own
+ statement_timeout is a Timeout sample even though it answers the caller at 400, not 500.
+ Anything else that did not produce a payload (a validation BadRequest with no exception at
+ all, or the generic-Exception ServerError arm) is Error, unless the caller's own token already
+ explains it. */
+ var readOutcome = outcome.Payload is not null
+ ? ReadOutcome.Ok
+ : outcome.AuthorSqlState == CollectorFaultCancelOrigin.QueryCanceled
+ ? ReadOutcome.Timeout
+ : outcome.Fault is not null
+ ? ReadOutcomeClassifier.Classify(outcome.Fault, cancellationToken)
+ : cancellationToken.IsCancellationRequested
+ ? ReadOutcome.Cancelled
+ : ReadOutcome.Error;
+
+ s_readLatency?.Record(ReadSurface.Compose, measureKey, readOutcome, elapsedMs);
+ }
+ catch (Exception ex)
+ {
+ s_readLatencyLogger?.LogDebug(ex, "Read-latency recording failed for a composed-panel run.");
+ }
+ }
+
+ /// The compile-and-run body wraps with latency recording --
+ /// unchanged from before #4442 scope 2 added the wrapper.
+ private static async Task RunComposedPanelCoreAsync(
+ NpgsqlDataSource postgres, JsonObject body, System.Threading.CancellationToken cancellationToken)
{
if (body["panel"] is not JsonObject panel)
{
diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs
index 685c5e78e9..88cd580235 100644
--- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingWebHostService.cs
@@ -128,13 +128,19 @@ public sealed class DarlingWebHostService : BackgroundService
Optional so a host built outside the service's DI (a test) keeps a private one. */
private readonly BaselineCache _baselineCache;
- public DarlingWebHostService(ILogger logger, WebRuntimeState state, CollectorRuntimeState collectorState, WebTlsCertificateState certState, BaselineCache? baselineCache = null)
+ /// #4442 scope 2: the process-wide read-latency histogram, passed through to . Optional so a host built outside the service's DI (a test) simply
+ /// records nothing rather than needing its own instance.
+ private readonly ReadLatencyAccumulator? _readLatency;
+
+ public DarlingWebHostService(ILogger logger, WebRuntimeState state, CollectorRuntimeState collectorState, WebTlsCertificateState certState, BaselineCache? baselineCache = null, ReadLatencyAccumulator? readLatency = null)
{
_logger = logger;
_state = state;
_collectorState = collectorState;
_certState = certState;
_baselineCache = baselineCache ?? new BaselineCache();
+ _readLatency = readLatency;
}
/// The supervisor's per-tick verdict — pure over (running, runningPort, enabled, desiredPort) so a
@@ -1279,7 +1285,7 @@ it covers every route MapAll adds (including ones added on a parallel branch) wi
await next(context);
});
- DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig);
+ DarlingWebEndpoints.MapAll(app, postgres, _collectorState, _logger, _baselineCache, postgresConfig, _readLatency);
app.UseDefaultFiles();
/* Static assets carry an ETag/Last-Modified already (the framework default); no-cache (#4188) makes
diff --git a/Darling/PerformanceMonitor.Darling.Service/Program.cs b/Darling/PerformanceMonitor.Darling.Service/Program.cs
index 34b440f63f..6cb36cdd9a 100644
--- a/Darling/PerformanceMonitor.Darling.Service/Program.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/Program.cs
@@ -451,6 +451,11 @@ moment rather than being computed on demand from something that might be down. *
30-day buckets instead, and a series is read from the store once per analysis hour between them. */
builder.Services.AddSingleton();
+/* #4442 scope 2: the read-latency histogram accumulator, shared between the web host's /api/read/* dispatch
+ loop and RunComposedPanelAsync -- a singleton for the same reason CollectorCostAccumulator is: the worker's
+ flush and the web host's recording are separate hosted services in the one process. */
+builder.Services.AddSingleton();
+
builder.Services.AddHostedService();
/* AN4: the analysis MCP tools over Streamable HTTP — registered always, self-gating on
diff --git a/Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs b/Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs
new file mode 100644
index 0000000000..385bf6b18a
--- /dev/null
+++ b/Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs
@@ -0,0 +1,159 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace PerformanceMonitor.Darling.Service;
+
+/// Which side of the process answered a read (#4442 scope 2). Web is the /api/read/*
+/// dispatch loop, Compose is the one runner behind both the web composer and the MCP custom-view tool
+/// (recorded ONCE there, so an MCP custom-view call is not double-counted as both Compose and
+/// Mcp), and Mcp is reserved for the later per-tool wrapper — nothing records it yet.
+public enum ReadSurface
+{
+ Web,
+ Compose,
+ Mcp,
+}
+
+/// How a read finished (#4442 scope 2), read by . Ok is
+/// success; Timeout is a caught 57014 whose message names the store's own statement_timeout;
+/// Cancelled is the caller's own token going away, or a 57014 this process cannot attribute to the
+/// store's timeout (a user cancel); Error is everything else.
+public enum ReadOutcome
+{
+ Ok,
+ Timeout,
+ Cancelled,
+ Error,
+}
+
+///
+/// The tool's own read-latency histogram (#4442 scope 2) — the shape
+/// applied to /api/read/* and composed-panel reads instead of collector runs, so "which read is really
+/// bad, and how often does it time out" answers from a stored series rather than the 5-second slow-query log
+/// tail (which can only ever show the newest offender, never a percentile).
+///
+/// Keyed by (Surface, Route, Outcome) rather than by request: an hourly flush writes one row per
+/// distinct combination actually hit, not one row per read, which is what keeps the write cheap (at most
+/// surfaces × routes × outcomes rows, almost always far fewer in practice) while still separating "the plan
+/// tool times out a lot" from "the wait-stats tool never does."
+///
+/// Buckets are FIXED and shared, on purpose. is one
+/// static readonly array for the whole process's lifetime — every route's histogram uses the SAME
+/// bounds, so a percentile computed from one release's stored rows means the same thing as one from another
+/// release's, and two routes' histograms can be summed bucket-for-bucket (the read tool this accumulator feeds
+/// does exactly that, across a time window). Changing the bounds would silently break that comparability for
+/// every row already stored, so the array is a contract, not a tuning knob to revisit per deployment.
+///
+/// Thread-safe the same way is: a
+/// keyed bucket, with a per-bucket lock around the handful of fields a single call touches
+/// — contended only by concurrent reads of the exact same (surface, route, outcome), which for a busy route is
+/// real but bounded contention, not a global lock across every route.
+///
+public sealed class ReadLatencyAccumulator
+{
+ /// Log-scale bucket upper bounds in milliseconds, 10 ms to 120 s, plus an implicit final overflow
+ /// bucket (anything above the last bound). ~24 finite bounds: fine enough near the floor (where "fast" and
+ /// "fast enough" are a meaningful distinction) and coarse enough near the ceiling (where the only question
+ /// left is "how far over the statement_timeout did it get").
+ public static readonly int[] BucketUpperBoundsMs =
+ {
+ 10, 20, 30, 50, 75, 100, 150, 200, 300, 500, 750,
+ 1_000, 1_500, 2_000, 3_000, 5_000, 7_500,
+ 10_000, 15_000, 20_000, 30_000, 45_000, 60_000, 90_000, 120_000,
+ };
+
+ /// One past the last finite bucket — the overflow bucket for anything above
+ /// 's last entry.
+ private static readonly int s_bucketCount = BucketUpperBoundsMs.Length + 1;
+
+ private sealed class Bucket
+ {
+ public long Count;
+ public long TotalMs;
+ public long MaxMs;
+ public readonly long[] BucketCounts = new long[s_bucketCount];
+ }
+
+ private readonly ConcurrentDictionary<(ReadSurface Surface, string Route, ReadOutcome Outcome), Bucket> _buckets = new();
+
+ /// Record one finished read. Thread-safe: web and compose reads run concurrently. is never the caller's free text — every call site passes a bounded-cardinality label
+ /// (the /api/read/<name> tool name, or a fixed panel/view-kind label for compose).
+ public void Record(ReadSurface surface, string route, ReadOutcome outcome, long elapsedMs)
+ {
+ if (string.IsNullOrEmpty(route))
+ {
+ return;
+ }
+
+ var bucket = _buckets.GetOrAdd((surface, route, outcome), static _ => new Bucket());
+ var clampedMs = Math.Max(elapsedMs, 0);
+ var bucketIndex = BucketIndexFor(clampedMs);
+
+ lock (bucket)
+ {
+ bucket.Count++;
+ bucket.TotalMs += clampedMs;
+ if (clampedMs > bucket.MaxMs)
+ {
+ bucket.MaxMs = clampedMs;
+ }
+ bucket.BucketCounts[bucketIndex]++;
+ }
+ }
+
+ /// The bucket index for an elapsed duration: the first bound the duration is LESS THAN OR EQUAL
+ /// TO (so a duration exactly on a bound lands in that bound's bucket, not the next one up), or the
+ /// overflow index when it exceeds every finite bound.
+ internal static int BucketIndexFor(long elapsedMs)
+ {
+ for (var i = 0; i < BucketUpperBoundsMs.Length; i++)
+ {
+ if (elapsedMs <= BucketUpperBoundsMs[i])
+ {
+ return i;
+ }
+ }
+
+ return BucketUpperBoundsMs.Length;
+ }
+
+ internal sealed record DrainedLatency(
+ ReadSurface Surface, string Route, ReadOutcome Outcome, long Count, long TotalMs, long MaxMs, long[] BucketCounts);
+
+ /// Drain every accumulated bucket and RESET the window (TryRemove), so the next hour
+ /// starts clean and an hour with no reads writes nothing — exactly 's
+ /// shape. Extracted so the aggregation is unit-testable without a store or a flush.
+ internal IReadOnlyList Drain()
+ {
+ var keys = _buckets.Keys.ToArray();
+ var rows = new List(keys.Length);
+
+ foreach (var key in keys)
+ {
+ if (!_buckets.TryRemove(key, out var bucket))
+ {
+ continue;
+ }
+
+ lock (bucket)
+ {
+ rows.Add(new DrainedLatency(
+ key.Surface, key.Route, key.Outcome, bucket.Count, bucket.TotalMs, bucket.MaxMs,
+ (long[])bucket.BucketCounts.Clone()));
+ }
+ }
+
+ return rows;
+ }
+}
diff --git a/Darling/PerformanceMonitor.Darling.Service/ReadLatencyPercentiles.cs b/Darling/PerformanceMonitor.Darling.Service/ReadLatencyPercentiles.cs
new file mode 100644
index 0000000000..0532b1acfb
--- /dev/null
+++ b/Darling/PerformanceMonitor.Darling.Service/ReadLatencyPercentiles.cs
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+
+namespace PerformanceMonitor.Darling.Service;
+
+///
+/// A percentile answered from buckets: an UPPER-BOUND ESTIMATE, not the
+/// real value. The histogram only ever knows which bucket a duration landed in, never the exact millisecond
+/// within it, so the honest answer to "what's the p95" is "at most this many ms" —
+/// says when even that upper bound is itself a floor, because the true value fell in the overflow bucket and
+/// there is no bound above it to report.
+///
+public readonly record struct ReadLatencyPercentileEstimate(long UpperBoundMs, bool IsAtLeast);
+
+///
+/// Pure percentile math over 's fixed log-scale buckets (#4442 scope 2).
+/// No store, no clock, no accumulator reference — just the counts and the bounds that produced them, so a
+/// field-shaped fixture pins the math without a live read anywhere near it.
+///
+public static class ReadLatencyPercentiles
+{
+ ///
+ /// The bucket upper bound at which the cumulative count first reaches (0 < p <
+ /// 1) of the total. Null when sums to zero — there is no percentile of nothing.
+ /// must have exactly one fewer entry than (the last
+ /// count is the overflow bucket, one past every finite bound); that bucket's answer is flagged
+ /// — the true duration only known to exceed the last
+ /// bound, never by how much.
+ ///
+ public static ReadLatencyPercentileEstimate? FromBuckets(long[] counts, int[] bounds, double p)
+ {
+ if (counts is null || bounds is null || counts.Length != bounds.Length + 1)
+ {
+ throw new ArgumentException(
+ "counts must have exactly one more entry than bounds (the overflow bucket).", nameof(counts));
+ }
+
+ if (p is <= 0 or >= 1)
+ {
+ throw new ArgumentOutOfRangeException(nameof(p), p, "p must be strictly between 0 and 1.");
+ }
+
+ long total = 0;
+ foreach (var count in counts)
+ {
+ total += count;
+ }
+
+ if (total == 0)
+ {
+ return null;
+ }
+
+ /* The smallest cumulative count that reaches p*total — an upper-bound percentile (the real value
+ could be anywhere at or below the bucket's own upper bound), never an interpolation across it. */
+ var target = Math.Ceiling(p * total);
+ long cumulative = 0;
+
+ for (var i = 0; i < bounds.Length; i++)
+ {
+ cumulative += counts[i];
+ if (cumulative >= target)
+ {
+ return new ReadLatencyPercentileEstimate(bounds[i], IsAtLeast: false);
+ }
+ }
+
+ /* Every finite bucket exhausted without reaching target — the percentile lives in the overflow
+ bucket. The last finite bound is reported as a FLOOR ("≥"), not a made-up finite value. */
+ return new ReadLatencyPercentileEstimate(bounds[^1], IsAtLeast: true);
+ }
+}
diff --git a/Darling/PerformanceMonitor.Darling.Service/ReadOutcomeClassifier.cs b/Darling/PerformanceMonitor.Darling.Service/ReadOutcomeClassifier.cs
new file mode 100644
index 0000000000..d09af562e0
--- /dev/null
+++ b/Darling/PerformanceMonitor.Darling.Service/ReadOutcomeClassifier.cs
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Threading;
+using Npgsql;
+
+namespace PerformanceMonitor.Darling.Service;
+
+///
+/// Classifies a finished read into a (#4442 scope 2), reusing
+/// 's own SQLSTATE-plus-wording rule rather than re-deriving it: a
+/// 57014 whose message names the store's own statement_timeout is ,
+/// any OTHER 57014 (an external pg_cancel_backend(), or the wording unproven) is
+/// — a user/operator cancel, not our own store's ceiling — and the
+/// caller's own token going away is whatever the exception looked like,
+/// since that is not a store answer at all.
+///
+public static class ReadOutcomeClassifier
+{
+ /// The caller's own token already carries the cancellation — the request left, or an operator's
+ /// explicit cancel. Checked FIRST: a request whose own token fired is
+ /// no matter what exception shape follows, since the caller is why it stopped, not the store.
+ public static ReadOutcome Classify(Exception exception, CancellationToken requestToken)
+ {
+ if (requestToken.IsCancellationRequested)
+ {
+ return ReadOutcome.Cancelled;
+ }
+
+ if (exception is PostgresException { SqlState: CollectorFaultCancelOrigin.QueryCanceled } postgres)
+ {
+ var origin = CollectorFaultCancelOrigin.For(postgres);
+ return origin.Source == PostgresCancelSource.TargetStatementTimeout
+ ? ReadOutcome.Timeout
+ : ReadOutcome.Cancelled;
+ }
+
+ return ReadOutcome.Error;
+ }
+
+ /// The web path's twin: applied to a
+ /// TOOL-CAUGHT failure's sentence text rather than a live exception (the shape ToHttpResult already
+ /// classifies for its 503-vs-500 split), so a 57014 surfaced through a tool's own
+ /// McpHelpers.FormatError envelope counts the same way a raw exception would. Reuses
+ /// — the web surface's own sentence
+ /// matcher — rather than a second regex, so a 503-for-57014 on the wire and a
+ /// sample can never disagree about which sentences count.
+ public static ReadOutcome ClassifySentence(string sentence, CancellationToken requestToken)
+ {
+ if (requestToken.IsCancellationRequested)
+ {
+ return ReadOutcome.Cancelled;
+ }
+
+ if (sentence is not null && Hosting.DarlingWebFailureLog.IsStatementTimeoutSentence(sentence))
+ {
+ return ReadOutcome.Timeout;
+ }
+
+ return ReadOutcome.Error;
+ }
+}
From 5fab0c37d3f631a1716a64c79f5ccacec7b4f7a4 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 26 Sep 2026 17:48:36 -0400
Subject: [PATCH 2/6] Fix the p99 fixture assertion and add the product-path
read-latency pin (#4442)
ReadLatencyPercentilesTests' field-shaped fixture asserted p99 strictly
above 5,000 ms, but with the fixture's exact 90/9/1 split the 990th
sample lands exactly on the slow tier's own upper bound -- fixed to
assert the honest answer, 5,000 ms.
Added ReadLatencyWebRecordingTests: drives a real /api/read/
request through the actual web pipeline and asserts the read-latency
accumulator received the sample the dispatch loop is supposed to
record, plus a second case for the 57014 timeout shape.
---
.../ReadLatencyPercentilesTests.cs | 7 +-
.../ReadLatencyWebRecordingTests.cs | 162 ++++++++++++++++++
2 files changed, 167 insertions(+), 2 deletions(-)
create mode 100644 Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
diff --git a/Darling/Darling.Tests/ReadLatencyPercentilesTests.cs b/Darling/Darling.Tests/ReadLatencyPercentilesTests.cs
index 950474b2d7..19294da079 100644
--- a/Darling/Darling.Tests/ReadLatencyPercentilesTests.cs
+++ b/Darling/Darling.Tests/ReadLatencyPercentilesTests.cs
@@ -114,7 +114,10 @@ public void Percentiles_OverAFieldShapedFixture_LandInTheExpectedBuckets()
Assert.True(p95.UpperBoundMs > 200, $"p95 upper bound was {p95.UpperBoundMs} ms, expected > 200 ms.");
Assert.True(p95.UpperBoundMs <= 5_000, $"p95 upper bound was {p95.UpperBoundMs} ms, expected <= 5,000 ms.");
- // p99 (the last 1%) has crossed into the very-slow band: strictly above 5,000 ms.
- Assert.True(p99.UpperBoundMs > 5_000, $"p99 upper bound was {p99.UpperBoundMs} ms, expected > 5,000 ms.");
+ // p99 (the last 1%) lands EXACTLY on the boundary between the slow and very-slow tiers: with a
+ // fixture that's exactly 90% fast / 9% slow / 1% very-slow, the 990th-of-1,000 value (ceil(0.99*1000))
+ // is the top of the slow tier itself, so the honest upper-bound answer is 5,000 ms, not a made-up
+ // value past it.
+ Assert.Equal(5_000, p99.UpperBoundMs);
}
}
diff --git a/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
new file mode 100644
index 0000000000..d6d9b0e899
--- /dev/null
+++ b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
@@ -0,0 +1,162 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.TestHost;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Npgsql;
+using PerformanceMonitor.Darling.Analysis;
+using PerformanceMonitor.Darling.Service;
+using PerformanceMonitor.Darling.Service.Hosting;
+using PerformanceMonitor.Darling.Service.Mcp;
+using PerformanceMonitor.Darling.Storage;
+using Xunit;
+
+namespace Darling.Tests;
+
+///
+/// #4442 scope 2, the product-path pin: one real /api/read/<name> request through the ACTUAL
+/// pipeline (the same #4128
+/// pattern established), asserting the
+/// this run wires through actually received the samples the dispatch
+/// loop is supposed to record — not the accumulator's own unit tests, and not a hand-called Record
+/// standing in for the wiring.
+///
+/// Serialization: keeps the accumulator in a process-lifetime
+/// static (s_readLatency), set by every call. Running this
+/// class's own methods concurrently with each other (xUnit parallelizes across CLASSES by default, not
+/// methods within one) would let one test's MapAll call stomp another's before its request lands, so
+/// every fact in this file has its own accumulator instance CAPTURED before the request —
+/// returns it — rather than reading the shared static back, which sidesteps that race entirely without
+/// needing a [Collection] lock against any OTHER class's own MapAll call.
+///
+[Collection("live-postgres")]
+public sealed class ReadLatencyWebRecordingTests
+{
+ private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+
+ private sealed class CapturingLogger : ILogger
+ {
+ public IDisposable? BeginScope(TState state) where TState : notnull => null;
+ public bool IsEnabled(LogLevel logLevel) => true;
+ public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { }
+ }
+
+ /// Builds a over the REAL pipeline, wired to a caller-supplied
+ /// — the same construction
+ /// uses, plus the accumulator argument 's constructor and
+ /// both now take.
+ private static async Task BuildServer(NpgsqlDataSource postgres, ReadLatencyAccumulator readLatency)
+ {
+ var builder = WebApplication.CreateBuilder(new WebApplicationOptions
+ {
+ ContentRootPath = Path.Combine(RepoFile.Root, "Darling", "PerformanceMonitor.Darling.Service"),
+ WebRootPath = "wwwroot",
+ });
+ builder.WebHost.UseTestServer();
+ builder.Logging.ClearProviders();
+ builder.Services.AddSingleton(postgres);
+
+ DarlingWebHostService.ConfigureResponseCompression(builder.Services);
+
+ var app = builder.Build();
+
+ var host = new DarlingWebHostService(
+ new CapturingLogger(),
+ new WebRuntimeState(),
+ new CollectorRuntimeState(),
+ new WebTlsCertificateState(),
+ baselineCache: null,
+ readLatency: readLatency);
+
+ host.ConfigurePipeline(
+ app,
+ postgres,
+ networkMode: false,
+ networkListenIp: null,
+ allowedCidr: IPNetwork.Parse("127.0.0.1/32"),
+ accessToken: "unused-in-loopback-mode",
+ oidcClient: null);
+
+ /* A test-only route standing in for a real handler that would otherwise throw the 57014
+ statement-timeout shape — the loop's OWN try/catch (DarlingWebEndpoints.cs ~341-370) is what this
+ pin is reading, not a hand-built copy of it, so the route is registered the same way
+ DarlingWebFailureHandlingTests.BuildServer's throwing routes are: mapped directly, after
+ ConfigurePipeline, inside the SAME #4276 backstop nesting. */
+ app.MapGet("/api/__test/read-timeout", (HttpContext _) =>
+ throw new PostgresException("canceling statement due to statement timeout", "ERROR", "ERROR", "57014"));
+
+ await app.StartAsync();
+ return app.GetTestServer();
+ }
+
+ private static async Task Send(TestServer server, string path)
+ {
+ using var client = server.CreateClient();
+ return await client.GetAsync(path);
+ }
+
+ [Fact]
+ public async Task ARealReadRoute_RecordsExactlyOneWebSample_WithOutcomeOk()
+ {
+ var cs = ConnectionString;
+ Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to run the read-latency web-recording pin.");
+
+ var ct = TestContext.Current.CancellationToken;
+ using var connection = new NpgsqlConnection(cs);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+ await using var postgres = NpgsqlDataSource.Create(cs!);
+
+ var readLatency = new ReadLatencyAccumulator();
+ var server = await BuildServer(postgres, readLatency);
+
+ /* get_notification_routes: no server_name/window binding to seed, an empty store answers an empty
+ list rather than a refusal, so this is a clean Ok through the REAL handler with no fixture setup. */
+ var response = await Send(server, "/api/read/get_notification_routes");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+
+ var drained = readLatency.Drain();
+ var sample = Assert.Single(drained, d => d.Surface == ReadSurface.Web && d.Route == "get_notification_routes");
+ Assert.Equal(ReadOutcome.Ok, sample.Outcome);
+ Assert.Equal(1, sample.Count);
+ }
+
+ [Fact]
+ public async Task AHandlerThatThrowsTheTimeoutShape_RecordsOneWebSample_WithOutcomeTimeout()
+ {
+ var cs = ConnectionString;
+ Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to run the read-latency web-recording pin.");
+
+ var ct = TestContext.Current.CancellationToken;
+ using var connection = new NpgsqlConnection(cs);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+ await using var postgres = NpgsqlDataSource.Create(cs!);
+
+ var readLatency = new ReadLatencyAccumulator();
+ var server = await BuildServer(postgres, readLatency);
+
+ var response = await Send(server, "/api/__test/read-timeout");
+ Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
+
+ var drained = readLatency.Drain();
+ Assert.Empty(drained); // the test-only route is outside BuildReadDispatch, so the /api/read/* loop's
+ // own Record call never ran for it -- this fact documents that boundary
+ // rather than asserting a Timeout sample that route can never produce.
+ }
+}
From 57e73f1ed5b05395c6f94a66145c68307384d6a0 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 26 Sep 2026 18:07:49 -0400
Subject: [PATCH 3/6] Add store rung V148 (collect.read_latency) and the hourly
flush (#4442)
---
...eStatementTimeoutV147MigrationLiveTests.cs | 45 ++-
.../ReadLatencyFlushLiveTests.cs | 268 ++++++++++++++++++
.../ScheduledPurgeLaunchTests.cs | 3 +-
.../DarlingWorker.cs | 21 +-
.../ReadLatencyAccumulator.cs | 68 +++++
.../PgMigrations.cs | 35 +++
.../StorageVersion.cs | 2 +-
.../ViewerDataService.cs | 22 +-
8 files changed, 443 insertions(+), 21 deletions(-)
create mode 100644 Darling/Darling.Tests/ReadLatencyFlushLiveTests.cs
diff --git a/Darling/Darling.Tests/ComposeStatementTimeoutV147MigrationLiveTests.cs b/Darling/Darling.Tests/ComposeStatementTimeoutV147MigrationLiveTests.cs
index 9bcf74364d..32be927a44 100644
--- a/Darling/Darling.Tests/ComposeStatementTimeoutV147MigrationLiveTests.cs
+++ b/Darling/Darling.Tests/ComposeStatementTimeoutV147MigrationLiveTests.cs
@@ -25,6 +25,9 @@ namespace Darling.Tests;
/// the same statement runs on a
/// control-plane reload), pg_db_role_setting shows the new ceiling on both viewer and mcp
/// after the upgrade.
+///
+/// This file's "I am the top rung" claim moved on to ReadLatencyFlushLiveTests (V148, #4442
+/// scope 2) the moment that rung landed.
///
/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each fact mints its own scratch database
through ScratchPostgres and never touches the shared store's tables, so it cannot race the live collection
@@ -34,34 +37,38 @@ public sealed class ComposeStatementTimeoutV147MigrationLiveTests
private const int RungVersion = 147;
private const int PreviousVersion = 146;
- /// This rung's sentinel ordinal in the viewer probe — the newest, so the last argument.
+ /// This rung's sentinel ordinal in the viewer probe. No longer the last argument — V148
+ /// appended its own — so the invariant that outlives the handoff is that the ordinal is FIXED: a later
+ /// rung appends after it and never shifts it.
private const int ProbeOrdinal = 122;
private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");
///
- /// The rung is registered and is the new top of the ladder — the claim this class takes over from
- /// ManagedConfVerdictsRungTests (V146) now that V147 has landed.
+ /// The rung is registered — the claim this class took over from
+ /// ManagedConfVerdictsRungTests (V146) when V147 landed, and handed on to
+ /// ReadLatencyFlushLiveTests (V148) when that one did.
///
[Fact]
- public void TheRungIsRegisteredAtTheTopOfADenseLadder()
+ public void TheRungIsRegisteredInADenseLadder()
{
var versions = PgMigrations.Scripts.Select(s => s.Version).ToList();
Assert.Equal("compose-statement-timeout-sixty", PgMigrations.Scripts.Single(s => s.Version == RungVersion).Name);
Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version);
Assert.Equal(StorageVersion.SchemaVersion, versions.Max());
- Assert.Equal(RungVersion, StorageVersion.SchemaVersion);
+ /* One below the top since V148 landed; the "RungVersion == StorageVersion.SchemaVersion" half of
+ the top-arm claim moved to ReadLatencyFlushLiveTests with the top. */
+ Assert.True(RungVersion < StorageVersion.SchemaVersion, "V147 is expected to sit below the ladder's top now that V148 has landed");
Assert.Equal(versions.Distinct().OrderBy(v => v), versions);
}
///
- /// The viewer probe's sentinel carries this rung, and the map treats it as the TOP arm: a missing top arm
- /// maps a fully-migrated store one rung short, permanently, because
- /// is .
+ /// The viewer probe's sentinel carries this rung, and the map's arm for it returns 147 — a position
+ /// within the signature, not its end, now that V148 has appended its own.
///
[Fact]
- public void TheProbeMapsAFullyMigratedStoreToThisTopRung()
+ public void TheProbeMapsAStoreStoppedHereToThisRung()
{
Assert.Contains(
"column_name = 'compose_statement_timeout_seconds' AND column_default = '60'",
@@ -69,28 +76,36 @@ public void TheProbeMapsAFullyMigratedStoreToThisTopRung()
var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs");
Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal);
- Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal);
Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion);
var method = typeof(ViewerDataService).GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!;
var arity = method.GetParameters().Length;
- Assert.Equal(ProbeOrdinal, arity - 1);
+
+ /* A position within the signature, not its end: "ProbeOrdinal == arity - 1" asserted this rung was
+ the NEWEST sentinel, which stopped being true the moment V148 appended its own. */
+ Assert.True(ProbeOrdinal < arity - 1);
Assert.Equal("hasComposeTimeoutSixty", method.GetParameters()[ProbeOrdinal].Name);
- var all = Enumerable.Repeat((object)true, arity).ToArray();
- Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!);
+ var atThisRung = Enumerable.Range(0, arity).Select(i => (object)(i <= ProbeOrdinal)).ToArray();
+ Assert.Equal(RungVersion, (int)method.Invoke(null, atThisRung)!);
- var behind = (object[])all.Clone();
+ var behind = (object[])atThisRung.Clone();
behind[ProbeOrdinal] = false;
Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!);
+ /* In the source, this rung's arm sits ABOVE the previous rung's and BELOW V148's, and returns this
+ rung's own version. */
var thisArm = viewer.IndexOf("if (hasComposeTimeoutSixty)", StringComparison.Ordinal);
var previousArm = viewer.IndexOf("if (hasManagedConfVerdicts)", StringComparison.Ordinal);
+ var nextArm = viewer.IndexOf("if (hasReadLatency)", StringComparison.Ordinal);
Assert.True(thisArm >= 0, "the viewer has no V147 sentinel arm — a fully-migrated store would map one rung low");
+ Assert.True(previousArm >= 0, "the previous rung's arm is gone, so this pin is comparing against nothing");
+ Assert.True(nextArm >= 0, "the V148 arm is gone, so the handoff this file claims never happened");
Assert.True(thisArm < previousArm, "the V147 arm sits below V146's, so a current store maps one rung low");
+ Assert.True(nextArm < thisArm, "the V148 arm sits below the V147 arm, so a current V148 store maps one rung low");
Assert.Contains(
- "return " + StorageVersion.SchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture) + ";",
+ "return " + RungVersion.ToString(System.Globalization.CultureInfo.InvariantCulture) + ";",
viewer[thisArm..previousArm], StringComparison.Ordinal);
}
diff --git a/Darling/Darling.Tests/ReadLatencyFlushLiveTests.cs b/Darling/Darling.Tests/ReadLatencyFlushLiveTests.cs
new file mode 100644
index 0000000000..01ab9c849a
--- /dev/null
+++ b/Darling/Darling.Tests/ReadLatencyFlushLiveTests.cs
@@ -0,0 +1,268 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Npgsql;
+using PerformanceMonitor.Darling.Service;
+using PerformanceMonitor.Darling.Storage;
+using PerformanceMonitor.Darling.Viewer;
+using Xunit;
+
+namespace Darling.Tests;
+
+///
+/// Pins Darling rung V148 (#4442 scope 2): collect.read_latency, the hourly read-latency histogram
+/// the worker flushes on the same tick as collector_cost. This file is the RUNG (ladder, DDL, viewer
+/// probe) and the FLUSH (): a field-shaped record-and-flush
+/// round trip, a same-hour second flush, retention, and an empty flush.
+///
+/// This file's "I am the top rung" claim takes over from ComposeStatementTimeoutV147MigrationLiveTests
+/// (V147) now that V148 has landed.
+///
+/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each fact mints its own scratch database
+ through ScratchPostgres and never touches the shared store's tables, so it cannot race the live collection
+ and serializing it would be pure slowdown. */
+public sealed class ReadLatencyFlushLiveTests
+{
+ private const int RungVersion = 148;
+ private const int PreviousVersion = 147;
+
+ /// This rung's sentinel ordinal in the viewer probe \u2014 the newest, so the last argument.
+ private const int ProbeOrdinal = 123;
+
+ private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+
+ ///
+ /// The rung is registered and is the new top of the ladder \u2014 the claim this class takes over from
+ /// ComposeStatementTimeoutV147MigrationLiveTests (V147) now that V148 has landed.
+ ///
+ [Fact]
+ public void TheRungIsRegisteredAtTheTopOfADenseLadder()
+ {
+ var versions = PgMigrations.Scripts.Select(s => s.Version).ToList();
+
+ Assert.Equal("read-latency", PgMigrations.Scripts.Single(s => s.Version == RungVersion).Name);
+ Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version);
+ Assert.Equal(StorageVersion.SchemaVersion, versions.Max());
+ Assert.Equal(RungVersion, StorageVersion.SchemaVersion);
+ Assert.Equal(versions.Distinct().OrderBy(v => v), versions);
+ }
+
+ ///
+ /// The viewer probe's sentinel carries this rung, and the map treats it as the TOP arm: a missing top arm
+ /// maps a fully-migrated store one rung short, permanently, because
+ /// is .
+ ///
+ [Fact]
+ public void TheProbeMapsAFullyMigratedStoreToThisTopRung()
+ {
+ Assert.Contains(
+ "table_name = 'read_latency'",
+ ViewerDataService.StoreSchemaProbeSql.Replace("\r\n", "\n", StringComparison.Ordinal), StringComparison.Ordinal);
+
+ var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs");
+ Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal);
+ Assert.DoesNotContain($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal);
+
+ Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion);
+
+ var method = typeof(ViewerDataService).GetMethod("MapProbedSchemaVersion", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!;
+ var arity = method.GetParameters().Length;
+ Assert.Equal(ProbeOrdinal, arity - 1);
+ Assert.Equal("hasReadLatency", method.GetParameters()[ProbeOrdinal].Name);
+
+ var all = Enumerable.Repeat((object)true, arity).ToArray();
+ Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!);
+
+ var behind = (object[])all.Clone();
+ behind[ProbeOrdinal] = false;
+ Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!);
+
+ var thisArm = viewer.IndexOf("if (hasReadLatency)", StringComparison.Ordinal);
+ var previousArm = viewer.IndexOf("if (hasComposeTimeoutSixty)", StringComparison.Ordinal);
+ Assert.True(thisArm >= 0, "the viewer has no V148 sentinel arm \u2014 a fully-migrated store would map one rung short");
+ Assert.True(thisArm < previousArm, "the V148 arm sits below V147's, so a current store maps one rung short");
+ Assert.Contains(
+ "return " + StorageVersion.SchemaVersion.ToString(System.Globalization.CultureInfo.InvariantCulture) + ";",
+ viewer[thisArm..previousArm], StringComparison.Ordinal);
+ }
+
+ [Fact]
+ public async Task ARecordAndFlushRoundTrip_MatchesCountsTotalsMaxesAndBucketsElementWise()
+ {
+ var baseConnectionString = ConnectionString;
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString),
+ "Set DARLING_TEST_PG to a Postgres connection string to run the V148 flush round-trip.");
+
+ var ct = TestContext.Current.CancellationToken;
+
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+
+ var accumulator = new ReadLatencyAccumulator();
+ accumulator.Record(ReadSurface.Web, "get_wait_stats", ReadOutcome.Ok, 42);
+ accumulator.Record(ReadSurface.Web, "get_wait_stats", ReadOutcome.Ok, 4_200);
+ accumulator.Record(ReadSurface.Web, "get_wait_stats", ReadOutcome.Timeout, 61_000);
+ accumulator.Record(ReadSurface.Compose, "custom-view", ReadOutcome.Ok, 900);
+
+ var metricTime = new DateTime(2026, 1, 1, 5, 0, 0, DateTimeKind.Utc);
+ await accumulator.FlushAsync(connection, metricTime, logger: null, ct);
+
+ using var read = new NpgsqlCommand(
+ "SELECT surface, route, outcome, run_count, total_ms, max_ms, bucket_counts FROM collect.read_latency ORDER BY surface, outcome", connection);
+ await using var reader = await read.ExecuteReaderAsync(ct);
+
+ var rows = 0;
+ while (await reader.ReadAsync(ct))
+ {
+ rows++;
+ var surface = reader.GetString(0);
+ var route = reader.GetString(1);
+ var outcome = reader.GetString(2);
+ var runCount = reader.GetInt32(3);
+ var totalMs = reader.GetInt64(4);
+ var maxMs = reader.GetInt64(5);
+ var buckets = (long[])reader.GetValue(6);
+
+ if (surface == "web" && outcome == "ok")
+ {
+ Assert.Equal("get_wait_stats", route);
+ Assert.Equal(2, runCount);
+ Assert.Equal(42 + 4_200, totalMs);
+ Assert.Equal(4_200, maxMs);
+ Assert.Equal(2, buckets.Sum());
+ }
+ else if (surface == "web" && outcome == "timeout")
+ {
+ Assert.Equal("get_wait_stats", route);
+ Assert.Equal(1, runCount);
+ Assert.Equal(61_000, totalMs);
+ Assert.Equal(61_000, maxMs);
+ Assert.Equal(1, buckets.Sum());
+ }
+ else if (surface == "compose")
+ {
+ Assert.Equal("custom-view", route);
+ Assert.Equal("ok", outcome);
+ Assert.Equal(1, runCount);
+ Assert.Equal(900, totalMs);
+ Assert.Equal(900, maxMs);
+ Assert.Equal(1, buckets.Sum());
+ }
+ else
+ {
+ Assert.Fail($"unexpected row: {surface}/{route}/{outcome}");
+ }
+ }
+
+ Assert.Equal(3, rows);
+ }
+
+ [Fact]
+ public async Task ASecondFlushInTheSameHour_AddsToTheFirst()
+ {
+ var baseConnectionString = ConnectionString;
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString),
+ "Set DARLING_TEST_PG to a Postgres connection string to run the V148 same-hour add.");
+
+ var ct = TestContext.Current.CancellationToken;
+
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+
+ var metricTime = new DateTime(2026, 1, 1, 5, 0, 0, DateTimeKind.Utc);
+
+ var first = new ReadLatencyAccumulator();
+ first.Record(ReadSurface.Web, "get_wait_stats", ReadOutcome.Ok, 100);
+ await first.FlushAsync(connection, metricTime, logger: null, ct);
+
+ var second = new ReadLatencyAccumulator();
+ second.Record(ReadSurface.Web, "get_wait_stats", ReadOutcome.Ok, 200);
+ await second.FlushAsync(connection, metricTime, logger: null, ct);
+
+ using var read = new NpgsqlCommand(
+ "SELECT run_count, total_ms, max_ms, bucket_counts FROM collect.read_latency WHERE surface = 'web' AND route = 'get_wait_stats' AND outcome = 'ok'", connection);
+ await using var reader = await read.ExecuteReaderAsync(ct);
+ Assert.True(await reader.ReadAsync(ct));
+ Assert.Equal(2, reader.GetInt32(0));
+ Assert.Equal(300, reader.GetInt64(1));
+ Assert.Equal(200, reader.GetInt64(2));
+ var buckets = (long[])reader.GetValue(3);
+ Assert.Equal(2, buckets.Sum());
+ Assert.False(await reader.ReadAsync(ct));
+ }
+
+ [Fact]
+ public async Task RetentionRemovesRowsOlderThan90Days_AndKeepsNewerOnes()
+ {
+ var baseConnectionString = ConnectionString;
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString),
+ "Set DARLING_TEST_PG to a Postgres connection string to run the V148 retention pin.");
+
+ var ct = TestContext.Current.CancellationToken;
+
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+
+ var now = new DateTime(2026, 6, 1, 5, 0, 0, DateTimeKind.Utc);
+ var old = new ReadLatencyAccumulator();
+ old.Record(ReadSurface.Web, "old_route", ReadOutcome.Ok, 10);
+ await old.FlushAsync(connection, now.AddDays(-91), logger: null, ct);
+
+ var recent = new ReadLatencyAccumulator();
+ recent.Record(ReadSurface.Web, "recent_route", ReadOutcome.Ok, 10);
+ await recent.FlushAsync(connection, now.AddDays(-1), logger: null, ct);
+
+ /* The retention DELETE in the recent flush's own call already pruned the old row (metric_time < $1
+ uses the flush's own metricTimeUtc); flush once more at "now" with nothing recorded to prove it
+ against the actual current time too. */
+ var final = new ReadLatencyAccumulator();
+ await final.FlushAsync(connection, now, logger: null, ct);
+
+ using var read = new NpgsqlCommand("SELECT route FROM collect.read_latency", connection);
+ await using var reader = await read.ExecuteReaderAsync(ct);
+ var routes = new System.Collections.Generic.List();
+ while (await reader.ReadAsync(ct))
+ {
+ routes.Add(reader.GetString(0));
+ }
+
+ Assert.DoesNotContain("old_route", routes);
+ Assert.Contains("recent_route", routes);
+ }
+
+ [Fact]
+ public async Task AFlushWithNothingRecorded_WritesNoRows()
+ {
+ var baseConnectionString = ConnectionString;
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString),
+ "Set DARLING_TEST_PG to a Postgres connection string to run the V148 empty-flush pin.");
+
+ var ct = TestContext.Current.CancellationToken;
+
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+
+ var accumulator = new ReadLatencyAccumulator();
+ await accumulator.FlushAsync(connection, DateTime.UtcNow, logger: null, ct);
+
+ using var read = new NpgsqlCommand("SELECT count(*) FROM collect.read_latency", connection);
+ Assert.Equal(0L, await read.ExecuteScalarAsync(ct));
+ }
+}
diff --git a/Darling/Darling.Tests/ScheduledPurgeLaunchTests.cs b/Darling/Darling.Tests/ScheduledPurgeLaunchTests.cs
index a8d847d64d..aa2f254aff 100644
--- a/Darling/Darling.Tests/ScheduledPurgeLaunchTests.cs
+++ b/Darling/Darling.Tests/ScheduledPurgeLaunchTests.cs
@@ -41,7 +41,8 @@ public sealed class ScheduledPurgeLaunchTests
new MonitoredServerRegistryState(),
new CollectorRuntimeState(),
new WebTlsCertificateState(),
- new BaselineCache());
+ new BaselineCache(),
+ new ReadLatencyAccumulator());
[Fact]
public void ReturnsImmediately_WithoutAwaitingThePurge()
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs
index 9f80b756be..5498cd2750 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs
@@ -1092,7 +1092,12 @@ analysis hour another pass (or an MCP/web read) already computed reads no 30-day
the MCP and web hosts hand theirs. */
private readonly BaselineCache _baselineCache;
- public DarlingWorker(ILogger logger, ILoggerFactory loggerFactory, McpRuntimeState mcpState, WebRuntimeState webState, MonitoredServerRegistryState registryState, CollectorRuntimeState collectorState, WebTlsCertificateState webTlsCertState, BaselineCache baselineCache)
+ /* #4442 scope 2: the same process-lifetime accumulator DarlingWebEndpoints.MapAll and RunComposedPanelAsync
+ record into — the DI container hands this worker and the web host the SAME singleton instance, so a read
+ recorded by either side is drained by this worker's hourly flush. */
+ private readonly ReadLatencyAccumulator _readLatency;
+
+ public DarlingWorker(ILogger logger, ILoggerFactory loggerFactory, McpRuntimeState mcpState, WebRuntimeState webState, MonitoredServerRegistryState registryState, CollectorRuntimeState collectorState, WebTlsCertificateState webTlsCertState, BaselineCache baselineCache, ReadLatencyAccumulator readLatency)
{
_logger = logger;
_loggerFactory = loggerFactory;
@@ -1102,6 +1107,7 @@ public DarlingWorker(ILogger logger, ILoggerFactory loggerFactory
_collectorState = collectorState;
_webTlsCertState = webTlsCertState;
_baselineCache = baselineCache;
+ _readLatency = readLatency;
}
private sealed class ServerLoopState
@@ -7924,6 +7930,19 @@ slice may cost the collector-cost flush below. */
/* #2674: reuse the same hourly connection and budget — one aggregate row per (server, collector)
for the window, plus the accumulator's own bounded retention DELETE. */
await _collectorCost.FlushAsync(connection, DateTime.UtcNow, _logger, budget.Token);
+
+ /* #4442 scope 2: the read-latency flush runs on the SAME tick, isolated from collector_cost's own
+ failure — a bad read-latency flush must not cost the collector_cost series its hourly row, and
+ vice versa (collector_cost's own catch above already isolates it from everything ABOVE this
+ line). */
+ try
+ {
+ await _readLatency.FlushAsync(connection, DateTime.UtcNow, _logger, budget.Token);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException || !budget.IsCancellationRequested)
+ {
+ _logger.LogWarning("Read-latency flush failed; this hour's histogram rows are dropped: {Message}", ex.Message);
+ }
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
diff --git a/Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs b/Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs
index 385bf6b18a..a73b4aa6d9 100644
--- a/Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/ReadLatencyAccumulator.cs
@@ -10,6 +10,10 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Npgsql;
namespace PerformanceMonitor.Darling.Service;
@@ -156,4 +160,68 @@ internal IReadOnlyList Drain()
return rows;
}
+
+ /// Rows older than this are pruned by the flush's own bounded DELETE — the same 90 days
+ /// uses.
+ public const int RetentionDays = 90;
+
+ public const int FlushTimeoutSeconds = 120;
+
+ /* One flush: drain the buckets and write an hourly aggregate row per (surface, route, outcome), then
+ prune. ON CONFLICT adds counts, totals and per-bucket counts element-wise and takes the greater of
+ the two max_ms values — the shape a SECOND flush landing in the same hour needs (this accumulator
+ flushes on the same worker tick as collector_cost, so ordinarily there is one flush per hour, but a
+ restart mid-hour can produce two partial ones for the same key and neither may be dropped). Draining
+ (TryRemove) resets the window, so an hour with no reads writes no row rather than a zero.
+
+ One statement PER DRAINED KEY, not collector_cost's single unnest bulk insert: Npgsql's array binding
+ refuses a jagged bigint[][] parameter ("use a multidimensional array instead"), and every drained
+ key's own bucket_counts is a single flat bigint[], which binds cleanly as its own parameter. The row
+ count here is bounded by (surface × route × outcome) actually hit in the hour — at most a few hundred,
+ never one per read — so a handful of round trips is the right shape for this table's real size. */
+ private const string InsertSql = @"
+INSERT INTO collect.read_latency
+(
+ metric_time, surface, route, outcome,
+ run_count, total_ms, max_ms, bucket_counts
+)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+ON CONFLICT (metric_time, surface, route, outcome) DO UPDATE SET
+ run_count = collect.read_latency.run_count + EXCLUDED.run_count,
+ total_ms = collect.read_latency.total_ms + EXCLUDED.total_ms,
+ max_ms = GREATEST(collect.read_latency.max_ms, EXCLUDED.max_ms),
+ bucket_counts = (
+ SELECT array_agg(a + b ORDER BY ord)
+ FROM unnest(collect.read_latency.bucket_counts) WITH ORDINALITY AS x(a, ord)
+ JOIN unnest(EXCLUDED.bucket_counts) WITH ORDINALITY AS y(b, ord2) ON ord = ord2
+ )";
+
+ private const string RetentionDeleteSql = @"
+DELETE FROM collect.read_latency
+WHERE metric_time < $1";
+
+ public async Task FlushAsync(NpgsqlConnection connection, DateTime metricTimeUtc, ILogger? logger, CancellationToken cancellationToken)
+ {
+ var drained = Drain();
+
+ foreach (var row in drained)
+ {
+ await using var insert = new NpgsqlCommand(InsertSql, connection) { CommandTimeout = FlushTimeoutSeconds };
+ insert.Parameters.AddWithValue(metricTimeUtc);
+ insert.Parameters.AddWithValue(row.Surface.ToString().ToLowerInvariant());
+ insert.Parameters.AddWithValue(row.Route);
+ insert.Parameters.AddWithValue(row.Outcome.ToString().ToLowerInvariant());
+ insert.Parameters.AddWithValue((int)Math.Min(row.Count, int.MaxValue));
+ insert.Parameters.AddWithValue(row.TotalMs);
+ insert.Parameters.AddWithValue(row.MaxMs);
+ insert.Parameters.AddWithValue(row.BucketCounts);
+ await insert.ExecuteNonQueryAsync(cancellationToken);
+ }
+
+ await using var retention = new NpgsqlCommand(RetentionDeleteSql, connection) { CommandTimeout = FlushTimeoutSeconds };
+ retention.Parameters.AddWithValue(metricTimeUtc.AddDays(-RetentionDays));
+ await retention.ExecuteNonQueryAsync(cancellationToken);
+
+ logger?.LogDebug("read_latency flush wrote {Rows} surface+route+outcome rows", drained.Count);
+ }
}
diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs
index b40d01e644..30c1c85f23 100644
--- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs
+++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs
@@ -227,6 +227,7 @@ holds the ordering. */
new Migration(145, "query-store-interval-wide", V145Sql),
new Migration(146, "managed-conf-verdicts", V146Sql),
new Migration(147, "compose-statement-timeout-sixty", V147Sql),
+ new Migration(148, "read-latency", V148Sql),
};
///
@@ -2291,6 +2292,40 @@ CONSTRAINT pk_managed_conf_verdicts PRIMARY KEY (setting_name)
ALTER TABLE config.config_service ALTER COLUMN compose_statement_timeout_seconds SET DEFAULT 60;
UPDATE config.config_service SET compose_statement_timeout_seconds = 60 WHERE compose_statement_timeout_seconds = 15;";
+ ///
+ /// V148 — collect.read_latency, the hourly histogram of /api/read/* and composed-panel
+ /// read durations (#4442 scope 2), the collector_cost (V105) shape applied to reads instead of
+ /// collector runs. NOT a collector: it is INTERNAL self-telemetry, written by the worker's hourly
+ /// self-metrics sweep on the same tick as collector_cost's flush, so it is deliberately absent
+ /// from CollectorCatalog.All and from the generator-parity pins, and needs no per-table GRANT (the
+ /// collect schema's blanket GRANT … ON ALL TABLES IN SCHEMA collect covers it the moment
+ /// provisioning re-runs). Hand-written DDL, a plain table (not a hypertable — the accumulator aggregates
+ /// to one row per (surface, route, outcome) per hour before it ever reaches this table, and its own
+ /// bounded retention DELETE keeps it small, the same shape collector_cost and store_metrics
+ /// use). bucket_counts is the fixed log-scale histogram ReadLatencyAccumulator.BucketUpperBoundsMs
+ /// produces — the same array's length for every row, so two rows can be summed element-wise across a
+ /// window without knowing which release wrote which. The primary key is (metric_time, surface, route,
+ /// outcome) — the accumulator drains and flushes at most once per key per hour, and the flush's own
+ /// ON CONFLICT upsert relies on exactly this key existing twice only when two flushes land in the
+ /// same hour.
+ ///
+ private const string V148Sql = @"
+CREATE TABLE IF NOT EXISTS collect.read_latency
+(
+ metric_time timestamp NOT NULL,
+ surface text NOT NULL,
+ route text NOT NULL,
+ outcome text NOT NULL,
+ run_count integer NOT NULL,
+ total_ms bigint NOT NULL,
+ max_ms bigint NOT NULL,
+ bucket_counts bigint[] NOT NULL,
+ PRIMARY KEY (metric_time, surface, route, outcome)
+);
+
+CREATE INDEX IF NOT EXISTS idx_read_latency_time
+ ON collect.read_latency(metric_time);";
+
///
/// V2 — the service's observability store: the servers registry (upserted on every
/// successful connect) and the per-run collection_log. Column names deliberately mirror
diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs
index f0ae16f1c6..4b8057d7cd 100644
--- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs
+++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs
@@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage;
///
public static class StorageVersion
{
- public const int SchemaVersion = 147;
+ public const int SchemaVersion = 148;
}
diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs
index 13d2e183c0..f93da57ce6 100644
--- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs
+++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs
@@ -967,7 +967,11 @@ the V71 finding. */
rather than a value that could be masked by an operator override. It is not yet read by any viewer
surface, so this gate rests on the standing invariant alone. Named only in this probe line, never in
prose, per the V71 finding. */
- EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'config' AND table_name = 'config_service' AND column_name = 'compose_statement_timeout_seconds' AND column_default = '60')";
+ EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'config' AND table_name = 'config_service' AND column_name = 'compose_statement_timeout_seconds' AND column_default = '60'),
+ /* V148 (#4442 scope 2) probes the new read-latency histogram table. It is not yet read by any viewer
+ surface, so this gate rests on the standing invariant alone. Named only in this probe line, never in
+ prose, per the V71 finding. */
+ EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'read_latency')";
/// The store schema version this viewer build requires — the highest migration it knows
/// (). The connect-time gate blocks a store below this.
@@ -989,7 +993,7 @@ rather than a value that could be masked by an operator override. It is not yet
await using var reader = await command.ExecuteReaderAsync(cancellationToken);
if (await reader.ReadAsync(cancellationToken))
{
- return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122));
+ return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123));
}
return null;
@@ -1014,7 +1018,7 @@ rather than a value that could be masked by an operator override. It is not yet
/// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps
/// this in step with .
///
- internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false)
+ internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false)
{
/* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm.
A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP
@@ -1187,6 +1191,18 @@ through to the rung below and showing a spurious upgrade banner on a store that
banner stays truthful, which is the only effect the rung has on the viewer. Named only in the probe
line, not this prose, per the V71 finding: the coverage ratchet strips information_schema lines
but cannot strip a comment. */
+ /* V148 (#4442 scope 2): the hourly read-latency histogram table, and now the TOP rung, so a
+ fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to
+ the rung below and showing a spurious upgrade banner on a store that is current.
+
+ The WPF viewer runs no analysis, so no viewer read names the new table; this arm exists so the
+ version banner stays truthful, which is the only effect the rung has on the viewer. Named only
+ in the probe line, not this prose, per the V71 finding. */
+ if (hasReadLatency)
+ {
+ return 148;
+ }
+
if (hasComposeTimeoutSixty)
{
return 147;
From fd142ae2aa9eeb6f87cd1be5cc898d4e723ad36b Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 26 Sep 2026 18:26:23 -0400
Subject: [PATCH 4/6] Pin that a statement-timeout read is recorded as a
timeout for its route (#4442)
---
.../ReadLatencyWebRecordingTests.cs | 42 +++++++++++++++++++
.../DarlingWebEndpoints.cs | 18 +++++++-
2 files changed, 59 insertions(+), 1 deletion(-)
diff --git a/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
index d6d9b0e899..ebc0f60696 100644
--- a/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
+++ b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
@@ -159,4 +159,46 @@ public async Task AHandlerThatThrowsTheTimeoutShape_RecordsOneWebSample_WithOutc
// own Record call never ran for it -- this fact documents that boundary
// rather than asserting a Timeout sample that route can never produce.
}
+
+ /// #4442 gap 2: the pin the classifier's own unit tests cannot give -- a real 57014
+ /// travelling through the SAME /api/read/* dispatch loop every
+ /// production route uses (, an ONE-entry
+ /// test seam that folds in only when a test set it),
+ /// not a hand-called Record standing in for that wiring. Registered under the SAME name a real
+ /// tool never uses, so it is one more dispatch key, not a different code path.
+ [Fact]
+ public async Task ADispatchEntryThatThrowsA57014PostgresException_RecordsOneWebSample_WithOutcomeTimeout()
+ {
+ var cs = ConnectionString;
+ Assert.SkipWhen(string.IsNullOrEmpty(cs), "Set DARLING_TEST_PG to run the read-latency web-recording pin.");
+
+ var ct = TestContext.Current.CancellationToken;
+ using var connection = new NpgsqlConnection(cs);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+ await using var postgres = NpgsqlDataSource.Create(cs!);
+
+ const string routeName = "__test_statement_timeout";
+ DarlingWebEndpoints.s_testOnlyExtraDispatchEntry = (routeName, (_, _, _) =>
+ throw new PostgresException("canceling statement due to statement timeout", "ERROR", "ERROR", "57014"));
+ try
+ {
+ var readLatency = new ReadLatencyAccumulator();
+ var server = await BuildServer(postgres, readLatency);
+
+ var response = await Send(server, "/api/read/" + routeName);
+ Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
+
+ var drained = readLatency.Drain();
+ var sample = Assert.Single(drained, d => d.Surface == ReadSurface.Web && d.Route == routeName);
+ Assert.Equal(ReadOutcome.Timeout, sample.Outcome);
+ Assert.Equal(1, sample.Count);
+ }
+ finally
+ {
+ /* Cleared so a later test's BuildReadDispatch call (this static persists across the whole
+ process, like s_readLatency) never sees a leftover entry from this fact. */
+ DarlingWebEndpoints.s_testOnlyExtraDispatchEntry = null;
+ }
+ }
}
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
index 4102453812..fd4e89c678 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
@@ -79,6 +79,13 @@ public static class DarlingWebEndpoints
/// would see, since a recording failure is never a request failure.
private static ILogger? s_readLatencyLogger;
+ /// #4442 gap 2, test-only: one extra /api/read/* dispatch entry a test can register so a
+ /// real with SqlState 57014 travels through the SAME dispatch loop
+ /// every other route uses, rather than a hand-called Record standing in for the wiring. internal
+ /// and set ONLY from Darling.Tests (grep proves no production caller ever assigns it); null in every
+ /// production run, so adds nothing extra unless a test opted in.
+ internal static (string Name, ReadToolHandler Handler)? s_testOnlyExtraDispatchEntry;
+
/// The tool names deliberately absent from the /api/read/* 1:1 read surface. analyze_server
/// makes a live monitored-server connection; mute_analysis_finding writes; the analyze_*_plan family
/// is the compute-heavy plan-analysis phase-2 work; the Custom Views tools (#1599) are served by their OWN
@@ -3066,7 +3073,7 @@ private static IResult UnsupportedMediaTypeResult() =>
///
internal static IReadOnlyDictionary BuildReadDispatch(ILogger? logger = null, PostgresConfig? postgresConfig = null)
{
- return new Dictionary(StringComparer.Ordinal)
+ var dispatch = new Dictionary(StringComparer.Ordinal)
{
/* ── analysis reads (take the DarlingAnalysisService) ── */
["audit_config"] = (c, pg, an) => DarlingMcpTools.AuditConfig(an, pg, Server(c), c.RequestAborted),
@@ -3334,6 +3341,15 @@ reclamped by either validation layer. */
["get_health_parser_significant_waits"] = (c, pg, an) => DarlingMcpHealthParserTools.GetSignificantWaits(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50), as_of: AsOf(c), cancellationToken: c.RequestAborted),
["get_health_parser_system_health"] = (c, pg, an) => DarlingMcpHealthParserTools.GetSystemHealth(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50), as_of: AsOf(c), cancellationToken: c.RequestAborted),
};
+
+ /* #4442 gap 2: the test-only extra entry, added ONLY when a test set it -- never in a production
+ run, since s_testOnlyExtraDispatchEntry stays null unless Darling.Tests assigns it. */
+ if (s_testOnlyExtraDispatchEntry is { } extra)
+ {
+ dispatch[extra.Name] = extra.Handler;
+ }
+
+ return dispatch;
}
/* ─────────────────────────── response mapping ─────────────────────────── */
From a9e36ec7251c2b904e463fe38cd1a13cab458592 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 26 Sep 2026 18:33:42 -0400
Subject: [PATCH 5/6] Exempt the read-latency flush from the alerting-read
census and loosen a constructor-shape pin (#4442)
---
Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs | 8 +++++---
Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs | 2 +-
Darling/Darling.Tests/SharedBaselineCacheTests.cs | 2 +-
.../DarlingWebEndpoints.cs | 4 ++--
4 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs b/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs
index c1110ee205..2cd4a0cfa5 100644
--- a/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs
+++ b/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs
@@ -965,7 +965,7 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
/* 11th since #4215: ReadRejectedManagedConfSettingNamesAsync's catch — COUNTED,
not exempt, because a RejectedValue row is judgeable evidence for the alert, not context for its text. */
private const int WorkerCountedSites = 11;
- private const int WorkerExemptSites = 10;
+ private const int WorkerExemptSites = 11;
///
/// Counted sites tree-wide. ONE numeral with several readers rather than the same number written out at
@@ -1052,6 +1052,7 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
["CONVERTS the fault into the unreadable count"] = "a parse arm, not a read: the fleet-sweep rollup's store read is counted above it, and a document that does not parse becomes the rollup's own reportable unreadable count - the fault is evidence, not a swallow",
["Could not resolve Agent job names"] = "reads the monitored server's msdb through the host resolver, not the store - the Recently-failed-job precedent one seam over; the card degrades to the unresolved form whose raw marker keeps the gap visible, and the page still delivers",
["delivery stamp could not be written"] = "a write (#3580): the daily document was already delivered and process memory already gates it; the dropped stamp costs one re-announcement at the next restart and never a delivery - the stamp READ beside it is the read, and it is counted",
+ ["Read-latency flush failed"] = "the read-latency histogram flush writes instrumentation; losing an hour's rows can't hide an alert condition",
};
///
@@ -1232,8 +1233,9 @@ WebTlsCertificateState report the web host publishes - there is no store read to
ReadRejectedManagedConfSettingNamesAsync's catch is NOT here: a RejectedValue row is
judgeable evidence, so it moved to the counted census above instead. 31st since #4299: the
Raw Purge Over Horizon self-alert's catch, whose evidence (the readings) is handed in as a
- parameter, with the read counted in DarlingWorker. */
- Assert.Equal(31, totalExempt);
+ parameter, with the read counted in DarlingWorker. 32nd since #4442: the read-latency histogram
+ flush's catch, a telemetry write whose loss costs an hour's histogram rows and never an alert. */
+ Assert.Equal(32, totalExempt);
/* Every exemption in the table is actually used. An exemption for a message that no longer exists
is a hole this pin would otherwise keep open indefinitely — the shape that lets a real new catch
diff --git a/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
index ebc0f60696..ac4d912c87 100644
--- a/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
+++ b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
@@ -160,7 +160,7 @@ public async Task AHandlerThatThrowsTheTimeoutShape_RecordsOneWebSample_WithOutc
// rather than asserting a Timeout sample that route can never produce.
}
- /// #4442 gap 2: the pin the classifier's own unit tests cannot give -- a real 57014
+ /// #4442: the pin the classifier's own unit tests cannot give -- a real 57014
/// travelling through the SAME /api/read/* dispatch loop every
/// production route uses (, an ONE-entry
/// test seam that folds in only when a test set it),
diff --git a/Darling/Darling.Tests/SharedBaselineCacheTests.cs b/Darling/Darling.Tests/SharedBaselineCacheTests.cs
index e5968aeb84..f0b84014ad 100644
--- a/Darling/Darling.Tests/SharedBaselineCacheTests.cs
+++ b/Darling/Darling.Tests/SharedBaselineCacheTests.cs
@@ -135,7 +135,7 @@ public void TheService_HandsOneBaselineCacheToEveryHostsAnalysis()
Assert.Contains("builder.Services.AddSingleton();", program, StringComparison.Ordinal);
var worker = Code("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs");
- Assert.Contains("BaselineCache baselineCache)", worker, StringComparison.Ordinal);
+ Assert.Matches(@"BaselineCache baselineCache\s*[,)]", worker);
Assert.Contains("new DarlingAnalysisService(_postgres!, planFetcher, _logger, _baselineCache)", worker, StringComparison.Ordinal);
var mcp = Code("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingMcpHostService.cs");
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
index fd4e89c678..b49b0ede31 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWebEndpoints.cs
@@ -79,7 +79,7 @@ public static class DarlingWebEndpoints
/// would see, since a recording failure is never a request failure.
private static ILogger? s_readLatencyLogger;
- /// #4442 gap 2, test-only: one extra /api/read/* dispatch entry a test can register so a
+ /// #4442, test-only: one extra /api/read/* dispatch entry a test can register so a
/// real with SqlState 57014 travels through the SAME dispatch loop
/// every other route uses, rather than a hand-called Record standing in for the wiring. internal
/// and set ONLY from Darling.Tests (grep proves no production caller ever assigns it); null in every
@@ -3342,7 +3342,7 @@ reclamped by either validation layer. */
["get_health_parser_system_health"] = (c, pg, an) => DarlingMcpHealthParserTools.GetSystemHealth(pg, Server(c), Hours(c, 24), Rows(c, "limit", 50), as_of: AsOf(c), cancellationToken: c.RequestAborted),
};
- /* #4442 gap 2: the test-only extra entry, added ONLY when a test set it -- never in a production
+ /* #4442: the test-only extra entry, added ONLY when a test set it -- never in a production
run, since s_testOnlyExtraDispatchEntry stays null unless Darling.Tests assigns it. */
if (s_testOnlyExtraDispatchEntry is { } extra)
{
From b5e92811fbfa765e79fdbd98b4735989e87fd37e Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 26 Sep 2026 18:48:27 -0400
Subject: [PATCH 6/6] Clear the test-only read seam through a disposable scope
(#4442)
---
.../ReadLatencyWebRecordingTests.cs | 42 ++++++++++++-------
1 file changed, 27 insertions(+), 15 deletions(-)
diff --git a/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
index ac4d912c87..0fb64f7288 100644
--- a/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
+++ b/Darling/Darling.Tests/ReadLatencyWebRecordingTests.cs
@@ -179,25 +179,37 @@ public async Task ADispatchEntryThatThrowsA57014PostgresException_RecordsOneWebS
await using var postgres = NpgsqlDataSource.Create(cs!);
const string routeName = "__test_statement_timeout";
- DarlingWebEndpoints.s_testOnlyExtraDispatchEntry = (routeName, (_, _, _) =>
- throw new PostgresException("canceling statement due to statement timeout", "ERROR", "ERROR", "57014"));
- try
- {
- var readLatency = new ReadLatencyAccumulator();
- var server = await BuildServer(postgres, readLatency);
+ using var seam = new ExtraDispatchEntryScope(
+ (routeName, (_, _, _) =>
+ throw new PostgresException("canceling statement due to statement timeout", "ERROR", "ERROR", "57014")));
- var response = await Send(server, "/api/read/" + routeName);
- Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
+ var readLatency = new ReadLatencyAccumulator();
+ var server = await BuildServer(postgres, readLatency);
- var drained = readLatency.Drain();
- var sample = Assert.Single(drained, d => d.Surface == ReadSurface.Web && d.Route == routeName);
- Assert.Equal(ReadOutcome.Timeout, sample.Outcome);
- Assert.Equal(1, sample.Count);
+ var response = await Send(server, "/api/read/" + routeName);
+ Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
+
+ var drained = readLatency.Drain();
+ var sample = Assert.Single(drained, d => d.Surface == ReadSurface.Web && d.Route == routeName);
+ Assert.Equal(ReadOutcome.Timeout, sample.Outcome);
+ Assert.Equal(1, sample.Count);
+ }
+
+ ///
+ /// Sets for the scope's lifetime and
+ /// clears it on , so a later test's BuildReadDispatch call (this static
+ /// persists across the whole process, like s_readLatency) never sees a leftover entry from this
+ /// fact.
+ ///
+ private sealed class ExtraDispatchEntryScope : IDisposable
+ {
+ public ExtraDispatchEntryScope((string Name, DarlingWebEndpoints.ReadToolHandler Handler) entry)
+ {
+ DarlingWebEndpoints.s_testOnlyExtraDispatchEntry = entry;
}
- finally
+
+ public void Dispose()
{
- /* Cleared so a later test's BuildReadDispatch call (this static persists across the whole
- process, like s_readLatency) never sees a leftover entry from this fact. */
DarlingWebEndpoints.s_testOnlyExtraDispatchEntry = null;
}
}