From 05517ab62baa7fad8d28f0fb107dbfe6002c3275 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 25 Sep 2026 12:49:50 -0400 Subject: [PATCH] Pin the PgTarget anomaly and blocking tile tests to the hour, not the wall clock (#4274) Both tests derived their analysis window from a raw DateTime.UtcNow minute, so the target-local-hour tile holding the seeded spike/chain varied in size run to run. AnomalyGate.EvaluateTiles skips any tile under MinTileSamples (3), so a tile landing at 1-2 samples got excluded and the wrong tile won "worst-scoring", misreporting the peak or dragging the mean below the pinned assertion range. Anchoring the window end to the hour (:59 of the prior hour, always) keeps every scored tile a safe size regardless of wall time. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01TszxYhJJbTEh4LrZ56NYo3 --- Darling/Darling.Tests/PgTargetAnomalyTests.cs | 33 ++++++++++++++++- .../Darling.Tests/PgTargetBlockingTests.cs | 36 +++++++++++++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/Darling/Darling.Tests/PgTargetAnomalyTests.cs b/Darling/Darling.Tests/PgTargetAnomalyTests.cs index fed5d66584..f0aa24370d 100644 --- a/Darling/Darling.Tests/PgTargetAnomalyTests.cs +++ b/Darling/Darling.Tests/PgTargetAnomalyTests.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -1460,7 +1461,14 @@ public async Task TheAuroraWaitProfile_OneHotCollectionStaysQuiet_ASustainedShif var bodySucceeded = false; try { - var end = TruncateToMinutes(DateTime.UtcNow).AddMinutes(-1); + /* #4274: anchored on the hour, not on the raw minute. The spike sits at the window's LAST + collection (n == minutes, below), so the worst-scoring tile it lands in (#3653 A8 option B) + is the target-local hour ending at `end`. A raw `TruncateToMinutes(UtcNow)` anchor let that + hour's tile hold as few as 1 sample when `UtcNow`'s minute was :00 or :01 — under + AnomalyThresholds.MinTileSamples (3) — so EvaluateTiles skipped it and the worst SCORED tile + came from the flat 1,500 stretch instead, reporting current_ms_per_sec 1,500, not 3,200. See + AnchorEndUtc's doc comment for the fix and the proof matrix in PR #4274's description. */ + var end = AnchorEndUtc(); const int minutes = 31 * 24 * 60; var start = end.AddMinutes(-minutes); var windowStart = end.AddHours(-4); @@ -1589,6 +1597,29 @@ private static async Task DeleteWaitGateRowsAsync(NpgsqlConnection connection, C private static DateTime TruncateToMinutes(DateTime value) => DateTime.SpecifyKind(new DateTime(value.Ticks - (value.Ticks % TimeSpan.TicksPerMinute)), DateTimeKind.Unspecified); + private static DateTime TruncateToHour(DateTime value) => + DateTime.SpecifyKind(new DateTime(value.Ticks - (value.Ticks % TimeSpan.TicksPerHour)), DateTimeKind.Unspecified); + + /// #4274's test-only clock seam: DARLING_TEST_NOW_UTC (ISO-8601, e.g. + /// 2026-09-25T00:15:00Z) stands in for DateTime.UtcNow when set, so the wall-clock proof + /// matrix in PR #4274 can drive at chosen instants without waiting for real + /// clock minutes to land there. Unset in every normal run (CI included) — falls through to the real clock. + private static DateTime SimulatedUtcNow() + { + var raw = Environment.GetEnvironmentVariable("DARLING_TEST_NOW_UTC"); + return string.IsNullOrEmpty(raw) + ? DateTime.UtcNow + : DateTime.Parse(raw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + } + + /// #4274: the window's END, pinned to the hour instead of the raw minute. Truncating to the + /// CURRENT hour and stepping back one minute always lands on :59 of the PRIOR hour — deterministic + /// regardless of what minute (or the real clock) happens to read, unlike + /// the old TruncateToMinutes(DateTime.UtcNow).AddMinutes(-1), whose minute-of-hour varied with + /// wall-clock time and could land the spike's tile (or, in the blocking test, windowStart's tile) + /// at 1-2 samples — under AnomalyThresholds.MinTileSamples (3) — on roughly a 2-in-60 draw. + private static DateTime AnchorEndUtc() => TruncateToHour(SimulatedUtcNow()).AddMinutes(-1); + private static async Task PlantSeriesAsync(NpgsqlConnection connection, string sql, DateTime start, int spikeFrom, int deadlocksFrom, int minutes, CancellationToken ct) { using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 120 }; diff --git a/Darling/Darling.Tests/PgTargetBlockingTests.cs b/Darling/Darling.Tests/PgTargetBlockingTests.cs index 62c5748dcf..8a60d6e63e 100644 --- a/Darling/Darling.Tests/PgTargetBlockingTests.cs +++ b/Darling/Darling.Tests/PgTargetBlockingTests.cs @@ -653,7 +653,12 @@ public async Task ThirtyOneDaysOfLightBlocking_ThenAFourHourChainUnderAnIdleHead { await PgTargetFactCollectorTests.RegisterServerAsync(connection, ServerId, ServerName, "postgres", 18, ct); - var windowEnd = TruncateToMinutes(DateTime.UtcNow).AddMinutes(-1); + /* #4274: anchored on the hour, not on the raw minute — see AnchorEndUtc's doc comment. The + chain's edges start at windowStart's minute + 1 (below), so windowStart's own minute is the + window's only zero-blocked reading; a raw TruncateToMinutes(UtcNow) anchor let that reading + land in a tile large enough to drag the worst tile's mean under the pinned 2.8 floor (observed + 2.727 in CI) on some wall clocks. See the proof matrix in PR #4274's description. */ + var windowEnd = AnchorEndUtc(); var windowStart = windowEnd.AddHours(-4); var historyStart = windowStart.AddDays(-31); @@ -789,7 +794,10 @@ sessions family emits no saturation fact (context only) and the idle fact roots Assert.Equal(3, anomaly.Metadata["peak_blocked_sessions"]); /* #3653 A8 option B: avg_blocked_sessions now comes from the WORST-SCORING TILE's own mean (one target-local hour of the 4h chain), not the whole window's mean across all four hours — so it can differ slightly from - the pre-tile whole-window figure (was pinned 2.9–3.0; the worst tile's hour reads 2.8333...). */ + the pre-tile whole-window figure. #4274: with AnchorEndUtc's hour-pinned windowStart, the one tile that + would hold the window's sole zero-blocked minute (windowStart's own) is always sized 1 — below + MinTileSamples (3) — so it never scores, and every tile that does score reads a clean 3.0. The range + stays (not tightened to 3.0 exactly) because it is the pre-existing, deliberately loose assertion. */ Assert.InRange(anomaly.Metadata["avg_blocked_sessions"], 2.8, 3.0); Assert.Equal(0, anomaly.Metadata["baseline_low_quality"]); Assert.Equal(0, anomaly.Metadata["threshold_lineage"]); @@ -1184,6 +1192,30 @@ private static Fact Chain( private static DateTime TruncateToMinutes(DateTime value) => DateTime.SpecifyKind(new DateTime(value.Ticks - (value.Ticks % TimeSpan.TicksPerMinute)), DateTimeKind.Unspecified); + private static DateTime TruncateToHour(DateTime value) => + DateTime.SpecifyKind(new DateTime(value.Ticks - (value.Ticks % TimeSpan.TicksPerHour)), DateTimeKind.Unspecified); + + /// #4274's test-only clock seam: DARLING_TEST_NOW_UTC (ISO-8601, e.g. + /// 2026-09-25T00:15:00Z) stands in for DateTime.UtcNow when set, so the wall-clock proof + /// matrix in PR #4274 can drive at chosen instants without waiting for real + /// clock minutes to land there. Unset in every normal run (CI included) — falls through to the real clock. + private static DateTime SimulatedUtcNow() + { + var raw = Environment.GetEnvironmentVariable("DARLING_TEST_NOW_UTC"); + return string.IsNullOrEmpty(raw) + ? DateTime.UtcNow + : DateTime.Parse(raw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + } + + /// #4274: the window's END, pinned to the hour instead of the raw minute. Truncating to the + /// CURRENT hour and stepping back one minute always lands on :59 of the PRIOR hour — deterministic + /// regardless of what minute (or the real clock) happens to read, unlike + /// the old TruncateToMinutes(DateTime.UtcNow).AddMinutes(-1), whose minute-of-hour varied with + /// wall-clock time and could land windowStart's own tile (the one holding the window's sole + /// zero-blocked minute) at a size large enough to drag the worst tile's mean under the pinned floor, on + /// roughly a 1-in-60 draw. + private static DateTime AnchorEndUtc() => TruncateToHour(SimulatedUtcNow()).AddMinutes(-1); + /* ───────────────────────── planting ───────────────────────── */ private static async Task PlantSettingsSnapshotAsync(NpgsqlConnection connection, DateTime at, CancellationToken ct)