diff --git a/Darling/Darling.Tests/AlertEngineTests.cs b/Darling/Darling.Tests/AlertEngineTests.cs
index 871bd9e1e8..9e0ce7a664 100644
--- a/Darling/Darling.Tests/AlertEngineTests.cs
+++ b/Darling/Darling.Tests/AlertEngineTests.cs
@@ -623,6 +623,128 @@ latency to it. */
Assert.Single(h.Deliverer.Outcomes);
}
+ /* ---------------- CPU: the #3744 identity rule — equality, not order ---------------- */
+
+ /// One sweep carrying one sample instant, breaching or not, with the same CPU pair the other CPU
+ /// fixtures use; the #3744 fixtures hand-pick each instant, so a shared driver would obscure the sequence.
+ private static Task ObserveCpuAsync(AlertEngine engine, DateTime sampleInstant, bool breaching) =>
+ engine.EvaluateServerAsync(Harness.Snapshot(
+ sqlCpu: breaching ? 70 : 20, totalCpu: breaching ? 95 : 30, cpuSampleTime: sampleInstant));
+
+ [Fact]
+ public async Task Cpu_AFallBack_RunsTheStampBackwards_AndTheStreakStillAdvances()
+ {
+ /* THE #3744 defect, first shape. cpu_utilization.sample_time is the monitored server's LOCAL wall
+ clock, and on a zone that observes DST that clock repeats an hour every autumn: 01:59 is followed
+ by 01:00. Under the pre-#3744 rule (fresh = strictly GREATER than the last counted instant) every
+ sample in the repeated hour read as stale, so a breach that began at 01:58 never reached the bar
+ and an open incident never cleared, for an hour, on every non-UTC server, once a year. The rule is
+ now equality — a different instant is a different sample whichever clock stamped it — and the two
+ samples one local hour apart across the fall-back are the DISTINCT identities they always were.
+
+ The instants are local-frame stamps on purpose: this is what the gate was fed before V134 and what
+ it is still fed off a pre-rung row, and the fix must hold there too, not only once the identity has
+ moved to the UTC twin. Under `>` the third observation below is "not fresh" and the fire never
+ comes; that is the assertion that reddens if the predicate goes back to order. */
+ var h = new Harness();
+ h.Settings.CpuEnabled = true;
+ var engine = h.Build();
+
+ var local = new DateTime(2026, 11, 1, 1, 58, 0); /* 01:58 EDT, Kind=Unspecified as stored */
+ await ObserveCpuAsync(engine, local, breaching: true); /* 1 */
+ await ObserveCpuAsync(engine, local.AddMinutes(1), breaching: true); /* 01:59 EDT — 2 */
+ Assert.Empty(h.Deliverer.Outcomes);
+
+ var fallenBack = new DateTime(2026, 11, 1, 1, 0, 0); /* 01:00 EST: one real minute later, 59 local minutes EARLIER */
+ Assert.True(fallenBack < local, "the fixture must run the stamp backwards, or it tests nothing #3744 changed");
+ await ObserveCpuAsync(engine, fallenBack, breaching: true); /* 3 — the bar */
+ Assert.Single(h.Deliverer.Outcomes);
+
+ /* The same rule on the falling edge: an incident open across the fall-back clears in the repeated
+ hour rather than waiting for the local clock to climb past 01:59 again. */
+ await ObserveCpuAsync(engine, fallenBack.AddMinutes(1), breaching: false);
+ Assert.Empty(h.Resolutions);
+ await ObserveCpuAsync(engine, fallenBack.AddMinutes(2), breaching: false);
+ Assert.Single(h.Resolutions);
+
+ /* And a repeat of the LAST counted instant is still the same sample: the streak holds and nothing is
+ written — equality lost none of #3282. */
+ var writes = h.StateStore.SavedPersistence.Count;
+ await ObserveCpuAsync(engine, fallenBack.AddMinutes(2), breaching: false);
+ Assert.Equal(writes, h.StateStore.SavedPersistence.Count);
+ }
+
+ [Fact]
+ public async Task Cpu_TheUpgradeFrameSwitch_EastOfUtc_IsFreshExactlyOnce_AndNeverFreezes()
+ {
+ /* THE #3744 defect, second shape — the one #3730 saw coming and worked around by leaving the gate's
+ identity on the local stamp. The identity is now the row's UTC twin where the store has one, so a
+ record persisted by a pre-#3744 build (a LOCAL instant) meets a UTC instant on the first post-upgrade
+ sweep. East of UTC the UTC instant reads EARLIER by the offset: at UTC+3, local 12:00 becomes UTC
+ 09:00. Under `>` the gate would have seen no fresh sample until the UTC clock climbed past the last
+ local stamp — three hours of frozen CPU alerting on every server east of UTC, once, on upgrade day.
+ Under equality the switch is one new identity: counted ONCE, then the same UTC instant re-read is
+ the same sample, then `!=` within the UTC frame as the samples advance.
+
+ A new engine over the same store IS the restart the upgrade implies, so the seed path is the one
+ that hands the local record to the UTC-fed sweep. */
+ var h = new Harness();
+ h.Settings.CpuEnabled = true;
+
+ /* The pre-#3744 build: one breaching sample, identity = the local stamp. */
+ var local = new DateTime(2026, 9, 21, 12, 0, 0);
+ await ObserveCpuAsync(h.Build(), local, breaching: true); /* 1 */
+ Assert.Equal(local, Assert.Single(h.StateStore.Persistence).Value.LastObservedSampleUtc);
+
+ /* The upgraded build's first sweep: the SAME sample, now identified by its UTC twin — three hours
+ earlier as a value. Fresh exactly once (the accepted one-time re-anchor), never frozen. */
+ var upgraded = h.Build();
+ var utc = local.AddHours(-3);
+ Assert.True(utc < local, "east of UTC the twin must read earlier than the local stamp, or this is the west fixture");
+ await ObserveCpuAsync(upgraded, utc, breaching: true); /* 2 */
+ Assert.Empty(h.Deliverer.Outcomes);
+
+ /* The re-read of that identity on the next sweep is the same sample: NOT counted, NOT written. If the
+ switch were treated as "fresh by frame" on every sweep rather than once, this is where the bar would
+ be reached early — the assertion that reddens if a re-anchor is not one-time. */
+ var writes = h.StateStore.SavedPersistence.Count;
+ await ObserveCpuAsync(upgraded, utc, breaching: true);
+ Assert.Empty(h.Deliverer.Outcomes);
+ Assert.Equal(writes, h.StateStore.SavedPersistence.Count);
+
+ /* Within the UTC frame the samples advance and the bar is reached where it should be. */
+ await ObserveCpuAsync(upgraded, utc.AddMinutes(1), breaching: true); /* 3 — the bar */
+ Assert.Single(h.Deliverer.Outcomes);
+ Assert.Equal(utc.AddMinutes(1), Assert.Single(h.StateStore.Persistence).Value.LastObservedSampleUtc);
+ }
+
+ [Fact]
+ public async Task Cpu_TheUpgradeFrameSwitch_WestOfUtc_CountsTheSameSampleOnceMoreAtMost()
+ {
+ /* The other direction of the same switch. West of UTC the twin reads LATER than the local stamp
+ (UTC−5: local 12:00 is UTC 17:00), so even under `>` the first post-upgrade sweep counted the
+ sample it had already counted — #3730 named that as the west-of-UTC cost of switching. Equality
+ has the same cost and the same bound, and this pins the bound: the same physical sample is counted
+ once more at the switch, the re-read of its UTC identity is held, and the streak then needs the
+ genuinely new samples it always needed. One extra count of one sample per server, once, is the
+ whole price of moving the identity to the honest clock. */
+ var h = new Harness();
+ h.Settings.CpuEnabled = true;
+
+ var local = new DateTime(2026, 9, 21, 12, 0, 0);
+ await ObserveCpuAsync(h.Build(), local, breaching: true); /* 1 */
+
+ var upgraded = h.Build();
+ var utc = local.AddHours(5);
+ await ObserveCpuAsync(upgraded, utc, breaching: true); /* 2 — the once-more */
+ await ObserveCpuAsync(upgraded, utc, breaching: true); /* held */
+ await ObserveCpuAsync(upgraded, utc, breaching: true); /* held */
+ Assert.Empty(h.Deliverer.Outcomes);
+
+ await ObserveCpuAsync(upgraded, utc.AddMinutes(1), breaching: true); /* 3 — the bar */
+ Assert.Single(h.Deliverer.Outcomes);
+ }
+
[Fact]
public async Task Cpu_TheMeasuredTwoMinuteExcursion_IsNotAnIncident()
{
diff --git a/Darling/Darling.Tests/TimeHonestyRungTests.cs b/Darling/Darling.Tests/TimeHonestyRungTests.cs
index 40d74d5c3f..87cc1c4e97 100644
--- a/Darling/Darling.Tests/TimeHonestyRungTests.cs
+++ b/Darling/Darling.Tests/TimeHonestyRungTests.cs
@@ -31,8 +31,10 @@ namespace Darling.Tests;
/// Two lies retired: a server-LOCAL sample stamp that every UTC-window reader had to de-skew by a derived
/// offset (an hour wrong across a DST transition, silently), and an OFFSET that cannot say which side of a
/// transition an instant fell on where a ZONE can. The rung, the one passthrough refresh, the viewer probe's top
-/// arm, the two windowed CPU readers that now prefer the stored UTC instant, the latest-row reads that
-/// deliberately do not, and the get_server_properties payload that publishes the clock pair.
+/// arm, the two windowed CPU readers that now prefer the stored UTC instant, the latest-row reads (three that
+/// order on the local stamp as a within-batch tiebreak and never name the twin; the CPU alert gate's, which since
+/// #3744 projects the twin as the gate's identity beside the local stamp it still orders on), and the
+/// get_server_properties payload that publishes the clock pair.
///
/// The "I am the top rung" claims this file carried when it landed (moved here off
/// PgNumbackendsAndSampledMsRungTests, V133) moved on again to LongRunningQueryExclusionKnobRungTests
@@ -283,19 +285,22 @@ public void BothWindowedCpuReads_PreferTheStoredUtcInstant_AndKeepTheDeSkewAsThe
}
///
- /// The latest-row reads keep the LOCAL sample_time and never name the twin: DarlingWorker.LatestCpuSql
- /// projects it as the CPU alert gate's observation identity, compared only against itself by >, and a
- /// one-time frame change there would freeze the gate for one offset's worth of hours on every server east of
- /// UTC (#3282's trap); the other three ORDER BY it as a within-batch tiebreak, which carries no frame. The
- /// worker's doc says so in the words a reader will look for.
+ /// The latest-row reads all keep the LOCAL sample_time as their within-batch tiebreak, and the three
+ /// display/fleet reads never name the twin at all. The CPU alert gate's read is the exception since #3744: it
+ /// PROJECTS sample_time_utc beside the local stamp (the twin is the gate's observation identity where
+ /// the row has one, the local stamp where it does not) but still does not ORDER on it, because a
+ /// cross-batch order on the twin, or on a COALESCE of the two, would compare a pre-rung local stamp
+ /// against a post-rung UTC one and sort a stale row as newest east of UTC. #3730 pinned this read to the
+ /// local stamp because the gate compared identities by > and a one-time frame change would have
+ /// frozen it for one offset's worth of hours; #3744 moved the gate to equality and this pin flipped with it.
+ /// The worker's doc says all of that in the words a reader will look for.
///
[Fact]
- public void TheLatestRowReads_KeepTheLocalStamp_AndTheWorkerSaysWhy()
+ public void TheLatestRowReads_KeepTheLocalTiebreak_TheGatesReadProjectsTheTwin_AndTheWorkerSaysWhy()
{
foreach (var sql in new[]
{
- DarlingWorker.LatestCpuSql, DarlingHealthReader.ServerSummaryCpuSql, DarlingFleetReader.FleetCpuSql,
- ViewerDataService.ServerSummaryCpuSql,
+ DarlingHealthReader.ServerSummaryCpuSql, DarlingFleetReader.FleetCpuSql, ViewerDataService.ServerSummaryCpuSql,
})
{
Assert.DoesNotContain("sample_time_utc", sql, StringComparison.Ordinal);
@@ -303,15 +308,38 @@ public void TheLatestRowReads_KeepTheLocalStamp_AndTheWorkerSaysWhy()
Assert.Contains("collection_time DESC, sample_time DESC", sql, StringComparison.Ordinal);
}
- Assert.Contains("SELECT sqlserver_cpu_utilization, other_process_cpu_utilization, sample_time", DarlingWorker.LatestCpuSql, StringComparison.Ordinal);
+ /* The gate's read: both stamps projected, local first (ordinal 2) and the twin last (ordinal 3), so the
+ pre-#3744 ordinals still read what they always read; the twin named exactly once, in the SELECT list
+ and nowhere else — not in the ORDER BY, not in a predicate. */
+ var gate = DarlingWorker.LatestCpuSql;
+ Assert.Contains("SELECT sqlserver_cpu_utilization, other_process_cpu_utilization, sample_time, sample_time_utc", gate, StringComparison.Ordinal);
+ Assert.Contains("ORDER BY collection_time DESC, sample_time DESC", gate, StringComparison.Ordinal);
+ Assert.Single(Regex.Matches(gate, "sample_time_utc"));
+ Assert.DoesNotContain("COALESCE", gate, StringComparison.Ordinal);
var worker = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs");
var doc = worker[..worker.IndexOf("internal const string LatestCpuSql", StringComparison.Ordinal)];
doc = doc[doc.LastIndexOf("/// ", StringComparison.Ordinal)..];
- foreach (var phrase in new[] { "Not sample_time_utc, deliberately", "OBSERVATION IDENTITY", "#3282", "EAST of UTC", "frozen", "TimeHonestyRungTests" })
+ foreach (var phrase in new[]
+ {
+ "the UTC twin is the identity where the row has one (#3744", "OBSERVATION IDENTITY", "#3282", "EAST of UTC",
+ "frozen", "fall-back", "EQUALITY", "The ORDER BY stays on the local stamp", "TimeHonestyRungTests",
+ })
{
Assert.Contains(phrase, doc, StringComparison.Ordinal);
}
+
+ Assert.DoesNotContain("Not sample_time_utc, deliberately", doc, StringComparison.Ordinal);
+
+ /* And the reader folds the pair the documented way — twin first, local as the fallback — rather than
+ reading only one of them. Source pin because ReadLatestCpuAsync is private and needs a live store. */
+ var reader = CSharpSourceWalker.StripCommentsAndStrings(worker);
+ var head = reader.IndexOf("ReadLatestCpuAsync(int serverId", StringComparison.Ordinal);
+ Assert.True(head >= 0, "DarlingWorker has no ReadLatestCpuAsync");
+ var body = reader.Substring(head, Math.Min(2500, reader.Length - head));
+ Assert.Contains("reader.IsDBNull(3) ? (DateTime?)null : reader.GetDateTime(3)", body, StringComparison.Ordinal);
+ Assert.Contains("reader.IsDBNull(2) ? (DateTime?)null : reader.GetDateTime(2)", body, StringComparison.Ordinal);
+ Assert.Contains("sampleTime = sampleTimeUtc ?? sampleTimeLocal;", body, StringComparison.Ordinal);
}
/// The properties read projects the clock pair after the pre-rung columns, the row type carries them
@@ -496,22 +524,29 @@ SELECT deskewed FROM (
Assert.Equal(newestUtc.AddHours(-1), (DateTime)(await deskewOnly.ExecuteScalarAsync(ct))!);
}
- /* The worker's latest-row read keeps the LOCAL stamp: the gate's identity did not change frame.
- And it shows the pre-existing fall-back quirk this rung deliberately leaves alone: the tiebreak
- orders on the local stamp, so inside the one batch that straddles the fall-back the EDT-side
- sample (cpu 31, sixty seconds OLDER) sorts as the newest — a stale-by-a-minute reading once a
- year, and an identity that the next batch's EST stamps will not exceed for an hour (reported,
- not fixed here: switching the identity to the twin would trade that for a one-time freeze of one
- offset's worth of hours east of UTC at the upgrade, #3282's trap). */
+ /* The worker's latest-row read projects BOTH stamps (#3744): the local stamp at ordinal 2, where
+ it always was, and the UTC twin at ordinal 3, which is what the gate now takes as this row's
+ identity. It also still shows the one-batch fall-back quirk the tiebreak leaves: ordering on the
+ local stamp, inside the single batch that straddles the fall-back the EDT-side sample (cpu 31,
+ sixty seconds OLDER in UTC) sorts as the newest — a reading stale by one sample, once a year.
+ Before #3744 that identity was also one the next batch's EST stamps would not EXCEED for an
+ hour, and the gate compared by >, so it froze; the gate now compares for equality, and the next
+ batch's twins simply differ from this one. The ORDER BY cannot move to the twin without
+ comparing a pre-rung local stamp against a post-rung UTC one across batches — the worker's doc
+ says why — so the one-sample staleness inside the straddling batch is the accepted residue. */
using (var latest = new NpgsqlCommand(DarlingWorker.LatestCpuSql, connection))
{
latest.Parameters.AddWithValue(ServerId);
using var reader = await latest.ExecuteReaderAsync(ct);
Assert.True(await reader.ReadAsync(ct));
+ Assert.Equal(4, reader.FieldCount);
Assert.Equal(31, reader.GetInt32(0));
Assert.Equal(straddleUtc.AddHours(-4), reader.GetDateTime(2));
+ Assert.Equal("sample_time_utc", reader.GetName(3));
+ Assert.Equal(straddleUtc, reader.GetDateTime(3));
}
+
/* server_properties through ServerPropertiesCollector.WritePayload: a 2022+ row with the zone and
an older-engine row with NULL, the offset on both. */
var older = now.AddMinutes(-10);
@@ -549,6 +584,21 @@ and the note says what NULL means. */
samples.Select(s => (s.SampleTime, s.SqlServerCpu)).ToArray());
}
+ /* Last, because it removes the post-rung batch: a store whose NEWEST CPU row predates the rung —
+ fixture 5 of #3744, today's behaviour. The gate's read returns the pre-rung row with its local
+ stamp at ordinal 2 and a NULL twin at ordinal 3, which is the reader's cue to fall back to the
+ local stamp as the identity. */
+ await DarlingMcpTestData.ExecAsync(connection, ct, "DELETE FROM cpu_utilization_stats WHERE server_id = $1 AND sample_time_utc IS NOT NULL", ServerId);
+ using (var latest = new NpgsqlCommand(DarlingWorker.LatestCpuSql, connection))
+ {
+ latest.Parameters.AddWithValue(ServerId);
+ using var reader = await latest.ExecuteReaderAsync(ct);
+ Assert.True(await reader.ReadAsync(ct));
+ Assert.Equal(22, reader.GetInt32(0));
+ Assert.Equal(pre2Utc.AddHours(-4), reader.GetDateTime(2));
+ Assert.True(reader.IsDBNull(3), "a pre-rung row must read a NULL twin, or the fallback arm is never exercised");
+ }
+
bodySucceeded = true;
}
finally
diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs
index ea76b53414..8201b8f41a 100644
--- a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs
@@ -3909,7 +3909,9 @@ degrading to (null, null) costs this tick its CPU alert and nothing else. */
IsAzureSqlDb: runtime.Target.IsAzureSqlDb,
Suppressed: false,
/* #3282: the gate counts breaching SAMPLES, not sweeps — the sweep runs every 30 s and this
- sample advances about once a minute, so without the instant a re-read would count twice. */
+ sample advances about once a minute, so without the instant a re-read would count twice.
+ #3744: the row's UTC twin where the store has one, the local stamp before V134 — resolved
+ in ReadLatestCpuAsync, compared for equality by the gate. */
CpuSampleTimeUtc: cpuSampleTime);
await engine.EvaluateServerAsync(snapshot, cancellationToken);
@@ -5460,20 +5462,34 @@ the same way a real backend id already is on its own. */
/// invert east of UTC). An ORDER BY carries no clock frame at all. Internal so the shape and the
/// same-row-across-offsets behaviour are both pinned by test.
///
- /// Not sample_time_utc, deliberately (V134, #3653 item 13). Since that rung every
- /// row carries the same instant in UTC beside the local stamp, and the windowed CPU reads prefer it. This
- /// read does not, because the value it projects is not display data and not a window bound: it is the
- /// CPU alert gate's OBSERVATION IDENTITY (#3282), compared only against the value this same read stored
- /// last sweep, by >. A frame is irrelevant to that comparison as long as it never changes — and
- /// switching the projection to the UTC column would change it exactly once, at the upgrade, so that on
- /// every server EAST of UTC the first post-rung sample (UTC) reads OLDER than the last pre-rung one (local)
- /// and the gate sees no fresh sample for one offset's worth of hours: CPU alerting frozen, silently, on
- /// the half of the world where it is morning. West of UTC the same switch would count one stale sample as
- /// fresh. The tiebreak in the ORDER BY carries no frame either way, and the local column is what every
- /// row has. Pinned by TimeHonestyRungTests.
+ /// Both stamps, and the UTC twin is the identity where the row has one (#3744; V134, #3653
+ /// item 13). Since that rung every row carries the same instant in UTC (sample_time_utc) beside
+ /// the local stamp, and the windowed CPU reads prefer it. This read projects both, and
+ /// hands the gate sample_time_utc ?? sample_time as the CPU alert
+ /// gate's OBSERVATION IDENTITY (#3282): the stored UTC instant on every row written since the rung, the
+ /// local stamp on a pre-rung row (the twin is NULL there; nothing was backfilled, for the reason V134
+ /// gives). #3730 left this read on the local stamp on purpose and wrote the trap down: the gate then
+ /// compared identities by >, so a frame that changed once at the upgrade would have read every
+ /// first post-rung UTC instant as OLDER than the last local one on every server EAST of UTC and frozen
+ /// CPU alerting for one offset's worth of hours, and counted one stale sample as fresh west of it. The same
+ /// > was also why the gate went blind for the repeated hour after every autumn fall-back on a
+ /// non-UTC server — the local clock runs backwards and every new sample read as stale. #3744 changed the
+ /// gate's test to EQUALITY (AlertEngine.ObservePersistenceAsync says why order was the wrong test):
+ /// a different instant is a different sample whichever clock stamped it, so the frame switch costs at most
+ /// one extra count of one sample per server, once, and the identity can be the honest UTC instant.
+ ///
+ /// The ORDER BY stays on the local stamp. The tiebreak orders rows INSIDE one batch, where
+ /// every row shares a frame (a poll writes the twin on every row or on none), so it carries no frame
+ /// problem; ordering on the twin, or on a COALESCE of the two, would compare a pre-rung LOCAL stamp
+ /// against a post-rung UTC one across batches and, east of UTC, sort a stale pre-rung row as newest for one
+ /// offset's worth of hours after the upgrade. What the tiebreak does leave is the one-batch fall-back quirk
+ /// TimeHonestyRungTests shows: inside the single poll that straddles the transition the EDT-side
+ /// sample sorts as newest by local stamp, a reading stale by one sample, once a year — and with the gate on
+ /// equality that costs nothing downstream, because the next batch's identities differ from it. Pinned by
+ /// TimeHonestyRungTests and LatestCpuReadShapeTests.
///
internal const string LatestCpuSql = @"
-SELECT sqlserver_cpu_utilization, other_process_cpu_utilization, sample_time
+SELECT sqlserver_cpu_utilization, other_process_cpu_utilization, sample_time, sample_time_utc
FROM cpu_utilization_stats
WHERE server_id = $1
ORDER BY collection_time DESC, sample_time DESC
@@ -5501,13 +5517,18 @@ FROM cpu_utilization_stats
{
sqlCpu = reader.IsDBNull(0) ? null : Convert.ToDouble(reader.GetValue(0), CultureInfo.InvariantCulture);
otherCpu = reader.IsDBNull(1) ? null : Convert.ToDouble(reader.GetValue(1), CultureInfo.InvariantCulture);
- /* #3282: the sample's own instant, which is the persistence gate's observation identity. Left
- Kind=Unspecified as it comes off the `timestamp` column (server-LOCAL on the ring-buffer arm,
- UTC on Azure's — a frame this compare never needs; V134's UTC twin is deliberately not read
- here, see LatestCpuSql) — it is only ever compared against the value this same read stored
- last sweep, so coercing it to Kind=Utc would shift one side of that comparison by the host's
- offset and, east of UTC, freeze the gate. */
- sampleTime = reader.IsDBNull(2) ? null : reader.GetDateTime(2);
+ /* #3282: the sample's own instant, which is the persistence gate's observation identity. #3744:
+ the row's UTC twin (V134's sample_time_utc, ordinal 3) where the store has one, the local stamp
+ (ordinal 2) on a pre-rung row — the same `??` Lite's MainWindow.AlertEngine applies to its
+ overview read's two columns, so both SKUs hand the shared gate the same identity for the same
+ row. Left Kind=Unspecified as both come off `timestamp` columns (the twin is naive UTC by the
+ store-wide convention; the local stamp is the server's wall clock on the ring-buffer arm and UTC
+ on Azure's): the value is only ever compared for EQUALITY against the one this same read stored
+ last sweep, so coercing either to Kind=Utc would buy nothing and, through Npgsql's timestamptz
+ inference on the way back into the store, could shift the persisted copy by the host's offset. */
+ var sampleTimeUtc = reader.IsDBNull(3) ? (DateTime?)null : reader.GetDateTime(3);
+ var sampleTimeLocal = reader.IsDBNull(2) ? (DateTime?)null : reader.GetDateTime(2);
+ sampleTime = sampleTimeUtc ?? sampleTimeLocal;
}
double? totalCpu = sqlCpu.HasValue ? sqlCpu.Value + (otherCpu ?? 0) : null;
diff --git a/Darling/README.md b/Darling/README.md
index f90fbcc39b..b1c7b1c16e 100644
--- a/Darling/README.md
+++ b/Darling/README.md
@@ -884,7 +884,7 @@ The notable rungs are below. For the **complete** current schema, read `Darling/
| **V129** — PostgreSQL log events | `collect.pg_log_events` + its time index (#3601): the classified server-log events — `occurred_at`, `family`, `severity`, `sqlstate`, `database_name`, `user_name`, `application_name`, `pid`, redacted `message` / `detail` / `context`, `statement_fingerprint` (a hash of the REDACTED statement; the text itself is never stored) and `raw_line_hash`, the identity every read dedupes on because the `pg_read_file` route re-reads an overlapping tail. Column-for-column what the generator emits, like every collector rung, which is why the `(server_id, family, collection_time)` index a family-filtered read over a long window would want is a SEPARATE rung when it comes — V104's reason. Thirty-day retention, from the schedule table. **V130** (#3602, #3603) widened the row rather than adding sibling tables: `relation_name`, `bytes`, `duration_ms`, `pages_removed` / `pages_remaining`, `tuples_removed` / `tuples_remaining`, `buffer_hits` / `buffer_misses` / `buffer_dirtied`, `wal_records` / `wal_bytes`, `is_analyze` — all nullable, no default, no backfill; `bytes` on a `temp_file` event, the rest on an `autovacuum` one, null on every other family. Same ALTER-plus-CREATE pairing as V101: the CREATE carries them for a fresh store, the ALTER for one that climbed through V129 first |
| **V131** — notification routes | `config.config_notification_routes` (#3598): the sparse routes that send alert FAMILIES to different channels than the parent `config_notification` row's — `metric_match` (a family name or an exact metric name), one destination per channel type under the parent's own column names (`teams_url`, `slack_url`, `generic_url`, `pagerduty_routing_key`, `smtp_recipients`; empty = inherit), a GENERATED `configured_channels` presence column so the roles denied the URLs can still say which channels a route sets, `enabled`, `modified_at`, and a V117-shaped `config_version` bump trigger so the running service hot-reloads a route with no restart. Zero rows = the pre-routes fan-out, byte for byte. The four webhook-class destinations join the fail-closed viewer/mcp secret carve; `mcp` may toggle `enabled` and DELETE, never INSERT or write a destination |
| **V133** — PostgreSQL saturation numerator and sampler duty cycle | `collect.pg_database_stats.numbackends` and `collect.pg_wait_sampling.sampled_ms` (#3691), the one rung of the v2 wave: the client backends connected per database at the instant of the read (a level, never differenced — the `max_connections` fraction's honest numerator, which `pg_session_states` could not supply because it captures only sessions holding an old open transaction), and the milliseconds of each five-minute interval the service-side wait sampler actually observed (30,000 on a full window; NULL from the extension arm, which watches the whole interval) so a rate read stops understating that arm ~10×. Both nullable, no default, no backfill, appended last, on compressed hypertables that have no `v_` view to refresh; the V83 / V96 CREATE texts carry them for a fresh store and this rung's two ALTERs for an upgraded one (the V101 rule). Written by the collectors from this rung; read by nothing yet — the consumers are follow-on lanes. Lite stores no `pg_*` table, so no DuckDB bump |
-| **V134** — time honesty | `collect.cpu_utilization_stats.sample_time_utc` and `collect.server_properties.time_zone_id` (#3653 item 13, rulings Q7 + Q8; Lite schema v63 is the twin). `cpu_utilization_stats.sample_time` is the MONITORED SERVER'S LOCAL wall clock — the one column in the store that is not naive UTC, by a documented convention (Lite plots it in the server's frame, and it is the collector's watermark) — so every reader that aligned it against UTC derived the UTC value from an offset (the #1262 per-batch de-skew here, the collected `utc_offset_minutes` on Lite), and both derivations place the samples nearest a DST transition an hour wrong, silently. The collector now writes the same instant in UTC beside the unchanged local stamp (`SYSUTCDATETIME()` through the same arithmetic; on Azure SQL DB `end_time`, which is already UTC), and the readers that window against UTC — the viewer's raw CPU read, the MCP `get_cpu_utilization`, the Lite CPU window — take `COALESCE(sample_time_utc, …)` with the old derivation as the fallback; readers that want the server's frame, and the latest-row reads whose `sample_time` is the CPU alert gate's observation identity, keep the local column. `time_zone_id` is `CURRENT_TIMEZONE_ID()`, the engine's own zone name, beside the offset V16 added — a zone can say which side of a transition an instant fell on where an offset cannot (#3231); NULL where the engine cannot say, which is every SQL Server before 2022 (the function is a missing built-in there, not a missing object, so the collector reads it in its own gated `sp_executesql` batch), and `get_server_properties` publishes the pair and says what a NULL zone means. Both nullable, no default, no backfill — the offset a row's server had at a past sample's instant is exactly what the store never recorded, and no zone can be recovered from an offset; `v_cpu_utilization_stats` is refreshed for the V14 reason, `server_properties` has no view |
+| **V134** — time honesty | `collect.cpu_utilization_stats.sample_time_utc` and `collect.server_properties.time_zone_id` (#3653 item 13, rulings Q7 + Q8; Lite schema v63 is the twin). `cpu_utilization_stats.sample_time` is the MONITORED SERVER'S LOCAL wall clock — the one column in the store that is not naive UTC, by a documented convention (Lite plots it in the server's frame, and it is the collector's watermark) — so every reader that aligned it against UTC derived the UTC value from an offset (the #1262 per-batch de-skew here, the collected `utc_offset_minutes` on Lite), and both derivations place the samples nearest a DST transition an hour wrong, silently. The collector now writes the same instant in UTC beside the unchanged local stamp (`SYSUTCDATETIME()` through the same arithmetic; on Azure SQL DB `end_time`, which is already UTC), and the readers that window against UTC — the viewer's raw CPU read, the MCP `get_cpu_utilization`, the Lite CPU window — take `COALESCE(sample_time_utc, …)` with the old derivation as the fallback; readers that want the server's frame keep the local column, and the latest-row reads keep it as their within-batch tiebreak (every row has it; an order on the twin would compare a pre-rung local stamp against a post-rung UTC one). The CPU alert gate's latest-row read kept the local stamp as the gate's observation identity at this rung because the gate compared identities by `>` and a one-time frame change would have frozen it for one offset's worth of hours east of UTC; #3744 moved the gate to equality and that read now projects the twin beside the local stamp and hands the gate `sample_time_utc ?? sample_time`. `time_zone_id` is `CURRENT_TIMEZONE_ID()`, the engine's own zone name, beside the offset V16 added — a zone can say which side of a transition an instant fell on where an offset cannot (#3231); NULL where the engine cannot say, which is every SQL Server before 2022 (the function is a missing built-in there, not a missing object, so the collector reads it in its own gated `sp_executesql` batch), and `get_server_properties` publishes the pair and says what a NULL zone means. Both nullable, no default, no backfill — the offset a row's server had at a past sample's instant is exactly what the store never recorded, and no zone can be recovered from an offset; `v_cpu_utilization_stats` is refreshed for the V14 reason, `server_properties` has no view |
| **V135** — Long-Running Query opt-out knob | `config.config_alert_settings.long_running_query_excluded_program_name_prefixes` and `.long_running_query_excluded_logins` (`text[] NOT NULL`, DEFAULT the production read's seeds — the SQL Agent job-step prefix; the two `NT AUTHORITY` service logins) — the #3653 A5/Q5 knob's store home; a pre-rung row reads as the seeds, a cleared list stays `{}` |
| **V72** — Query Store plan map | `collect.query_store_plan_map` — `(server_id, database_name, plan_id)` → digest, so Query Store facts can reference plan XML they no longer carry once that content moves into the shared `query_plan_dim`. Plan XML was stored INLINE on `query_store_stats` at roughly 5x redundancy. Not a hypertable: one row per distinct plan per database, so it is dimension-shaped and pruned on `last_seen` rather than by `drop_chunks`. Its `last_seen` is load-bearing — the dimension GC sweeps on timestamps rather than counting references, so ending the re-shipping also ends the liveness signal that used to keep those dim rows alive |
| **V73** — PostgreSQL statement text | `collect.pg_statement_text` — `(server_id, queryid)` → statement text, refreshed hourly, so `get_pg_top_queries` returns something readable (#2219). `pg_statement_stats` stores no text because `showtext` is a real per-collection cost and normalized text is highly repetitive; but `queryid` is NOT stable across a major version upgrade, so without this the stored history joins to nothing after one — a list of integers that used to be your slowest queries, unrecoverable because the live view no longer holds the old ids. Text is INLINE rather than a `query_text_dim` digest: the dimension route needs the GC liveness interlock whose failure mode is silently missing text, and inline cannot dangle. Not a hypertable and not a collector table, exactly like V72 — a bespoke upsert path, pruned on `last_seen` with a margin that makes text OUTLIVE the statistics referencing it |
diff --git a/Lite.Tests/LiteAlertForwardingTests.cs b/Lite.Tests/LiteAlertForwardingTests.cs
index 51ec6293cb..89c8cca550 100644
--- a/Lite.Tests/LiteAlertForwardingTests.cs
+++ b/Lite.Tests/LiteAlertForwardingTests.cs
@@ -416,6 +416,69 @@ public async Task Cpu_LiteStateStore_PersistsTheStreakUnderTheSharedMetricName()
Assert.NotNull(persisted.Value.LastObservedSampleUtc);
}
+ /* ---------------- CPU: the #3744 identity rule, on the Lite path ---------------- */
+
+ /// One sweep carrying one hand-picked sample instant (the #3744 fixtures run the stamp backwards,
+ /// which the increasing-instant drivers above cannot express).
+ private static Task ObserveCpuAsync(AlertEngine engine, DateTime sampleInstant, bool breaching) =>
+ engine.EvaluateServerAsync(Harness.Snapshot(
+ sqlCpu: breaching ? 70 : 20, totalCpu: breaching ? 95 : 30, cpuSampleTime: sampleInstant));
+
+ [Fact]
+ public async Task Cpu_AFallBack_RunsTheStampBackwards_AndTheStreakStillAdvances_OnLiteToo()
+ {
+ /* The Lite twin of AlertEngineTests' fall-back fixture (#3744). Lite feeds the shared gate
+ ServerSummaryItem.CpuSampleTimeUtc ?? CpuSampleTime off its overview read — the monitored server's
+ LOCAL wall clock on every pre-v63 row — and that clock repeats an hour every autumn. The gate now
+ decides "same sample" by EQUALITY of the instant rather than by order, so the sample stamped 01:00
+ after the fall-back is the new observation it is, and neither a building streak nor an open
+ incident waits an hour for the local clock to climb past 01:59 again. The arithmetic is the shared
+ engine's and is pinned in depth Darling-side; what this pins is that the Lite path reaches it
+ through its own settings, snapshot and store. */
+ DisableAllChecks();
+ App.AlertCpuEnabled = true;
+ var h = new Harness();
+ var engine = h.Build();
+
+ var local = new DateTime(2026, 11, 1, 1, 58, 0);
+ await ObserveCpuAsync(engine, local, breaching: true);
+ await ObserveCpuAsync(engine, local.AddMinutes(1), breaching: true);
+ Assert.Empty(h.Deliverer.Outcomes);
+
+ var fallenBack = new DateTime(2026, 11, 1, 1, 0, 0);
+ Assert.True(fallenBack < local, "the fixture must run the stamp backwards, or it tests nothing #3744 changed");
+ await ObserveCpuAsync(engine, fallenBack, breaching: true);
+ Assert.Single(h.Deliverer.Outcomes);
+
+ await ObserveCpuAsync(engine, fallenBack.AddMinutes(1), breaching: false);
+ await ObserveCpuAsync(engine, fallenBack.AddMinutes(2), breaching: false);
+ Assert.Single(h.Resolutions);
+ }
+
+ [Fact]
+ public async Task Cpu_TheUpgradeFrameSwitch_IsFreshExactlyOnce_OnLiteToo()
+ {
+ /* The identity moved from the local stamp to the row's UTC twin (#3744), so a record Lite persisted
+ before that meets a UTC instant on the first post-upgrade sweep — earlier than the local stamp east
+ of UTC, later west of it. Either way it is ONE new identity: counted once, its re-read held, then the
+ genuinely new samples the bar always needed. A new engine over the same store is the restart. */
+ DisableAllChecks();
+ App.AlertCpuEnabled = true;
+ var h = new Harness();
+
+ var local = new DateTime(2026, 9, 21, 12, 0, 0);
+ await ObserveCpuAsync(h.Build(), local, breaching: true);
+
+ var upgraded = h.Build();
+ var utc = local.AddHours(-3);
+ await ObserveCpuAsync(upgraded, utc, breaching: true);
+ await ObserveCpuAsync(upgraded, utc, breaching: true);
+ Assert.Empty(h.Deliverer.Outcomes);
+
+ await ObserveCpuAsync(upgraded, utc.AddMinutes(1), breaching: true);
+ Assert.Single(h.Deliverer.Outcomes);
+ }
+
[Fact]
public async Task Cpu_LiteSchemaCarriesThePersistenceTable()
{
diff --git a/Lite.Tests/TimeHonestyRungTests.cs b/Lite.Tests/TimeHonestyRungTests.cs
index 2363c5f492..99bf87b022 100644
--- a/Lite.Tests/TimeHonestyRungTests.cs
+++ b/Lite.Tests/TimeHonestyRungTests.cs
@@ -114,6 +114,42 @@ public void BothMcpServerPropertiesTools_PublishTheClockPair_InTheSameWords()
Assert.Contains(phrase, darling, StringComparison.Ordinal);
}
}
+
+ ///
+ /// #3744, the Lite half of the CPU alert gate's identity: the overview read projects the UTC twin BESIDE the
+ /// local stamp (trailing, so the pre-#3744 ordinals read what they always read), still orders on the local
+ /// stamp (every row has it; a cross-row order on the twin or a COALESCE would compare a pre-rung local stamp
+ /// against a post-rung UTC one), carries both onto ServerSummaryItem under names that say which clock
+ /// each is, and MainWindow.AlertEngine folds them twin ?? local into the snapshot — the same rule
+ /// Darling's ReadLatestCpuAsync applies, pinned there by Darling.Tests/TimeHonestyRungTests. Source
+ /// pins because the snapshot construction sits inside a WPF window.
+ ///
+ [Fact]
+ public void TheOverviewRead_ProjectsTheTwinBesideTheLocalStamp_AndTheSnapshotFoldsThem()
+ {
+ var overview = Lite.Tests.ParitySource.ReadFile("Lite/Services/LocalDataService.Overview.cs");
+ Assert.Contains(
+ "SELECT sqlserver_cpu_utilization, other_process_cpu_utilization, sample_time, collection_time, sample_time_utc\nFROM v_cpu_utilization_stats",
+ overview.Replace("\r\n", "\n", StringComparison.Ordinal), StringComparison.Ordinal);
+ Assert.Contains("ORDER BY sample_time DESC\nLIMIT 1", overview.Replace("\r\n", "\n", StringComparison.Ordinal), StringComparison.Ordinal);
+ Assert.DoesNotContain("ORDER BY sample_time_utc", overview, StringComparison.Ordinal);
+ Assert.DoesNotContain("ORDER BY COALESCE", overview, StringComparison.Ordinal);
+ Assert.Contains("cpuSampleTimeUtc = reader.IsDBNull(4) ? null : reader.GetDateTime(4);", overview, StringComparison.Ordinal);
+ Assert.Contains("CpuSampleTimeUtc = cpuSampleTimeUtc,", overview, StringComparison.Ordinal);
+ Assert.Contains("#3744", overview, StringComparison.Ordinal);
+
+ Assert.Equal(typeof(DateTime?), typeof(ServerSummaryItem).GetProperty("CpuSampleTime")!.PropertyType);
+ Assert.Equal(typeof(DateTime?), typeof(ServerSummaryItem).GetProperty("CpuSampleTimeUtc")!.PropertyType);
+
+ var window = Lite.Tests.ParitySource.ReadFile("Lite/MainWindow.AlertEngine.cs");
+ Assert.Contains("CpuSampleTimeUtc: summary.CpuSampleTimeUtc ?? summary.CpuSampleTime);", window, StringComparison.Ordinal);
+ Assert.Contains("#3744", window, StringComparison.Ordinal);
+
+ /* And the shared gate's side of the contract, in the words both hosts rely on: identity by EQUALITY. */
+ var engine = Lite.Tests.ParitySource.ReadFile("PerformanceMonitor.Alerting/AlertEngine.cs");
+ Assert.Contains("|| sampleUtc.Value != priorRecord.LastObservedSampleUtc.Value;", engine, StringComparison.Ordinal);
+ Assert.DoesNotContain("|| sampleUtc.Value > priorRecord.LastObservedSampleUtc.Value;", engine, StringComparison.Ordinal);
+ }
}
///
@@ -239,6 +275,41 @@ window on the wrong clock and fail. */
}
}
+ ///
+ /// #3744: the overview read that feeds the CPU alert gate carries BOTH stamps of the newest row. A pre-rung row
+ /// (NULL twin) surfaces its local stamp and a null CpuSampleTimeUtc — the host then hands the gate the
+ /// local stamp, today's behaviour (fixture 5); a post-rung row surfaces both, and the twin is what the gate
+ /// gets. Newest by the LOCAL stamp, which every row has.
+ ///
+ [Fact]
+ public async Task TheOverviewRead_CarriesBothStampsOfTheNewestCpuRow()
+ {
+ var now = Truncate(DateTime.UtcNow);
+
+ /* Newest row pre-rung: local stamp only. */
+ var utcPre = now.AddMinutes(-2);
+ await SeedCpuAsync(_serverId, "TimeHonestySrv", utcPre.AddMinutes(CollectedOffset), null, 41);
+
+ var summary = await _dataService.GetServerSummaryAsync(_serverId, "TimeHonestySrv");
+ Assert.NotNull(summary);
+ Assert.Equal(41, summary!.CpuPercent);
+ Assert.Equal(utcPre.AddMinutes(CollectedOffset), summary.CpuSampleTime);
+ Assert.Null(summary.CpuSampleTimeUtc);
+
+ /* A newer post-rung row: both stamps, and the twin is the honest instant. */
+ var utcPost = now.AddMinutes(-1);
+ await SeedCpuAsync(_serverId, "TimeHonestySrv", utcPost.AddMinutes(CollectedOffset), utcPost, 42);
+
+ summary = await _dataService.GetServerSummaryAsync(_serverId, "TimeHonestySrv");
+ Assert.NotNull(summary);
+ Assert.Equal(42, summary!.CpuPercent);
+ Assert.Equal(utcPost.AddMinutes(CollectedOffset), summary.CpuSampleTime);
+ Assert.Equal(utcPost, summary.CpuSampleTimeUtc);
+
+ /* The fold the snapshot applies (MainWindow.AlertEngine, pinned as source above): twin where present. */
+ Assert.Equal(utcPost, summary.CpuSampleTimeUtc ?? summary.CpuSampleTime);
+ }
+
/// The picker branch (server-local from/to) converts its bounds back to UTC for the twin exactly
/// as GetTimeRange does, so a post-rung row is selected by its instant there too.
[Fact]
diff --git a/Lite/MainWindow.AlertEngine.cs b/Lite/MainWindow.AlertEngine.cs
index 3614c61cca..6126e9e7ce 100644
--- a/Lite/MainWindow.AlertEngine.cs
+++ b/Lite/MainWindow.AlertEngine.cs
@@ -85,8 +85,12 @@ only a dictionary lookup. */
Suppressed: suppressPopups,
/* #3282: the gate counts breaching CPU SAMPLES rather than sweeps. Lite's sweep is the
30-second overview timer and the ring-buffer sample behind CpuPercent advances about once a
- minute, so without the instant one sample would fill the streak on its own. */
- CpuSampleTimeUtc: summary.CpuSampleTime);
+ minute, so without the instant one sample would fill the streak on its own.
+ #3744: the row's UTC twin where the store has one (every row since v63), the local stamp on a
+ pre-rung row — the same `??` Darling's ReadLatestCpuAsync applies, so the shared gate gets one
+ identity per row on both SKUs. The gate compares it for equality, so the one-time local→UTC
+ switch at the upgrade reads as a single new sample rather than a freeze. */
+ CpuSampleTimeUtc: summary.CpuSampleTimeUtc ?? summary.CpuSampleTime);
AlertSweepResult sweep;
try
diff --git a/Lite/Services/LocalDataService.Overview.cs b/Lite/Services/LocalDataService.Overview.cs
index 7d33f65432..44146078f4 100644
--- a/Lite/Services/LocalDataService.Overview.cs
+++ b/Lite/Services/LocalDataService.Overview.cs
@@ -27,6 +27,7 @@ public partial class LocalDataService
double? cpuPercent = null;
double? otherProcessCpuPercent = null;
DateTime? cpuSampleTime = null;
+ DateTime? cpuSampleTimeUtc = null;
DateTime? cpuCollectionTime = null;
double? memoryMb = null;
DateTime? memoryCollectionTime = null;
@@ -35,11 +36,19 @@ public partial class LocalDataService
DateTime? lastCollection = null;
/* Latest CPU — read both SQL Server CPU and other-process CPU so the UI can surface
- total non-idle CPU alongside the SQL-only number. */
+ total non-idle CPU alongside the SQL-only number.
+
+ #3744: both stamps. sample_time is the monitored server's LOCAL wall clock (the chart's frame and
+ this ordering's key); sample_time_utc is the same instant in UTC on every row written since v63
+ (#3730) and NULL before. The alert gate's identity is the twin where the row has one, resolved at
+ the snapshot (MainWindow.AlertEngine) exactly as Darling resolves its own read. The ORDER BY stays
+ on the local stamp on purpose: every row has it, and ordering on the twin or a COALESCE of the two
+ would compare a pre-rung local stamp against a post-rung UTC one and, east of UTC, sort a stale
+ pre-rung row as newest for one offset's worth of hours after the upgrade. */
using (var cmd = connection.CreateCommand())
{
cmd.CommandText = @"
-SELECT sqlserver_cpu_utilization, other_process_cpu_utilization, sample_time, collection_time
+SELECT sqlserver_cpu_utilization, other_process_cpu_utilization, sample_time, collection_time, sample_time_utc
FROM v_cpu_utilization_stats
WHERE server_id = $1
ORDER BY sample_time DESC
@@ -61,6 +70,10 @@ as cpu_captured_at. sample_time above is the monitored server's local wall clock
gate's identity; collection_time is the instant the monitor stored it, comparable with every
other captured_at on the MCP surface. */
cpuCollectionTime = reader.IsDBNull(3) ? null : reader.GetDateTime(3);
+ /* #3744: the UTC twin, NULL on a pre-v63 row. Kept beside the local stamp rather than folded
+ into it so the summary says which clock each value is in; the alert snapshot does the
+ folding (twin ?? local) where the gate's identity is built. */
+ cpuSampleTimeUtc = reader.IsDBNull(4) ? null : reader.GetDateTime(4);
}
}
@@ -132,6 +145,7 @@ FROM v_collection_log
CpuPercent = cpuPercent,
OtherProcessCpuPercent = otherProcessCpuPercent,
CpuSampleTime = cpuSampleTime,
+ CpuSampleTimeUtc = cpuSampleTimeUtc,
CpuCollectionTime = cpuCollectionTime,
MemoryMb = memoryMb,
MemoryCollectionTime = memoryCollectionTime,
@@ -288,13 +302,24 @@ public class ServerSummaryItem
public double? OtherProcessCpuPercent { get; set; }
///
- /// The sample_time of the CPU reading came from (#3282) — the shared
- /// engine's persistence-gate observation identity, NOT display data. Distinct from
+ /// The sample_time of the CPU reading came from (#3282) — the monitored
+ /// server's LOCAL wall clock, and the shared engine's persistence-gate observation identity on a row that
+ /// has no (#3744). NOT display data. Distinct from
/// , which is the newest collection of anything and is what the
/// freshness band is computed from.
///
public DateTime? CpuSampleTime { get; set; }
+ ///
+ /// The same sample's instant in UTC — the row's sample_time_utc twin (Lite v63, #3730), null on a
+ /// row written before that rung. Where present it is the gate's observation identity in place of
+ /// (#3744); MainWindow.AlertEngine folds the pair as twin ?? local
+ /// when it builds the snapshot, the same rule Darling's ReadLatestCpuAsync applies, so both SKUs hand
+ /// the shared gate the same identity for the same row. Two properties rather than one pre-folded value so
+ /// this summary never holds a timestamp without saying which clock it is in.
+ ///
+ public DateTime? CpuSampleTimeUtc { get; set; }
+
/// The store's collection_time for the CPU row came from (#3541
/// A10) — the stamp get_server_summary publishes as cpu_captured_at. UTC, comparable with every other
/// captured_at; is the monitored server's local clock and the gate's identity.
diff --git a/PerformanceMonitor.Alerting/AlertEngine.cs b/PerformanceMonitor.Alerting/AlertEngine.cs
index af347db210..2f7997a635 100644
--- a/PerformanceMonitor.Alerting/AlertEngine.cs
+++ b/PerformanceMonitor.Alerting/AlertEngine.cs
@@ -765,7 +765,9 @@ instant the SAME sample would advance the streak on consecutive sweeps and CpuBr
be reached inside 90 seconds — shorter than every excursion #3282 measured, i.e. the defect intact
behind a gate that looked like it fixed it. The fresh-sample rule, the null-instant fallback and
the skip-a-write-when-nothing-moved rule live in ObservePersistenceAsync since #3653 (A5) put a
- second arm behind the same gate. */
+ second arm behind the same gate. The instant is the row's UTC twin where the store has one and the
+ target's local stamp before V134 (#3744); the gate compares it for EQUALITY, so which clock stamped
+ it is not a gate input — see ObservePersistenceAsync for why order was the wrong test. */
var (outcome, incidentOpen) = await ObservePersistenceAsync(
_cpuPersistence, CpuPersistenceMetric, key, snapshot.CpuSampleTimeUtc, breaching,
CpuBreachSamples, CpuClearSamples);
@@ -894,11 +896,40 @@ await NotifyResolutionAsync(new AlertResolution(
/// every five at the light tier). Without the SAME row would advance the
/// streak on consecutive sweeps and a K-sample bar would be reached inside K × 30 s of wall clock
/// regardless of how many samples were actually taken — the measured flap intact behind a gate that
- /// looked like it fixed it. A sample instant that has not moved past
+ /// looked like it fixed it. A sample instant EQUAL to
/// is therefore not an observation at all:
/// the streak holds, nothing is written, and the caller still learns whether an incident is open so the
/// standing-condition reminder can run on a sweep that brought no new sample.
///
+ /// "Not counted yet" is decided by EQUALITY, not by order (#3744). Until #3744 the rule was
+ /// sampleUtc > LastObservedSampleUtc, and the difference between the two predicates is exactly
+ /// the set of observations whose instant reads OLDER than the one recorded. Every caller hands this method
+ /// the identity of the NEWEST stored row (each read is an ORDER BY … DESC LIMIT 1 or a
+ /// MAX(collection_time) over a table its collector only ever appends to — at the store's clock, or
+ /// above its watermark), so between two sweeps that identity can stay equal
+ /// (the same row — not fresh) or change to a different row's; the only way it can change to an EARLIER
+ /// value is a clock running backwards behind the stamp, and both real cases of that are new samples that
+ /// MUST count. The first is the autumn fall-back: cpu_utilization_stats.sample_time is the monitored
+ /// server's LOCAL wall clock, which repeats an hour once a year, so under > every sample in the
+ /// repeated hour read as stale and the CPU gate neither advanced a streak nor cleared an incident for that
+ /// hour on every non-UTC server — at the one instant nobody is watching. The second is the frame change at
+ /// the V134 upgrade (#3730): the CPU reads now prefer the row's UTC twin (sample_time_utc) as the
+ /// identity, so a record persisted by a pre-#3744 build holds a LOCAL instant and the first post-upgrade
+ /// sweep hands over a UTC one — east of UTC that reads older by the offset, and > would have frozen
+ /// the gate for one offset's worth of hours on the half of the world where it is morning, which is why
+ /// #3730 left the identity on the local stamp and wrote the trap down. Equality is frame-blind: a
+ /// different instant is a different observation whichever clock stamped it, so the frame switch costs at
+ /// most ONE extra count of one sample per server, once (the same sample read as local 12:00 and then as
+ /// UTC 17:00 is two identities — the accepted one-time re-anchor), and a fall-back costs nothing at all.
+ /// No persisted frame tag was needed for that, which matters because
+ /// config.alert_persistence_state cannot grow a column without a rung. The residual failure was
+ /// weighed and is stated: should a NEWEST row ever be deleted from under the gate (nothing does that —
+ /// retention removes the oldest), the read would fall back to an older row, equality would count it once,
+ /// and the streak would run ONE sample ahead, where > would have held it until a newer row landed.
+ /// One extra count in a path that does not exist against an hour of silence in one that runs every year is
+ /// not a close call. Pinned in both directions by AlertEngineTests' fall-back and upgrade fixtures
+ /// (fresh) and its repeated-instant fixture (not fresh).
+ ///
/// A null sample instant counts every sweep instead (see
/// and
/// for which hosts supply none): the persistence is then weaker, but the alert still fires, and silence
@@ -918,7 +949,11 @@ await NotifyResolutionAsync(new AlertResolution(
///
/// The per-server record cache for this metric, seeded from the store at startup.
/// The (server, metric) key the record is persisted under.
- /// The observation's sample instant, or null for a host that has none.
+ ///
+ /// The observation's sample instant — the NEWEST stored row's identity, compared for equality only — or
+ /// null for a host that has none. Naive UTC for every arm today, except the CPU arm on a row written before
+ /// V134 (the target's local clock; see ).
+ ///
/// The edge this observation produced, and whether the subject is firing after it.
private async Task<(PersistenceOutcome Outcome, bool IncidentOpen)> ObservePersistenceAsync(
ConcurrentDictionary records, string metricName, string key,
@@ -926,9 +961,11 @@ await NotifyResolutionAsync(new AlertResolution(
{
var priorRecord = records.TryGetValue(key, out var cached) ? cached : AlertPersistenceRecord.Initial;
+ /* != and not > — a different instant is a different sample whichever clock stamped it; the summary
+ above says why order was the wrong test (#3744). */
bool freshSample = !sampleUtc.HasValue
|| !priorRecord.LastObservedSampleUtc.HasValue
- || sampleUtc.Value > priorRecord.LastObservedSampleUtc.Value;
+ || sampleUtc.Value != priorRecord.LastObservedSampleUtc.Value;
if (!freshSample)
{
diff --git a/PerformanceMonitor.Alerting/AlertPersistenceRecord.cs b/PerformanceMonitor.Alerting/AlertPersistenceRecord.cs
index fe61eb1328..251a27e632 100644
--- a/PerformanceMonitor.Alerting/AlertPersistenceRecord.cs
+++ b/PerformanceMonitor.Alerting/AlertPersistenceRecord.cs
@@ -21,8 +21,11 @@ namespace PerformanceMonitor.Alerting;
/// looking at the same observation twice. Counting sweeps would therefore satisfy "three consecutive breaches"
/// inside a 90-second excursion, which is shorter than the excursions #3282 measured and would leave the
/// defect in place while appearing to fix it. is what makes the count mean
-/// samples: an observation whose sample instant has not advanced past it is not counted at all, and the streak
-/// simply holds.
+/// samples: an observation whose sample instant IS it is the same sample seen again, is not counted at all,
+/// and the streak simply holds. Sameness is decided by equality, not by order (#3744): the instant is an
+/// IDENTITY, and a different one is a different sample whichever clock stamped it — see
+/// AlertEngine.ObservePersistenceAsync for why "strictly greater" was the wrong test (it read every
+/// sample in a fall-back's repeated hour, and the first sample after a clock-frame change, as stale).
///
/// A record struct so the whole answer moves as ONE value — a caller cannot advance the counters and
/// forget the sample instant, which would make every subsequent re-read look fresh.
@@ -32,6 +35,13 @@ namespace PerformanceMonitor.Alerting;
/// The sample instant of the newest observation already counted into , or null when
/// this subject has counted none yet (a fresh subject, or a host that supplies no sample instant — see
/// for what that degrades to).
+/// Naive UTC, as every gated arm's identity is — with one stated exception the name does not show: the
+/// CPU arm's identity is the row's sample_time_utc twin only where the store has one (every row written
+/// since Darling V134 / Lite v63), and the target's LOCAL sample_time on a row written before that, so
+/// a record persisted by a pre-#3744 build, or off a pre-rung row, holds a local instant under this name. The
+/// store cannot say which (config.alert_persistence_state has no frame column and this wave added no
+/// rung), and it does not need to: the value is only ever compared for EQUALITY against the next read's
+/// identity, so the frame is not load-bearing and a frame change reads as one new sample, once.
///
public readonly record struct AlertPersistenceRecord(PersistenceState State, DateTime? LastObservedSampleUtc)
{
diff --git a/PerformanceMonitor.Alerting/AlertServerSnapshot.cs b/PerformanceMonitor.Alerting/AlertServerSnapshot.cs
index d47e8d14d8..fd9037db0d 100644
--- a/PerformanceMonitor.Alerting/AlertServerSnapshot.cs
+++ b/PerformanceMonitor.Alerting/AlertServerSnapshot.cs
@@ -40,11 +40,23 @@ namespace PerformanceMonitor.Alerting;
/// SqlEngineEdition != 5 call-site gate.
///
///
-/// The sample_time of the CPU reading /
-/// came from, or null when the host has no sample instant for it. This is the CPU check's persistence-gate
+/// The instant of the CPU sample / came
+/// from, or null when the host has no sample instant for it. This is the CPU check's persistence-gate
/// observation identity (#3282), not display data: the gate counts consecutive breaching SAMPLES, and the
/// alert sweep is twice as fast as the samples arrive, so without it a re-read of one sample would count as
/// a second observation of the condition.
+/// Which clock (#3744). The row's sample_time_utc twin where the store has one — every
+/// cpu_utilization_stats row written since Darling V134 / Lite v63 (#3730), so on any live store the
+/// name is true within one collector poll of the upgrade — and otherwise the row's sample_time, which
+/// on the ring-buffer arm is the MONITORED SERVER'S LOCAL wall clock (CollectorTimestampFrameTests pins
+/// that frame). Both hosts resolve it the same way, sample_time_utc ?? sample_time, at the read: Darling
+/// in DarlingWorker.ReadLatestCpuAsync, Lite in MainWindow.AlertEngine off the overview read's
+/// two columns. There is no frame field beside this one, deliberately: the gate compares identities for
+/// EQUALITY (AlertEngine.ObservePersistenceAsync), so which clock stamped the value is not an input
+/// to any decision the engine makes — the local→UTC switch at the upgrade reads as one new sample, once, and
+/// the fall-back's repeated hour as an hour of new samples, exactly as it should. A field nothing reads is a
+/// field that drifts. Before #3744 the name was a lie on every row (the value was always the local stamp);
+/// it is now true wherever the store can say the instant, and documented-local where it cannot.
/// No default, so every host states it. A null degrades the gate to counting SWEEPS rather than
/// samples — weaker persistence, but the alert still fires, which is the correct direction for a monitoring
/// product where silence is indistinguishable from health. The opposite default (treat an unknown instant as