diff --git a/Darling/Darling.Tests/DarlingFleetReaderTests.cs b/Darling/Darling.Tests/DarlingFleetReaderTests.cs index f1c6376f5b..de9e5f86cc 100644 --- a/Darling/Darling.Tests/DarlingFleetReaderTests.cs +++ b/Darling/Darling.Tests/DarlingFleetReaderTests.cs @@ -208,9 +208,11 @@ public void FleetLastCollectionSql_IsWindowed_NotAFullHistoryScan() Assert.Contains("collection_time >= $1", sql, StringComparison.Ordinal); /* #3895: bounded is not enough — as a GROUP BY the 48 hours still aggregated every collector run of every server in them (490,940 rows on DARLING01) for one timestamp each. One probe per enabled - registry server, the window kept so a long-dark server still falls out exactly as it did. */ + registry server, the window kept. #3935 made the join LEFT: a long-dark server is reported with an + empty window rather than dropped, so the card can tell it from one never collected + (FleetCardTellsDarkFromNeverCollectedTests). */ Assert.Contains("FROM servers AS s", sql, StringComparison.Ordinal); - Assert.Contains("CROSS JOIN LATERAL", sql, StringComparison.Ordinal); + Assert.Contains("LEFT JOIN LATERAL", sql, StringComparison.Ordinal); Assert.Matches(@"ORDER BY collection_time DESC\s+LIMIT 1", sql); Assert.Contains("WHERE s.is_enabled", sql, StringComparison.Ordinal); Assert.Contains("s.server_id <> 0", sql, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs b/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs index bc35e729e2..b7b539ca96 100644 --- a/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs +++ b/Darling/Darling.Tests/FleetCardCollectionStaleNamesItsPopulationTests.cs @@ -172,6 +172,11 @@ exactly as they are in the payload. */ /// age, explode that band into the shared flags, take the flag, put it on the card. There is no error /// count in the chain, no collection_log read, and no use of the roll-up's published window — which /// is why the card's window_start / window_end never bounded it. + /// + /// #3935 routed the classification through ClassifyWindowedFreshness, which adds the + /// registration instant and one arm, Offline. It cannot produce Stale, so this flag's population is + /// unchanged; + /// holds that over the combinations. /// [Fact] public void TheMcpCard_DerivesTheFlagFromFreshnessAndNothingElse() @@ -180,7 +185,7 @@ public void TheMcpCard_DerivesTheFlagFromFreshnessAndNothingElse() "Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingFleetReader.cs"); Assert.Contains( - "var freshness = ServerHealthClassifier.ClassifyFreshness(lastCollection, now);\n" + "var freshness = ClassifyWindowedFreshness(lastCollection, registeredAt, now);\n" + " var flags = ServerCollectionStatusRules.FlagsFor(freshness);", reader, StringComparison.Ordinal); Assert.Contains("var collectionStale = flags.CollectionStale;", reader, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/FleetCardTellsDarkFromNeverCollectedTests.cs b/Darling/Darling.Tests/FleetCardTellsDarkFromNeverCollectedTests.cs new file mode 100644 index 0000000000..9bab625693 --- /dev/null +++ b/Darling/Darling.Tests/FleetCardTellsDarkFromNeverCollectedTests.cs @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.RegularExpressions; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3935: the fleet card tells a server that went dark before its last-collection window apart from one that +/// has never collected. The window is 48 hours and stays (a probe with none plans every retained chunk of +/// collection_log); a server silent for longer used to reach the card as a null, and a null is what +/// calls never collected — "Awaiting first collection", +/// banded Warning — while the WPF viewer, reading the same timestamp with no window, called it Offline. +/// +/// What decides it. The last-collection read reports every enabled registry server, with the +/// registry's created_date (the first successful connect) beside a newest collection that is null +/// when the window holds none. Registered before the window: Offline. Registered inside it: never collected, +/// exactly, because the worker collects only after connecting. These pins hold the rule, its boundary, the +/// card and the reason it produces, and the read's shape; +/// holds the seam against a store. +/// +public sealed class FleetCardTellsDarkFromNeverCollectedTests +{ + private static readonly DateTime Now = new(2026, 9, 23, 12, 0, 0, DateTimeKind.Utc); + + private static DateTime WindowStart => DarlingFleetReader.LastCollectionWindowStart(Now); + + /// + /// The windowed ladder is the shared one plus ONE arm. Every newest collection the window holds bands + /// exactly as bands it, whatever the registration + /// says; an empty window is Offline only on a registration older than the window, and keeps the shared + /// reading (never collected) on one inside it or on none at all. + /// + [Fact] + public void TheWindowedLadder_DepartsFromTheSharedOne_OnlyForAnEmptyWindowOnAnOlderRegistration() + { + var registrations = new DateTime?[] + { + null, + Now.AddDays(-30), + WindowStart.AddTicks(-1), + WindowStart, + Now.AddMinutes(-10), + }; + var collections = new DateTime?[] + { + Now.AddSeconds(-30), // fresh + Now - ServerHealthThresholds.StaleThreshold - TimeSpan.FromMinutes(1), // stale + Now - ServerHealthThresholds.OfflineThreshold - TimeSpan.FromHours(3), // dark, inside the window + WindowStart, // the window's first instant + }; + + foreach (var registeredAt in registrations) + { + foreach (var lastCollection in collections) + { + Assert.Equal( + ServerHealthClassifier.ClassifyFreshness(lastCollection, Now), + DarlingFleetReader.ClassifyWindowedFreshness(lastCollection, registeredAt, Now)); + } + } + + Assert.Equal(ServerFreshness.Offline, DarlingFleetReader.ClassifyWindowedFreshness(null, Now.AddDays(-30), Now)); + Assert.Equal(ServerFreshness.NeverCollected, DarlingFleetReader.ClassifyWindowedFreshness(null, Now.AddMinutes(-10), Now)); + Assert.Equal(ServerFreshness.NeverCollected, DarlingFleetReader.ClassifyWindowedFreshness(null, null, Now)); + } + + /// + /// The boundary is the instant the read's $1 is bound to. A server registered AT the window's start + /// could only have collected at or after it, which is inside the window the read searched (its predicate + /// is collection_time >= $1), so an empty window proves it never has. One tick earlier and a + /// collection outside the window is possible, so it is dark. + /// + [Fact] + public void TheBoundary_IsTheInstantTheReadIsBoundTo() + { + Assert.Equal(ServerFreshness.NeverCollected, DarlingFleetReader.ClassifyWindowedFreshness(null, WindowStart, Now)); + Assert.Equal(ServerFreshness.Offline, DarlingFleetReader.ClassifyWindowedFreshness(null, WindowStart.AddTicks(-1), Now)); + + /* One derivation for the bind and the rule: two days, as naive UTC (the store's convention — a + Kind=Utc value binds as timestamptz and PostgreSQL zone-shifts it against the naive column). */ + Assert.Equal(TimeSpan.FromHours(48), DarlingFleetReader.LastCollectionWindow); + Assert.Equal(new DateTime(2026, 9, 21, 12, 0, 0), WindowStart); + Assert.Equal(DateTimeKind.Unspecified, WindowStart.Kind); + } + + /// + /// A server the read did not report at all is handed the row's default, two nulls, and keeps the + /// no-claim reading. That is the lesson of the rotating false-Offlines on actively-collecting servers + /// that the reader's null fallback was written for: an ABSENCE never becomes a red Offline. Only the + /// read's positive report of an empty window can. + /// + [Fact] + public void AServerTheReadDidNotReport_KeepsTheNoClaimReading() + { + var missing = default(DarlingFleetReader.LastCollectionRow); + + Assert.Null(missing.LastCollection); + Assert.Null(missing.RegisteredAt); + Assert.Equal( + ServerFreshness.NeverCollected, + DarlingFleetReader.ClassifyWindowedFreshness(missing.LastCollection, missing.RegisteredAt, Now)); + + var card = Card(missing.LastCollection, missing.RegisteredAt); + Assert.NotEqual(FleetHealthBand.Offline, card.Band); + Assert.NotEqual(false, card.IsOnline); + } + + /// + /// The issue's card: registered weeks ago, nothing in the window. It reads Offline on every field a + /// consumer keys on — the band, the status word, is_online, the awaiting flag — and the + /// "needs attention" reason says so, where it used to say "Awaiting first collection". The window's + /// newest collection is null because there is none; status is what tells this null from the + /// never-collected one. + /// + [Fact] + public void ADarkServersCard_ReadsOffline_NotAwaitingItsFirstCollection() + { + var card = Card(null, Now.AddDays(-30)); + + Assert.Equal(FleetHealthBand.Offline, card.Band); + Assert.Equal(ServerCollectionStatus.Offline.Word(), card.Status); + Assert.False(card.IsOnline); + Assert.False(card.AwaitingFirstCollection); + Assert.False(card.CollectionStale); + Assert.Null(card.LastCollectionTime); + Assert.Equal("Offline - no recent collection", DarlingFleetReader.BuildReason(card)); + + /* And the roll-up counts it where the viewer's would: Offline, not Warning. */ + var rollup = DarlingFleetReader.BuildRollup(new[] { card }, Now, Now.AddHours(-1), Now); + Assert.Equal(1, rollup.OfflineCount); + Assert.Equal(0, rollup.WarningCount); + Assert.Equal(FleetHealthBand.Offline, Assert.Single(rollup.WorstServers).Band); + } + + /// + /// The other half, and the reason the registry is consulted at all rather than calling every empty + /// window Offline: a server the service has just connected to and not yet collected from is the bootstrap + /// state, and a red Offline on it is what sent a 24-server field report chasing a phantom scheduler bug + /// (). + /// + [Fact] + public void ANewlyRegisteredServersCard_StillReadsAwaitingFirstCollection() + { + var card = Card(null, Now.AddMinutes(-10)); + + Assert.Equal(FleetHealthBand.Warning, card.Band); + Assert.Equal(ServerCollectionStatus.AwaitingFirstCollection.Word(), card.Status); + Assert.Null(card.IsOnline); + Assert.True(card.AwaitingFirstCollection); + Assert.Equal(ServerCollectionStatus.AwaitingFirstCollection.Word(), DarlingFleetReader.BuildReason(card)); + } + + /// + /// The new arm produces Offline and nothing else, so collection_stale keeps exactly the population + /// its name claims (#3098): the Stale band, which only a collection inside the window can produce. + /// + [Fact] + public void TheCollectionStaleFlag_KeepsItsPopulation() + { + foreach (var registeredAt in new DateTime?[] { null, Now.AddDays(-30), Now.AddMinutes(-10) }) + { + foreach (var age in new TimeSpan?[] { null, TimeSpan.FromSeconds(5), ServerHealthThresholds.StaleThreshold + TimeSpan.FromMinutes(1), TimeSpan.FromHours(20) }) + { + DateTime? lastCollection = age.HasValue ? Now - age.Value : null; + Assert.Equal( + ServerCollectionStatusRules.FlagsFor(ServerHealthClassifier.ClassifyFreshness(lastCollection, Now)).CollectionStale, + Card(lastCollection, registeredAt).CollectionStale); + } + } + } + + /// + /// The read's shape: still the per-server probe #3895 made it, still windowed on the partition column, + /// but a LEFT join, so a server whose window is empty is REPORTED with a null rather than dropped, and the + /// registration rides the same row. It reads collection_log exactly once; the discriminator is a + /// registry column, not a second, unwindowed look at the history (the price the window exists to avoid). + /// + [Fact] + public void TheLastCollectionRead_ReportsEveryEnabledServer_WithItsRegistration() + { + var sql = DarlingFleetReader.FleetLastCollectionSql; + + Assert.Contains("FROM servers AS s", sql, StringComparison.Ordinal); + Assert.Contains("LEFT JOIN LATERAL", sql, StringComparison.Ordinal); + Assert.Contains(") AS latest ON TRUE", sql, StringComparison.Ordinal); + Assert.Contains("s.created_date", sql, StringComparison.Ordinal); + Assert.Contains("collection_time >= $1", sql, StringComparison.Ordinal); + Assert.Matches(@"ORDER BY collection_time DESC\s+LIMIT 1", sql); + Assert.Single(Regex.Matches(sql, @"\bv_collection_log\b")); + Assert.DoesNotContain("CROSS JOIN", sql, StringComparison.Ordinal); + Assert.DoesNotContain("GROUP BY", sql, StringComparison.Ordinal); + Assert.True(FleetReadsAreBoundedByTheFleetTests.IsBoundedByTheFleet(sql)); + + /* The reader binds the window through the same derivation the rule uses, so the window the read + searched and the window the card reasons about cannot drift apart: a bind narrower than the rule + would call a server that collected between the two "never collected". */ + var reader = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingFleetReader.cs"); + Assert.Contains("AddTimestamp(command, LastCollectionWindowStart(now));", reader, StringComparison.Ordinal); + Assert.DoesNotContain("AddHours(-48)", reader, StringComparison.Ordinal); + } + + /// A calm SQL Server card whose only variables are the two freshness inputs, assembled by the + /// reader's own . + private static FleetServerCard Card(DateTime? lastCollection, DateTime? registeredAt) => + DarlingFleetReader.BuildCard( + new DarlingFleetReader.FleetServerRow(1, "sql-01", "sql-01", 3, MonitoredEngineKind.SqlServer, false), + new DarlingFleetReader.CpuRow(20, 5), + default, + default, + default, + new DarlingFleetReader.ThreadsRow(512, 100, 0, 0), + default, + default, + default, + lastCollection, + /* Forty collectors banded, none failing: the collectors row is a measured reading (#3539 A6). */ + new DarlingFleetReader.CollectorCounts(40, 0, 40, 0), + null, + Now, + TimeSpan.FromHours(1), + DeadlockRateThresholds.Default, + registeredAt); +} diff --git a/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs b/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs index 97a33cdfc4..ca6ef97c92 100644 --- a/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs +++ b/Darling/Darling.Tests/FleetReadsAreBoundedByTheFleetTests.cs @@ -236,7 +236,9 @@ private static Dictionary FleetStatements() => /// Sentinel servers, negative ids, cleaned up in finally: two collecting SQL Servers (one with a /// ring-buffer batch whose samples share a collection instant), one registered and never collected, one dark for /// five days, one never stamped with an engine, one disabled, and one PostgreSQL target carrying a planted -/// SQL Server row it could never really have — which is how the driver's engine filter is shown to be real. +/// SQL Server row it could never really have — which is how the driver's engine filter is shown to be real. +/// Each is registered the way the service registers one, with the first-connect instant #3935's fleet card +/// reads to tell the dark server from the never-collected one. /// [Collection("live-postgres")] public sealed class FleetReadsAreBoundedLivePostgresTests @@ -356,7 +358,8 @@ WHERE server_id <> 0 /// over a week of history and every sentinel shape — including a CPU batch whose samples tie on the /// collection instant (the tiebreak must still pick the newest sample), grant pools summed at the newest /// instant, a server dark for five days (the unbounded probe must still find it; the 48-hour read must still - /// drop it), and incident rows collected late and collected ahead of their own event's clock. + /// leave its collection out, and since #3935 report it with an empty window and its registration), and + /// incident rows collected late and collected ahead of their own event's clock. /// [Fact] public async Task TheBoundedReads_ReturnTheUnboundedReadsRows_AgainstDevPostgres() @@ -437,13 +440,22 @@ collection is stamped MORE than a chunk before the event itself — a monitored Assert.Single(oldDeadlocks); Assert.Empty(newDeadlocks); - /* Last collection: the 48 hours kept, so the five-days-dark server falls out of both. */ - var lastArgs = new object[] { now.AddHours(-48) }; + /* Last collection: the 48 hours kept, so the five-days-dark server's collection is outside the window + in both. #3935: the probe REPORTS that server, and the never-collected one, with an empty window + and the registration beside it, where the oracle simply has no row; every collection the oracle + finds, the probe finds identically. */ + var lastArgs = new object[] { DarlingFleetReader.LastCollectionWindowStart(now) }; var oldLast = Scoped(await ReadRowsAsync(connection, OldLastCollectionSql, lastArgs, ct), s_compared); var newLast = Scoped(await ReadRowsAsync(connection, DarlingFleetReader.FleetLastCollectionSql, lastArgs, ct), s_compared); - Assert.Equal(oldLast, newLast); - Assert.Equal(3, newLast.Count); - Assert.DoesNotContain(newLast, r => r.StartsWith(Id(DarkFiveDays), StringComparison.Ordinal)); + Assert.Equal(3, oldLast.Count); + Assert.Equal(oldLast, newLast.Where(r => !r.Contains("|null|", StringComparison.Ordinal)).Select(WithoutLastCell).ToList()); + Assert.Equal(s_compared.Length, newLast.Count); + Assert.Contains(Id(DarkFiveDays) + "null|" + Stamp(RegisteredLongAgo(now)), newLast); + Assert.Contains(Id(NeverCollected) + "null|" + Stamp(RegisteredJustNow(now)), newLast); + Assert.Contains(Id(CollectingA) + Stamp(now.AddMinutes(-1)) + "|" + Stamp(RegisteredLongAgo(now)), newLast); + Assert.DoesNotContain( + await ReadRowsAsync(connection, DarlingFleetReader.FleetLastCollectionSql, lastArgs, ct), + r => r.StartsWith(Id(Disabled), StringComparison.Ordinal)); /* The WPF viewer's sidebar freshness: every REGISTERED server, disabled included, unbounded — so the dark and the disabled servers keep their real last collection. */ @@ -474,7 +486,9 @@ dark and the disabled servers keep their real last collection. */ /// The seam: the overview itself, through , puts the /// newest reading on each card — the newest sample of a tied batch, the pools summed at the newest instant, /// a dark server's last values — leaves a never-collected server's metrics empty, and ignores the reading - /// planted under the PostgreSQL target. + /// planted under the PostgreSQL target. And (#3935) it bands the five-days-dark server Offline and the + /// just-registered one awaiting its first collection, the same words the WPF viewer's unbounded read gives + /// them. /// [Fact] public async Task TheOverview_BuildsEachCardFromItsNewestReading_AgainstDevPostgres() @@ -523,7 +537,15 @@ public async Task TheOverview_BuildsEachCardFromItsNewestReading_AgainstDevPostg var dark = cards[DarkFiveDays]; Assert.Equal(11, dark.CpuPercent); // five days old, still its newest Assert.Equal(1100, dark.MemoryMb); - Assert.NotEqual(true, dark.IsOnline); + + /* #3935: dark for five days, registered nine days ago — Offline, the viewer's word for it, and not + "Awaiting first collection". Its last collection is outside the kept window, so the card carries + none; status is what tells this null from the never-collected one below. */ + Assert.False(dark.IsOnline); + Assert.Equal(FleetHealthBand.Offline, dark.Band); + Assert.Equal(ServerCollectionStatus.Offline.Word(), dark.Status); + Assert.False(dark.AwaitingFirstCollection); + Assert.Null(dark.LastCollectionTime); var never = cards[NeverCollected]; Assert.Null(never.CpuPercent); @@ -531,6 +553,26 @@ public async Task TheOverview_BuildsEachCardFromItsNewestReading_AgainstDevPostg Assert.Null(never.TotalThreads); Assert.Null(never.GrantedMemoryMb); + /* Registered ten minutes ago and nothing collected since: still the bootstrap state, never red. */ + Assert.Null(never.IsOnline); + Assert.True(never.AwaitingFirstCollection); + Assert.Equal(FleetHealthBand.Warning, never.Band); + Assert.Equal(ServerCollectionStatus.AwaitingFirstCollection.Word(), never.Status); + + /* And both read what the WPF viewer reads for them: its sidebar freshness has no window, and each + card's status is the word that freshness bands to. */ + await using (var viewer = new ViewerDataService(connectionString!)) + { + var viewerFreshness = await viewer.GetServerFreshnessAsync(ct); + foreach (var id in new[] { DarkFiveDays, NeverCollected, CollectingA }) + { + DateTime? viewerLast = viewerFreshness.TryGetValue(id, out var seen) ? seen : null; + Assert.Equal( + ServerCollectionStatusRules.FromFreshness(ServerHealthClassifier.ClassifyFreshness(viewerLast, now)).Word(), + cards[id].Status); + } + } + var pg = cards[PostgresTarget]; Assert.Null(pg.MemoryMb); // the planted SQL Server row is never read @@ -603,6 +645,16 @@ seeded week's older days are not among the ones it plans. */ Assert.True(flooredChunks <= unflooredChunks - 5, $"the floored count planned {flooredChunks} of the oracle's {unflooredChunks} chunks for a one-hour window:\n{floored}"); + /* #3935 kept the last-collection read's window and told a dark server from a never-collected one + with a registry column instead, because the same probe with the window taken out plans every + retained chunk of collection_log on every call. Pinned at the plan, so a fix that drops the window + goes red here: the windowed read plans only the window's chunks. */ + var windowedLast = await ExplainAsync(connection, "EXPLAIN (COSTS OFF) " + DarlingFleetReader.FleetLastCollectionSql, new object[] { DarlingFleetReader.LastCollectionWindowStart(now) }, ct); + var unwindowedLast = await ExplainAsync(connection, "EXPLAIN (COSTS OFF) " + UnwindowedLastCollectionSql(), Array.Empty(), ct); + Assert.True(ChunkScans(unwindowedLast) >= 7, $"the seeded week should put the unwindowed probe across at least seven chunks:\n{unwindowedLast}"); + Assert.True(ChunkScans(windowedLast) <= ChunkScans(unwindowedLast) - 5, + $"the last-collection read planned {ChunkScans(windowedLast)} of the unwindowed probe's {ChunkScans(unwindowedLast)} chunks for a two-day window:\n{windowedLast}"); + bodySucceeded = true; } finally @@ -649,13 +701,19 @@ public async Task TheEngineFilter_SkipsExactlyWhatTheDecoderCallsPostgres_Agains /// private static async Task SeedAsync(NpgsqlConnection connection, DateTime now, CancellationToken ct) { - await InsertServerAsync(connection, CollectingA, "fleet3895-a", true, MonitoredEngineKind.SqlServer, ct); - await InsertServerAsync(connection, CollectingB, "fleet3895-b", true, MonitoredEngineKind.SqlServer, ct); - await InsertServerAsync(connection, NeverCollected, "fleet3895-never", true, MonitoredEngineKind.SqlServer, ct); - await InsertServerAsync(connection, PostgresTarget, "fleet3895-pg", true, MonitoredEngineKind.Postgres, ct); - await InsertServerAsync(connection, Disabled, "fleet3895-off", false, MonitoredEngineKind.SqlServer, ct); - await InsertServerAsync(connection, DarkFiveDays, "fleet3895-dark", true, MonitoredEngineKind.SqlServer, ct); - await InsertServerAsync(connection, Unstamped, "fleet3895-unstamped", true, null, ct); + /* Registration the way the service writes it (#3935): created_date is the first successful connect, + and collection only follows one. Everything with history connected nine days ago, ahead of its + eight days of it; the never-collected server and the PostgreSQL fixture connected ten minutes ago + and have written no collection_log row since. */ + var longAgo = RegisteredLongAgo(now); + var justNow = RegisteredJustNow(now); + await InsertServerAsync(connection, CollectingA, "fleet3895-a", true, MonitoredEngineKind.SqlServer, longAgo, ct); + await InsertServerAsync(connection, CollectingB, "fleet3895-b", true, MonitoredEngineKind.SqlServer, longAgo, ct); + await InsertServerAsync(connection, NeverCollected, "fleet3895-never", true, MonitoredEngineKind.SqlServer, justNow, ct); + await InsertServerAsync(connection, PostgresTarget, "fleet3895-pg", true, MonitoredEngineKind.Postgres, justNow, ct); + await InsertServerAsync(connection, Disabled, "fleet3895-off", false, MonitoredEngineKind.SqlServer, longAgo, ct); + await InsertServerAsync(connection, DarkFiveDays, "fleet3895-dark", true, MonitoredEngineKind.SqlServer, longAgo, ct); + await InsertServerAsync(connection, Unstamped, "fleet3895-unstamped", true, null, longAgo, ct); /* History: every two hours from eight days back to a day back, older values that the newest must outrank — eight daily chunks on a hypertable store, and enough rows that a read which streams them @@ -741,18 +799,26 @@ private static async Task SeedHistoryAsync(NpgsqlConnection connection, int id, } } - private static async Task InsertServerAsync(NpgsqlConnection connection, int id, string name, bool enabled, string? engineKind, CancellationToken ct) + private static async Task InsertServerAsync(NpgsqlConnection connection, int id, string name, bool enabled, string? engineKind, DateTime registeredAt, CancellationToken ct) { using var command = new NpgsqlCommand( - "INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_engine_edition, engine_kind) VALUES ($1, $2, $2, $3, $4, $5)", connection); + "INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_engine_edition, engine_kind, created_date, modified_date) VALUES ($1, $2, $2, $3, $4, $5, $6, $6)", connection); command.Parameters.AddWithValue(id); command.Parameters.AddWithValue(name); command.Parameters.AddWithValue(enabled); command.Parameters.Add(new NpgsqlParameter { TypedValue = engineKind == MonitoredEngineKind.Postgres ? 0 : 3 }); command.Parameters.Add(new NpgsqlParameter { TypedValue = engineKind, NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text }); + command.Parameters.AddWithValue(DateTime.SpecifyKind(registeredAt, DateTimeKind.Unspecified)); await command.ExecuteNonQueryAsync(ct); } + /// When the servers with history first connected: nine days back, ahead of the eight days of it. + private static DateTime RegisteredLongAgo(DateTime now) => now.AddDays(-9); + + /// When the never-collected server (and the PostgreSQL fixture) first connected: ten minutes back, + /// well inside the last-collection read's two-day window. + private static DateTime RegisteredJustNow(DateTime now) => now.AddMinutes(-10); + private static Task InsertCpuAsync(NpgsqlConnection connection, int id, DateTime at, DateTime sampleTime, int sqlCpu, int otherCpu, CancellationToken ct) => ExecAsync(connection, "INSERT INTO cpu_utilization_stats (collection_id, collection_time, server_id, server_name, sample_time, sqlserver_cpu_utilization, other_process_cpu_utilization) VALUES ($1, $2, $3, 'fleet3895', $4, $5, $6)", @@ -843,6 +909,22 @@ private static List Scoped(List rows, int[] ids) => private static string Id(int id) => id.ToString(CultureInfo.InvariantCulture) + "|"; + /// An instant as renders it. + private static string Stamp(DateTime instant) => instant.ToString("O", CultureInfo.InvariantCulture); + + /// A rendered row without its last cell — the last-collection probe's registration, which the + /// oracle does not carry. + private static string WithoutLastCell(string row) => row[..row.LastIndexOf('|')]; + + /// with its window taken out — option 1 of + /// #3935, derived rather than restated so it stays that statement minus one predicate. + private static string UnwindowedLastCollectionSql() + { + var sql = Regex.Replace(DarlingFleetReader.FleetLastCollectionSql, @"\s+AND\s+collection_time >= \$1", ""); + Assert.NotEqual(DarlingFleetReader.FleetLastCollectionSql, sql); + return sql; + } + private static async Task ExplainAsync(NpgsqlConnection connection, string sql, object[] args, CancellationToken ct) { using var command = new NpgsqlCommand(sql, connection); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index ab791d2d85..ea2a4b8bae 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -208,8 +208,9 @@ WHERE s.is_enabled /// /// Falling out of this read is the correct answer, unlike /// 's 48 hours. That read's value is "when did we last see this - /// server", which a long-dark server must still report or its card drops out entirely. A CPU LEVEL is - /// the opposite: a four-hour-old reading is not this server's CPU now, and the card says so by carrying + /// server", so a long-dark server that falls out of it must still read Offline rather than never + /// collected (#3935, ). A CPU LEVEL is the opposite: a + /// four-hour-old reading is not this server's CPU now, and the card says so by carrying /// null and banding Unknown. The bound is DarlingPgCpuUtilizationReader.Freshness — three times /// the collector's own 5-minute cadence, so two missed cycles are tolerated — and it is that constant /// rather than a new number so the card and the High CPU alert agree on what "current" means. @@ -458,7 +459,8 @@ FROM sampled FROM config_alert_settings WHERE id = 1"; - /// Newest collection time per server — drives each card's freshness status. $1 window start. + /// Newest collection time per server — drives each card's freshness status. $1 window start + /// (). /// Bounded (not a bare GROUP BY over the whole table) so TimescaleDB can chunk-exclude: this table only /// grows, and every collector run adds a row, so an unbounded MAX(collection_time) over ALL history was /// re-scanning the server's ENTIRE collection archive (millions of rows) on every fleet-overview call just @@ -481,15 +483,29 @@ FROM config_alert_settings /// table (#3895). As a GROUP BY the window aggregated every collector run of every server in it /// — 490,940 rows and 103 MB on DARLING01's nine servers, for eleven timestamps — so it is now a /// per-server LIMIT 1 driven from the enabled registry, an index-only descent per server (0.25 ms - /// there; 160 ms to 1 ms on a 43-server rig). The window is KEPT, for the paragraph above: a server dark - /// for longer still falls out and reads as having no recent history, exactly as it did, and the planner - /// still excludes every chunk older than two days before it starts. + /// there; 160 ms to 1 ms on a 43-server rig). The window is KEPT, for the paragraph above, and the planner + /// still excludes every chunk older than two days before it starts. + /// + /// Nothing in the window is not "never collected" (#3935). A server dark for longer than the + /// window reached the card as a null and read "Awaiting first collection", banded Warning, while the WPF + /// viewer, whose per-server read has no window, called the same server Offline. Dropping the window would + /// have fixed the word at a price paid on every call: on DARLING01's 19 chunks the probe with no window + /// plans in 89–101 ms cold and 3.1–3.3 ms warm (8,937 and 273 planning buffers), against 9.8–13.1 ms and + /// 0.6–0.8 ms here (894 and 43), and it grows with the 60-day retention. So the window stays and the row + /// says what the window cannot: it is a LEFT JOIN, one row for EVERY enabled registry server, carrying + /// the registry's created_date — the server's first successful connect, written once by the + /// upsert — beside a newest collection that is null when the window holds none. + /// turns that pair into Offline or never-collected. A null the + /// read positively reported is the only miss that can become Offline: a server absent from the result + /// altogether (disabled between and this read) keeps the no-claim reading, + /// which is the lesson of the rotating false-Offlines this read's null fallback was written for. public const string FleetLastCollectionSql = @" SELECT s.server_id, - latest.collection_time AS last_collection_time + latest.collection_time AS last_collection_time, + s.created_date FROM servers AS s -CROSS JOIN LATERAL +LEFT JOIN LATERAL ( SELECT collection_time FROM v_collection_log @@ -497,10 +513,60 @@ FROM v_collection_log AND collection_time >= $1 ORDER BY collection_time DESC LIMIT 1 -) AS latest +) AS latest ON TRUE WHERE s.is_enabled AND s.server_id <> 0"; + /// How far back looks for a server's newest collection: two + /// days, deliberately far wider than (the statement + /// says why). + internal static readonly TimeSpan LastCollectionWindow = TimeSpan.FromHours(48); + + /// The instant 's $1 is bound to for a roll-up at + /// , as naive UTC. ONE derivation for the bind and for + /// , so the window the read searched and the window the card reasons + /// about cannot drift apart. + internal static DateTime LastCollectionWindowStart(DateTime now) => + DateTime.SpecifyKind(now - LastCollectionWindow, DateTimeKind.Unspecified); + + /// + /// A card's collection freshness from 's row (#3935): the shared + /// ladder, except for the one reading a bounded + /// read cannot hand that ladder, because to the ladder a null means "never". + /// + /// The rule. A newest collection inside the window bands exactly as it always did. With none, + /// the registry decides: a server first connected BEFORE the window began is Offline, and one first + /// connected inside it is awaiting its first collection. The second half is exact rather than a guess, + /// because the registry row is written at the first successful connect and the worker collects only after + /// connecting: a server registered inside the window could only have collected inside it, and the window + /// holds nothing. (On DARLING01 every one of the 17 registry rows, enabled or not, has its first retained + /// collection after its created_date.) The first half is the ruling: a server that connected more + /// than two days ago and has sent nothing since is dark, which is what the WPF viewer calls it from its + /// unbounded read, and "the service has not reached it yet" would be false — the service reached it, + /// which is what wrote the row. + /// + /// Where the two surfaces can still differ, and why that is the right way round. The viewer + /// reads "never" whenever it finds no row at all, so it says "Awaiting first collection" for a server whose + /// rows retention has all dropped (dark for longer than the 60-day collection_log horizon), and for + /// one that connected over two days ago and has written no collection_log row since. The second is + /// rare, because every collector run writes a row whatever its outcome, so it takes a connect no sweep + /// followed. This card reads both Offline: in each, the service reached the server and nothing has come + /// back for at least two days. + /// + /// A null registration keeps the ladder's reading. The service writes created_date on + /// every registry insert; a row without one was inserted by hand, and nothing here can say when it was + /// reached. The same holds for a server the read did not report at all, which the caller passes as two + /// nulls. + /// + /// The newest collection inside the window, or null when the window holds + /// none. + /// The registry's created_date from the same row, or null. + /// The roll-up's reference instant, the one the window was cut from. + internal static ServerFreshness ClassifyWindowedFreshness(DateTime? lastCollection, DateTime? registeredAt, DateTime now) => + !lastCollection.HasValue && registeredAt.HasValue && registeredAt.Value < LastCollectionWindowStart(now) + ? ServerFreshness.Offline + : ServerHealthClassifier.ClassifyFreshness(lastCollection, now); + /// Cross-server per-collector 7-day health aggregate — one row per (server, collector) pair carrying /// the columns the shared CollectorHealth.HealthStatus banding needs, so the caller counts each /// server's FAILING collectors exactly as the per-server Collection Health tab does. $1 window start (the @@ -829,22 +895,21 @@ and the band counts and the worst-first ranking would be derived from it. */ a PostgreSQL target is "no difference was taken" (unmeasured, not quiet), and for a SQL Server is a row BuildCard never looks at. */ pgDeadlocks.TryGetValue(server.ServerId, out var pgDeadlock); - /* Not `lastCollection.TryGetValue(..., out var lastColl)` — that leaves lastColl as - default(DateTime) (0001-01-01) on a miss, and default(DateTime) is NOT null, so it - does not hit ClassifyFreshness's NeverCollected branch: it falls through to the - age-vs-OfflineThreshold check with an age of ~2000 years and always bands Offline. - A server whose row fell outside the bounded window above (or, before that fix, was - ever missing for any other reason) must read as "no recent history", not as a fake - ancient timestamp. */ - DateTime? lastColl = lastCollection.TryGetValue(server.ServerId, out var lastCollValue) - ? lastCollValue - : null; + /* The row is a struct of NULLABLES, so a miss is two nulls rather than a value. This map used + to hold a bare DateTime, and `TryGetValue(..., out var lastColl)` then left default(DateTime) + (0001-01-01) on a miss — not null, so ClassifyFreshness skipped its NeverCollected branch, + computed an age of ~2000 years and banded Offline: the rotating false-Offlines on + actively-collecting servers the null fallback was written for. #3935 keeps that lesson: a + server the read reported with an empty window reads Offline or never-collected by its + registration (ClassifyWindowedFreshness); a server the read did not report at all keeps the + no-claim reading. */ + lastCollection.TryGetValue(server.ServerId, out var collected); failingCollectors.TryGetValue(server.ServerId, out var collectors); tags.TryGetValue(server.ServerId, out var serverTags); cards.Add(BuildCard( - server, c, pg, m, mp, t, b, deadlock, pgDeadlock, lastColl, collectors, serverTags, now, - windowEndUtc - windowStartUtc, deadlockTiers)); + server, c, pg, m, mp, t, b, deadlock, pgDeadlock, collected.LastCollection, collectors, serverTags, now, + windowEndUtc - windowStartUtc, deadlockTiers, collected.RegisteredAt)); } return BuildRollup(cards, now, windowStartUtc, windowEndUtc, worstCount, tagForest, collectionHealthAgeSeconds); @@ -867,6 +932,12 @@ ancient timestamp. */ /// structurally empty for a SQL Server and ignored for one. Two parameters rather than one pre-merged /// row so this method — the step that decides what a card CLAIMS — is where the engine picks, and a /// test can hand a card BOTH rows and assert which one it believed. + /// The newest collection inside 's + /// window, or null when it holds none. + /// The registry's created_date off the same row (#3935), which tells a + /// null apart: registered before the window is Offline, registered + /// inside it is awaiting its first collection. Trailing and defaulted to null, which keeps the shared + /// ladder's reading, so a caller that only exercises the metric rows need not invent a registration. internal static FleetServerCard BuildCard( FleetServerRow server, CpuRow cpu, @@ -882,7 +953,8 @@ internal static FleetServerCard BuildCard( List? tags, DateTime now, TimeSpan deadlockWindow, - DeadlockRateThresholds deadlockTiers) + DeadlockRateThresholds deadlockTiers, + DateTime? registeredAt = null) { /* Lite's XE-preferred / DMV-fallback, per server: XE when it has any row this window, else the DMV @@ -980,8 +1052,10 @@ because cpuForAlert there is percent of an allocation that moves. The source is /* Freshness -> the card's collection state, through the SAME mapping the WPF card and the sidebar row use (#2473). It was a hand-written copy of ApplyFreshness that happened to agree; the copy on - the sidebar row happened not to, which is the argument for none of them writing it out. */ - var freshness = ServerHealthClassifier.ClassifyFreshness(lastCollection, now); + the sidebar row happened not to, which is the argument for none of them writing it out. #3935: the + band itself comes from the shared ladder too, with the one departure a windowed read needs — an + empty window on a server registered before it is Offline, not "never collected". */ + var freshness = ClassifyWindowedFreshness(lastCollection, registeredAt, now); var flags = ServerCollectionStatusRules.FlagsFor(freshness); var isOnline = flags.IsOnline; var awaitingFirstCollection = flags.AwaitingFirstCollection; @@ -1587,19 +1661,21 @@ whose clamped deadlock differences overflow int is not a reading this card can r return map; } - private static async Task> ReadLastCollectionAsync(NpgsqlDataSource postgres, DateTime now, CancellationToken cancellationToken) + /// One per enabled registry server, INCLUDING the ones whose + /// window holds nothing (#3935): their row is the read's positive report that it looked and found none, + /// which is what needs before it may call a server Offline. + private static async Task> ReadLastCollectionAsync(NpgsqlDataSource postgres, DateTime now, CancellationToken cancellationToken) { - var map = new Dictionary(); + var map = new Dictionary(); await using var command = postgres.CreateCommand(FleetLastCollectionSql); command.CommandTimeout = McpCommandDeadlines.ReadSeconds; - AddTimestamp(command, DateTime.SpecifyKind(now.AddHours(-48), DateTimeKind.Unspecified)); + AddTimestamp(command, LastCollectionWindowStart(now)); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { - if (!reader.IsDBNull(1)) - { - map[reader.GetInt32(0)] = reader.GetDateTime(1); - } + map[reader.GetInt32(0)] = new LastCollectionRow( + reader.IsDBNull(1) ? null : reader.GetDateTime(1), + reader.IsDBNull(2) ? null : reader.GetDateTime(2)); } return map; @@ -1724,6 +1800,16 @@ internal readonly record struct PgCpuRow( internal readonly record struct ThreadsRow(int? TotalThreads, int? CurrentWorkers, int RunnableTasks, long WorkQueue); internal readonly record struct BlockingRow(int XeCount, long XeMaxWait, int DmvCount, long DmvMaxWait); internal readonly record struct DeadlockRow(int Count, DateTime? LastSeen); + + /// One server's row (#3935): its newest collection inside the + /// window and the registry's created_date, either of which can be null. Its default — two + /// nulls — is what a server the read did not report at all is handed, and it classifies exactly as a + /// null did before #3935: never collected, the amber no-claim reading, never a red Offline built out of + /// an absence. + /// The newest collection at or after , + /// or null when the window holds none. + /// The server's first successful connect, as the registry recorded it. + internal readonly record struct LastCollectionRow(DateTime? LastCollection, DateTime? RegisteredAt); /// The PostgreSQL deadlock reading (#3539): the summed counter differences, the sample that /// showed the newest step, and — how many differences the sum was taken /// over. Its default is zero intervals, which reads as unmeasured: a @@ -2046,6 +2132,13 @@ public sealed class FleetServerCard /// every collector's last run a success. /// [JsonPropertyName("collection_stale")] public bool CollectionStale { get; init; } + + /// The newest collection inside the fleet read's two-day window + /// (); null when the window holds none. A null means + /// one of two things and status says which (#3935): "Awaiting first collection" for a server + /// registered inside the window that has not collected yet, "Offline" for one that went quiet before the + /// window began. The WPF viewer and list_servers read this instant with no window, so for the + /// second kind they show the older time this card leaves out; the band agrees. [JsonPropertyName("last_collection")] public DateTime? LastCollectionTime { get; init; } /// SQL Server's OWN share of host CPU, from the ring buffer. Null on Azure SQL DB and on every diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js index 4c4a61b865..b232d16892 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/fleet.js @@ -557,9 +557,17 @@ function serverCard(c) { c.metric_count > 0 && c.measured_metric_count < c.metric_count ? " · " + c.measured_metric_count + " of " + c.metric_count + " measured" : ""; + /* #3935: an Offline card's last collection can be days old, or absent (nothing in the fleet read's + window), so it reads as an age, never as a bare time of day that implies today. */ + const lastCollect = + c.is_online === false + ? c.last_collection + ? " · last collect " + relTime(c.last_collection) + : " · no recent collection" + : " · last collect " + localClock(c.last_collection); const statusLine = c.awaiting_first_collection ? el("div", { class: "status-line awaiting", text: c.status }) - : el("div", { class: "status-line", text: c.status + " · last collect " + localClock(c.last_collection) + coverage }); + : el("div", { class: "status-line", text: c.status + lastCollect + coverage }); return el( "div", diff --git a/Darling/README.md b/Darling/README.md index 977f8fff3c..0af9f1094c 100644 --- a/Darling/README.md +++ b/Darling/README.md @@ -649,7 +649,7 @@ The embedded read-only **web dashboard** — a browser view of the monitoring st Once enabled, open `http://localhost:5153/` in a browser on the service host. Like the MCP server, `enabled`/`port` here are the file SEED; after first start they live in the control plane and the Viewer's Settings toggles them LIVE (the service starts/stops/rebinds the dashboard within seconds — no restart). If the port is already in use at startup, the web host logs an error and retries on a calm cadence; collection is unaffected. -**What you see.** The dashboard opens on a **Fleet Overview**: a card per enabled server with a status dot, six per-metric health bands (CPU, threads, memory, blocking, deadlocks, collectors), and its last collection time — all banded server-side, so the browser only renders (a server that has never reported shows an amber "Awaiting first collection", never a red offline). Above the cards a worst-first "Needs attention" list surfaces the servers to look at, or an all-healthy line when there is nothing to chase. Click a card to **drill into one server**: an overview, wait stats with a trend for the heaviest wait, active queries, a CPU chart, memory and file-I/O trends, and collection health — the same collected data the viewer shows, over inline charts. A fleet-wide **Alert History** page (with a server filter box) rounds out phase 1, a **Custom Views** page hosts the composer for the saved dashboards/notebooks in `config.custom_views`, and an **Alert Rules** page edits the user-authored custom alert rules. A **Fleet Sweeps** page ([#3466](https://github.com/erikdarlingdata/PerformanceMonitor/issues/3466)) renders the scheduled whole-fleet sweep reports as a timeline at a configurable viewing span (default the last hour): each sweep's document diffs against the previous sweep (band transitions with reasons, arrivals and departures), carries the watch-item worklist with its hysteresis position, states the alert master switch on every sweep and renders the would-have-paged ledger whenever one was taken under mute — and a sweep that could not prove its instruments were alive renders LOUDLY (a red banner and a marked timeline row), because quiet is not clean. The sweep cadence shown there is read from `get_alert_settings`' `fleet_sweep` group; the Settings window and `update_alert_settings` own the knob. Channel delivery of sweep content is bounded at a **daily rollup**, a ceiling rather than a default, per the issue's delivery ruling: at most one INFO post per day through the existing alert channels, summarizing the trailing day's band transitions, watch-item openings and closings, instrument-liveness incidents, and — whenever any covered sweep ran under the alert master switch — the would-have-paged ledger those sweeps carried, so the operator sees what the silence cost. A day with nothing to say posts nothing (the collector-cost digest's own discipline — the web feed is where the affirmative, instrument-proved all-clear lives at full cadence), and the rollup's delivery is itself master-gated: with `alerts_enabled: false` the sweeps keep publishing here and to `get_sweep_reports`, nothing reaches the channel, and the first rollup after re-enable states how many of its covered sweeps ran muted — older muted days stay readable on this page permanently rather than arriving as a re-enable flood. The HTTP API also carries the mute-rule write endpoints (`/api/mute-rules`, [#3450](https://github.com/erikdarlingdata/PerformanceMonitor/issues/3450)) — their page UI is deferred; the endpoints are the deliverable — all writes gated on the editing seat. Otherwise it is a read-only view — no settings, no live-server queries — and it refreshes every 60 seconds (pausing while the tab is hidden). The frontend ships fully self-contained (no CDN, no fonts, no remote anything), so it works on an air-gapped host with no internet access. +**What you see.** The dashboard opens on a **Fleet Overview**: a card per enabled server with a status dot, six per-metric health bands (CPU, threads, memory, blocking, deadlocks, collectors), and its last collection time — all banded server-side, so the browser only renders (a server the service has just registered and not yet collected from shows an amber "Awaiting first collection", never a red offline; one that stopped reporting reads Offline however long ago it stopped). Above the cards a worst-first "Needs attention" list surfaces the servers to look at, or an all-healthy line when there is nothing to chase. Click a card to **drill into one server**: an overview, wait stats with a trend for the heaviest wait, active queries, a CPU chart, memory and file-I/O trends, and collection health — the same collected data the viewer shows, over inline charts. A fleet-wide **Alert History** page (with a server filter box) rounds out phase 1, a **Custom Views** page hosts the composer for the saved dashboards/notebooks in `config.custom_views`, and an **Alert Rules** page edits the user-authored custom alert rules. A **Fleet Sweeps** page ([#3466](https://github.com/erikdarlingdata/PerformanceMonitor/issues/3466)) renders the scheduled whole-fleet sweep reports as a timeline at a configurable viewing span (default the last hour): each sweep's document diffs against the previous sweep (band transitions with reasons, arrivals and departures), carries the watch-item worklist with its hysteresis position, states the alert master switch on every sweep and renders the would-have-paged ledger whenever one was taken under mute — and a sweep that could not prove its instruments were alive renders LOUDLY (a red banner and a marked timeline row), because quiet is not clean. The sweep cadence shown there is read from `get_alert_settings`' `fleet_sweep` group; the Settings window and `update_alert_settings` own the knob. Channel delivery of sweep content is bounded at a **daily rollup**, a ceiling rather than a default, per the issue's delivery ruling: at most one INFO post per day through the existing alert channels, summarizing the trailing day's band transitions, watch-item openings and closings, instrument-liveness incidents, and — whenever any covered sweep ran under the alert master switch — the would-have-paged ledger those sweeps carried, so the operator sees what the silence cost. A day with nothing to say posts nothing (the collector-cost digest's own discipline — the web feed is where the affirmative, instrument-proved all-clear lives at full cadence), and the rollup's delivery is itself master-gated: with `alerts_enabled: false` the sweeps keep publishing here and to `get_sweep_reports`, nothing reaches the channel, and the first rollup after re-enable states how many of its covered sweeps ran muted — older muted days stay readable on this page permanently rather than arriving as a re-enable flood. The HTTP API also carries the mute-rule write endpoints (`/api/mute-rules`, [#3450](https://github.com/erikdarlingdata/PerformanceMonitor/issues/3450)) — their page UI is deferred; the endpoints are the deliverable — all writes gated on the editing seat. Otherwise it is a read-only view — no settings, no live-server queries — and it refreshes every 60 seconds (pausing while the tab is hidden). The frontend ships fully self-contained (no CDN, no fonts, no remote anything), so it works on an air-gapped host with no internet access. ### peers