From 89819093fbef106c45cbd1fb7c45100bd799b893 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:16:06 -0400 Subject: [PATCH 1/4] Decode the loader status instead of printing a raw Win32 number (#2186) A managed-Postgres bootstrap that could not start reported "initdb failed (exit code -1073741515) for ... Output:" and nothing else. -1073741515 is 0xC0000135, STATUS_DLL_NOT_FOUND: Windows killed the process in the loader before a line of its own code ran, which is also why Output: was blank and always would be for this class of failure. DarlingToolExitCode decodes an NTSTATUS exit code into its name, separates loader failures from crashes, names the two causes that account for nearly all of them (the bundled MSVC runtime absent from pg-runtime\pgsql\bin, or a service account that cannot read a user-profile install tree) and the two checks that tell them apart. An empty capture now says it is empty BECAUSE the process was killed before it could write. Wired into every bundled-binary failure: initdb (bootstrap and upgrade), pg_ctl start/status/stop/reload, pg_upgrade and its --check, and the post-upgrade analyze. Two messages also stopped blaming the wrong thing -- pg_ctl status no longer calls the data directory unusable when the verdict came from Windows rather than from pg_ctl, and pg_upgrade --check no longer reports incompatible clusters when pg_upgrade never ran to form an opinion. The #1738 refusal now quotes and decodes the probe exit code it already had instead of asserting that the binaries did not run without saying how it knew. Co-Authored-By: Claude Fable 5 --- .../DarlingManagedPostgresTests.cs | 123 ++++++++++ .../Darling.Tests/DarlingToolExitCodeTests.cs | 228 ++++++++++++++++++ .../DarlingManagedPostgres.cs | 95 ++++++-- .../DarlingStoreUpgrade.cs | 43 +++- .../DarlingToolExitCode.cs | 194 +++++++++++++++ 5 files changed, 650 insertions(+), 33 deletions(-) create mode 100644 Darling/Darling.Tests/DarlingToolExitCodeTests.cs create mode 100644 Darling/PerformanceMonitor.Darling.Service/DarlingToolExitCode.cs diff --git a/Darling/Darling.Tests/DarlingManagedPostgresTests.cs b/Darling/Darling.Tests/DarlingManagedPostgresTests.cs index f273a033ac..dc21f079a3 100644 --- a/Darling/Darling.Tests/DarlingManagedPostgresTests.cs +++ b/Darling/Darling.Tests/DarlingManagedPostgresTests.cs @@ -11,6 +11,7 @@ using System.IO; using System.Net; using System.Net.Sockets; +using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Threading; @@ -906,4 +907,126 @@ private static void TryDeleteRecursive(string path) } } } + + /* ============ #2186: the bootstrap's failure messages, in the operator's words ============ + These pin the SHIPPED strings, not the decoder — DarlingToolExitCodeTests owns the decode. + The distinction is the whole point: #1738 was a correct check that nothing invoked, and a + correct decoder no message calls would be the same defect wearing a new hat. */ + + private const int StatusDllNotFound = unchecked((int)0xC0000135); + private const string FieldBinDirectory = @"C:\PerformanceMonitorDarling\pg-runtime\pgsql\bin"; + private const string FieldDataDirectory = @"C:\ProgramData\PerformanceMonitorDarling\pg"; + + /// + /// The reported failure, rebuilt from the field's own numbers: exit -1073741515 and an empty capture. + /// Every clause the report was missing has to be present, and the clause it HAD has to survive — field + /// reports and the issue tracker are searchable by "initdb failed (exit code", so the fix must not + /// rename the thing operators paste into search. + /// + [Fact] + public void InitDbFailureMessage_TurnsTheFieldReportIntoADiagnosis() + { + var message = DarlingManagedPostgres.BuildInitDbFailureMessage( + -1073741515, Path.Combine(FieldBinDirectory, "initdb.exe"), FieldDataDirectory, string.Empty); + + Assert.StartsWith("initdb failed (exit code -1073741515", message, StringComparison.Ordinal); + Assert.Contains(FieldDataDirectory, message, StringComparison.Ordinal); + + /* What the number means, and that Windows rather than PostgreSQL set it. */ + Assert.Contains("0xC0000135", message, StringComparison.Ordinal); + Assert.Contains("STATUS_DLL_NOT_FOUND", message, StringComparison.Ordinal); + + /* The empty field is stated as expected. The report's "Output:" trailing a blank line is what made + the whole thing read as missing data. */ + Assert.Contains("Output:", message, StringComparison.Ordinal); + Assert.DoesNotContain("Output:\n\n", message, StringComparison.Ordinal); + Assert.DoesNotContain("Output:\n(none)", message, StringComparison.Ordinal); + Assert.Contains("expected", message, StringComparison.OrdinalIgnoreCase); + + /* Both causes, and the directory the DLLs are supposed to be in. */ + Assert.Contains(FieldBinDirectory, message, StringComparison.Ordinal); + Assert.Contains("vcruntime140_1.dll", message, StringComparison.Ordinal); + Assert.Contains("NT SERVICE", message, StringComparison.Ordinal); + } + + /// An ordinary initdb failure is left to speak for itself: its own stderr is the diagnosis, + /// and it must not be pushed below a screen of loader boilerplate that does not apply. + [Fact] + public void InitDbFailureMessage_LeavesARealInitDbErrorAlone() + { + const string stderr = "initdb: error: directory \"C:\\pg\" exists but is not empty"; + + var message = DarlingManagedPostgres.BuildInitDbFailureMessage( + 1, Path.Combine(FieldBinDirectory, "initdb.exe"), FieldDataDirectory, stderr); + + Assert.Equal($"initdb failed (exit code 1) for {FieldDataDirectory}.\nOutput:\n{stderr}", message); + } + + /// + /// pg_ctl status's own exit 4 really does mean the data directory is unusable, and the message keeps + /// saying so. A Windows status means pg_ctl never ran — keeping the verdict there would point an + /// operator at deleting a healthy store to fix a missing DLL. + /// + [Fact] + public void StatusFailureMessage_BlamesTheDataDirectoryOnlyWhenPgCtlActuallySaidSo() + { + var pgCtl = Path.Combine(FieldBinDirectory, "pg_ctl.exe"); + + var pgCtlVerdict = DarlingManagedPostgres.BuildStatusFailureMessage(4, pgCtl, FieldDataDirectory, "pg_ctl: could not open ..."); + Assert.Contains("the data directory is not usable", pgCtlVerdict, StringComparison.Ordinal); + + var loaderFailure = DarlingManagedPostgres.BuildStatusFailureMessage(StatusDllNotFound, pgCtl, FieldDataDirectory, string.Empty); + Assert.DoesNotContain("the data directory is not usable", loaderFailure, StringComparison.Ordinal); + Assert.Contains("STATUS_DLL_NOT_FOUND", loaderFailure, StringComparison.Ordinal); + } + + /// + /// The start failure's log tail has the initdb message's trap in another costume: on a loader status + /// pg_ctl never started a postmaster, so "(no server log written)" is true and useless. The diagnosis + /// has to arrive BEFORE the tail invites an operator to go read a log that was never going to exist. + /// + [Fact] + public void StartFailureMessage_DiagnosesBeforeItPointsAtAnEmptyServerLog() + { + var message = DarlingManagedPostgres.BuildStartFailureMessage( + StatusDllNotFound, Path.Combine(FieldBinDirectory, "pg_ctl.exe"), FieldDataDirectory, "(no server log written)"); + + var diagnosis = message.IndexOf("STATUS_DLL_NOT_FOUND", StringComparison.Ordinal); + var tail = message.IndexOf("Server log tail:", StringComparison.Ordinal); + + Assert.True(diagnosis >= 0, "the start failure must decode a Windows status"); + Assert.True(tail > diagnosis, "the loader diagnosis has to precede the server-log tail it explains"); + } + + /// + /// The wiring, pinned at the source: three correct builders that no throw site calls would leave the + /// shipped message exactly as it was reported. Behavioral coverage cannot reach these — reproducing + /// them needs a bundled Postgres that dies in the Windows loader, which is not something a CI runner + /// can be asked to arrange. + /// + [Fact] + public void TheBootstrapThrowSitesActuallyUseTheseMessages() + { + var source = ReadManagedPostgresSource(); + + Assert.Contains("throw new InvalidOperationException(BuildInitDbFailureMessage(exitCode, initDb, _dataDirectory, output));", source, StringComparison.Ordinal); + Assert.Contains("throw new InvalidOperationException(BuildStatusFailureMessage(exitCode, pgCtl, _dataDirectory, output)),", source, StringComparison.Ordinal); + Assert.Contains("BuildStartFailureMessage(exitCode, pgCtl, _dataDirectory, ReadServerLogTail())", source, StringComparison.Ordinal); + + /* And that no bootstrap failure went back to interpolating the bare code. */ + Assert.DoesNotContain("exit code {exitCode}", source, StringComparison.Ordinal); + } + + private static string ReadManagedPostgresSource([CallerFilePath] string thisFile = "") + { + var relative = Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingManagedPostgres.cs"); + var dir = Path.GetDirectoryName(thisFile); + while (dir is not null && !File.Exists(Path.Combine(dir, relative))) + { + dir = Path.GetDirectoryName(dir); + } + + Assert.False(dir is null, "could not locate the repo root from the test source path"); + return File.ReadAllText(Path.Combine(dir!, relative)); + } } diff --git a/Darling/Darling.Tests/DarlingToolExitCodeTests.cs b/Darling/Darling.Tests/DarlingToolExitCodeTests.cs new file mode 100644 index 0000000000..8b4cc5dd8b --- /dev/null +++ b/Darling/Darling.Tests/DarlingToolExitCodeTests.cs @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Darling.Service; +using Xunit; + +namespace Darling.Tests; + +/// +/// The bundled-Postgres tool exit-code decoder (#2186). The bug it exists for was reported twice from +/// the field: a managed bootstrap failed with initdb failed (exit code -1073741515) ... Output: +/// and nothing else — a signed Win32 integer plus an empty field, on the one class of failure where the +/// empty field is guaranteed rather than informative, because the process was killed in the LOADER +/// before it could write a line. The operator's attention then went to the follow-on missing-credential +/// message and darling.json, neither of which was the fault. +/// +/// These pin the decode itself. The pins that the SHIPPED messages carry it live in +/// — a correct decoder nothing calls is exactly the shape of +/// the defect #1738 already was. +/// +public sealed class DarlingToolExitCodeTests +{ + /* The four loader statuses, as .NET reports them from Process.ExitCode (signed). */ + private const int StatusDllNotFound = unchecked((int)0xC0000135); + private const int StatusEntryPointNotFound = unchecked((int)0xC0000139); + private const int StatusInvalidImageFormat = unchecked((int)0xC000007B); + private const int StatusDllInitFailed = unchecked((int)0xC0000142); + private const int StatusAccessDenied = unchecked((int)0xC0000022); + private const int StatusAccessViolation = unchecked((int)0xC0000005); + private const int StatusNoMemory = unchecked((int)0xC0000017); + + private const string InitDb = @"C:\PerformanceMonitorDarling\pg-runtime\pgsql\bin\initdb.exe"; + + /// + /// The literal number from the field report decodes. Written against the DECIMAL the operator + /// actually saw rather than a hex constant, so this test fails if the two ever stop being the same + /// number — that equality is the entire premise of the fix. + /// + [Fact] + public void Describe_DecodesTheExactCodeTheFieldReported() + { + Assert.Equal(StatusDllNotFound, -1073741515); + + var described = DarlingToolExitCode.Describe(-1073741515); + + Assert.Contains("-1073741515", described, StringComparison.Ordinal); + Assert.Contains("0xC0000135", described, StringComparison.Ordinal); + Assert.Contains("STATUS_DLL_NOT_FOUND", described, StringComparison.Ordinal); + } + + [Theory] + [InlineData(StatusEntryPointNotFound, "0xC0000139", "STATUS_ENTRYPOINT_NOT_FOUND")] + [InlineData(StatusInvalidImageFormat, "0xC000007B", "STATUS_INVALID_IMAGE_FORMAT")] + [InlineData(StatusDllInitFailed, "0xC0000142", "STATUS_DLL_INIT_FAILED")] + [InlineData(StatusAccessDenied, "0xC0000022", "STATUS_ACCESS_DENIED")] + [InlineData(StatusAccessViolation, "0xC0000005", "STATUS_ACCESS_VIOLATION")] + public void Describe_NamesTheOtherStatusesWorthNaming(int exitCode, string hex, string name) + { + var described = DarlingToolExitCode.Describe(exitCode); + + Assert.Contains(hex, described, StringComparison.Ordinal); + Assert.Contains(name, described, StringComparison.Ordinal); + } + + /// + /// An NTSTATUS this decoder has no name for still gets the half that matters most: that the number + /// is a WINDOWS status, not the program's own exit code, plus the hex to search for. Decoding only + /// the codes on a list would leave the next unfamiliar one exactly as opaque as -1073741515 was. + /// + [Fact] + public void Describe_StillSaysWindowsKilledItForAnUnlistedStatus() + { + var described = DarlingToolExitCode.Describe(StatusNoMemory); + + Assert.Contains("0xC0000017", described, StringComparison.Ordinal); + Assert.Contains("Windows", described, StringComparison.Ordinal); + } + + /// + /// A tool's OWN exit code is left completely alone. initdb exits 1 on a bad option and pg_ctl status + /// exits 3 for "not running"; dressing those up as Windows statuses would be a new lie in place of + /// the old one. + /// + [Theory] + [InlineData(0)] + [InlineData(1)] + [InlineData(3)] + [InlineData(4)] + [InlineData(127)] + public void Describe_LeavesAnOrdinaryExitCodeAsTheBareNumber(int exitCode) + { + Assert.Equal(exitCode.ToString(System.Globalization.CultureInfo.InvariantCulture), DarlingToolExitCode.Describe(exitCode)); + } + + /// + /// The diagnosis names BOTH causes the issue asks for and both checks that separate them, and it + /// points at the directory the DLLs must be in rather than at "the install" in the abstract. + /// + [Fact] + public void Diagnose_NamesBothCausesAndBothChecks() + { + var diagnosis = DarlingToolExitCode.Diagnose(StatusDllNotFound, InitDb); + + /* Cause 1: the bundled MSVC runtime, in the directory it is bundled into. */ + Assert.Contains(@"C:\PerformanceMonitorDarling\pg-runtime\pgsql\bin", diagnosis, StringComparison.Ordinal); + Assert.Contains("vcruntime140.dll", diagnosis, StringComparison.Ordinal); + Assert.Contains("vcruntime140_1.dll", diagnosis, StringComparison.Ordinal); + Assert.Contains("msvcp140.dll", diagnosis, StringComparison.Ordinal); + + /* Cause 2: the service account cannot read the install tree. */ + Assert.Contains("NT SERVICE", diagnosis, StringComparison.Ordinal); + + /* Check 1: run the binary by hand. Check 2: the Windows log that names the module. */ + Assert.Contains("--version", diagnosis, StringComparison.Ordinal); + Assert.Contains("Event Viewer", diagnosis, StringComparison.Ordinal); + } + + /// + /// The empty-output half of the report: the diagnosis has to say the blank field is EXPECTED for a + /// loader failure. Leaving it unexplained is what made the field report read as "no information + /// available" instead of "this is a load failure". + /// + [Fact] + public void Diagnose_SaysAnEmptyOutputIsExpectedNotMissing() + { + var diagnosis = DarlingToolExitCode.Diagnose(StatusDllNotFound, InitDb); + + Assert.Contains("expected", diagnosis, StringComparison.OrdinalIgnoreCase); + Assert.Contains("loader", diagnosis, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData(StatusEntryPointNotFound)] + [InlineData(StatusInvalidImageFormat)] + [InlineData(StatusDllInitFailed)] + [InlineData(StatusAccessDenied)] + public void Diagnose_CoversEveryLoaderStatusNotJustTheReportedOne(int exitCode) + { + var diagnosis = DarlingToolExitCode.Diagnose(exitCode, InitDb); + + Assert.Contains("vcruntime140.dll", diagnosis, StringComparison.Ordinal); + Assert.Contains("NT SERVICE", diagnosis, StringComparison.Ordinal); + } + + /// + /// A CRASH is not a load failure, and telling an operator to go looking for missing DLLs after an + /// access violation would be the same wrong-direction error the raw number caused — just pointed + /// somewhere new. The process ran; its output is real; the DLL advice must not appear. + /// + [Fact] + public void Diagnose_DoesNotSendACrashLookingForMissingDlls() + { + var diagnosis = DarlingToolExitCode.Diagnose(StatusAccessViolation, InitDb); + + Assert.DoesNotContain("vcruntime140", diagnosis, StringComparison.Ordinal); + Assert.Contains("crash", diagnosis, StringComparison.OrdinalIgnoreCase); + } + + /// An ordinary non-zero exit gets no paragraph at all — initdb exiting 1 with a real error + /// message on stderr needs no help from here, and burying that message under boilerplate would make + /// the common failure worse to read. + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(4)] + public void Diagnose_IsSilentForAnOrdinaryExitCode(int exitCode) + { + Assert.Equal(string.Empty, DarlingToolExitCode.Diagnose(exitCode, InitDb)); + } + + /// + /// The Output: field itself. On a Windows status an empty capture is stated as expected; on an + /// ordinary exit it is just absent, with no loader story attached to it. + /// + [Fact] + public void FormatOutput_ExplainsABlankCaptureOnlyWhenWindowsKilledIt() + { + var loader = DarlingToolExitCode.FormatOutput(string.Empty, StatusDllNotFound); + Assert.Contains("expected", loader, StringComparison.OrdinalIgnoreCase); + + var ordinary = DarlingToolExitCode.FormatOutput(" ", 1); + Assert.DoesNotContain("expected", ordinary, StringComparison.OrdinalIgnoreCase); + Assert.False(string.IsNullOrWhiteSpace(ordinary), "a blank capture still has to render as something an operator can read"); + } + + [Fact] + public void FormatOutput_PassesRealOutputThroughUntouched() + { + const string real = "initdb: error: directory \"C:\\pg\" exists but is not empty"; + + Assert.Equal(real, DarlingToolExitCode.FormatOutput(real, 1)); + Assert.Equal(real, DarlingToolExitCode.FormatOutput(real, StatusDllNotFound)); + } + + /// + /// The empirical half, and the reason this test uses a real process rather than a constant: it pins + /// that a Windows status really does arrive at as + /// the signed number this decoder is built around, with an EMPTY capture — the two facts the whole + /// fix rests on. cmd /c exit sets the same exit status the loader would; it is the status's + /// journey through the runner that is under test, not how it was produced. + /// + [Fact] + public async Task RunTool_SurfacesAWindowsStatusAsTheSignedFieldValueWithNoOutput() + { + if (!OperatingSystem.IsWindows()) + { + return; + } + + var cmd = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.System), "cmd.exe"); + + var (exitCode, output) = await DarlingManagedPostgres.RunToolAsync( + cmd, "/c exit 3221225781", TimeSpan.FromSeconds(30), CancellationToken.None); + + Assert.Equal(-1073741515, exitCode); + Assert.Equal(string.Empty, output); + Assert.Contains("STATUS_DLL_NOT_FOUND", DarlingToolExitCode.Describe(exitCode), StringComparison.Ordinal); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs index 3805f3f8eb..ccf9f8b6d1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingManagedPostgres.cs @@ -779,8 +779,9 @@ public async Task StopIfStartedByThisProcessAsync() try { var binDirectory = Path.Combine(_runtimeRoot, "pgsql", "bin"); + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (exitCode, output) = await RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"stop -D \"{_dataDirectory}\" -m fast -w -t {PgCtlWaitSeconds}", s_pgCtlTimeout, CancellationToken.None); @@ -791,7 +792,10 @@ public async Task StopIfStartedByThisProcessAsync() } else { - _logger.LogWarning("Managed Postgres stop reported exit code {ExitCode}: {Output}", exitCode, output); + _logger.LogWarning( + "Managed Postgres stop reported exit code {ExitCode}: {Output}", + DarlingToolExitCode.Describe(exitCode), + DarlingToolExitCode.FormatOutput(output, exitCode)); } } catch (Exception ex) @@ -879,16 +883,16 @@ holds the superuser password in the clear. */ TryHardenCredentialFile(passwordFile, allowInteractiveRead: false); try { + var initDb = Path.Combine(binDirectory, "initdb.exe"); var (exitCode, output) = await RunToolAsync( - Path.Combine(binDirectory, "initdb.exe"), + initDb, $"-D \"{_dataDirectory}\" -U {UserName} -A scram-sha-256 --pwfile=\"{passwordFile}\" -E UTF8 --locale=C --data-checksums", s_initDbTimeout, cancellationToken); if (exitCode != 0) { - throw new InvalidOperationException( - $"initdb failed (exit code {exitCode}) for {_dataDirectory}. Output:\n{output}"); + throw new InvalidOperationException(BuildInitDbFailureMessage(exitCode, initDb, _dataDirectory, output)); } } finally @@ -906,6 +910,18 @@ holds the superuser password in the clear. */ _logger.LogInformation("Managed Postgres cluster initialized (scram-sha-256, data checksums, UTF8/C locale)"); } + /// + /// The first-run initdb failure, as an operator reads it (#2186). The leading clause is unchanged on + /// purpose — it is what the existing field reports and the issue tracker are searchable by — and + /// everything the raw form withheld follows it: the exit code decoded, the loader diagnosis when + /// Windows set that code, and an Output: field that says it is empty BECAUSE the process was + /// killed before it could write, rather than looking like data that failed to arrive. + /// + internal static string BuildInitDbFailureMessage(int exitCode, string exePath, string dataDirectory, string output) + => $"initdb failed (exit code {DarlingToolExitCode.Describe(exitCode)}) for {dataDirectory}." + + DarlingToolExitCode.Diagnose(exitCode, exePath) + + $"\nOutput:\n{DarlingToolExitCode.FormatOutput(output, exitCode)}"; + /// /// Marker-guarded conf append, re-checked on EVERY start — heals the crash window between /// initdb and the first append, which would otherwise silently cost TimescaleDB @@ -1094,7 +1110,7 @@ private async Task EnsureDataDirectoryMajorAsync(string binDirectory, Cancellati { var dataMajor = DarlingStoreUpgrade.ParseDataDirectoryMajor( await File.ReadAllTextAsync(Path.Combine(_dataDirectory, "PG_VERSION"), cancellationToken)); - var bundledMajor = await ReadRuntimeMajorAsync(binDirectory, cancellationToken); + var (bundledMajor, probeExitCode) = await ReadRuntimeMajorAsync(binDirectory, cancellationToken); if (DarlingStoreUpgrade.MustRefuseUnidentifiableRuntime(dataMajor, bundledMajor)) { @@ -1105,12 +1121,15 @@ DARLING01 as "skipping the runtime version check. The store starts normally." an is the strongest possible evidence they must not be used, not a reason to wave them through. Refusing here costs nothing that proceeding would have saved — the start was going to fail regardless — and it converts a cryptic Win32 status code into a message naming the rescued - runtime to restore. */ + runtime to restore. #2186 finished the thought: the refusal used to assert that the binaries + did not run without ever saying HOW it knew, so the one piece of evidence it held — the probe's + own exit code — died in a local variable. It is now quoted, decoded, and diagnosed. */ throw new InvalidOperationException( - $"The store's data directory {_dataDirectory} is PostgreSQL {dataMajor}, but the runtime at {binDirectory} could not be identified — its binaries did not run. " + + $"The store's data directory {_dataDirectory} is PostgreSQL {dataMajor}, but the runtime at {binDirectory} could not be identified — pg_ctl --version exited {DarlingToolExitCode.Describe(probeExitCode)} instead of reporting a version. " + "A runtime that cannot report its own version cannot start this store either, so the service is stopping here rather than failing deeper with a Win32 error code. " + $"This usually means the wrong package was deployed. Restore the previous runtime from {PreviousRuntimeHint()} over {Path.GetDirectoryName(binDirectory)}, or redeploy a package whose PostgreSQL major is {dataMajor} or newer, then restart the service. " + - "The data directory has not been touched."); + "The data directory has not been touched." + + DarlingToolExitCode.Diagnose(probeExitCode, Path.Combine(binDirectory, "pg_ctl.exe"))); } if (dataMajor is null || bundledMajor is null) @@ -1172,7 +1191,7 @@ runtime to restore. */ /* The revert put the PREVIOUS runtime back behind the same bin path, so the identity read above now describes binaries that are no longer there. Re-read it, or the post-start completion would try to move the extension to a version this runtime does not ship. */ - _bundledMajor = await ReadRuntimeMajorAsync(binDirectory, cancellationToken) ?? 0; + _bundledMajor = (await ReadRuntimeMajorAsync(binDirectory, cancellationToken)).Major ?? 0; _bundledTimescaleVersion = ReadBundledTimescaleVersion(binDirectory); } } @@ -1181,12 +1200,14 @@ private string PreviousRuntimeHint() => Path.Combine(DarlingStoreUpgrade.PreviousRuntimeRootFor(_runtimeRoot), "pgsql"); /// The bundled runtime's PostgreSQL major, from the binaries themselves rather than from a - /// manifest that could disagree with what is on disk. - private static async Task ReadRuntimeMajorAsync(string binDirectory, CancellationToken cancellationToken) + /// manifest that could disagree with what is on disk. The probe's exit code rides out alongside it + /// (#2186): when the answer is "unidentifiable", that code is the ONLY evidence of why, and the + /// refusal that consumes it used to have to assert the reason instead of showing it. + private static async Task<(int? Major, int ExitCode)> ReadRuntimeMajorAsync(string binDirectory, CancellationToken cancellationToken) { var (exitCode, output) = await RunToolAsync( Path.Combine(binDirectory, "pg_ctl.exe"), "--version", s_statusTimeout, cancellationToken); - return exitCode == 0 ? DarlingStoreUpgrade.ParsePostgresMajor(output) : null; + return (exitCode == 0 ? DarlingStoreUpgrade.ParsePostgresMajor(output) : null, exitCode); } /// @@ -1218,8 +1239,9 @@ private string PreviousRuntimeHint() /// pg_ctl status: 0 = a postmaster is running on this data directory, 3 = not running, 4 = bad/inaccessible data directory. private async Task IsRunningAsync(string binDirectory, CancellationToken cancellationToken) { + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (exitCode, output) = await RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"status -D \"{_dataDirectory}\"", s_statusTimeout, cancellationToken); @@ -1228,11 +1250,25 @@ private async Task IsRunningAsync(string binDirectory, CancellationToken c { 0 => true, 3 => false, - _ => throw new InvalidOperationException( - $"pg_ctl status reported exit code {exitCode} for {_dataDirectory} — the data directory is not usable. Output:\n{output}"), + _ => throw new InvalidOperationException(BuildStatusFailureMessage(exitCode, pgCtl, _dataDirectory, output)), }; } + /// + /// The pg_ctl status failure (#2186). The data-directory verdict is CONDITIONAL: pg_ctl's own codes + /// (4 = bad or inaccessible data directory) do say the directory is unusable, but a Windows status + /// says only that pg_ctl never ran, and blaming the data directory for that sends an operator to + /// delete a perfectly good store over a missing DLL. + /// + internal static string BuildStatusFailureMessage(int exitCode, string exePath, string dataDirectory, string output) + { + var diagnosis = DarlingToolExitCode.Diagnose(exitCode, exePath); + return $"pg_ctl status reported exit code {DarlingToolExitCode.Describe(exitCode)} for {dataDirectory}" + + (diagnosis.Length == 0 ? " — the data directory is not usable." : ".") + + diagnosis + + $"\nOutput:\n{DarlingToolExitCode.FormatOutput(output, exitCode)}"; + } + /// /// pg_ctl start, windowed (-w): returns only when the server accepts connections. The -o /// runtime override carries the port (authoritative over the conf line), listen_addresses (always @@ -1263,8 +1299,9 @@ legacy file stays bounded too. Going forward the v6 logging collector owns the s server is down (this method only runs when nothing is listening), so nothing holds the file. */ CapLegacyServerLog(_serverLogPath, LegacyServerLogCapBytes, _logger); + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var exitCode = await RunDetachingToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"-D \"{_dataDirectory}\" -o \"{runtimeOptions}\" -l \"{_serverLogPath}\" -w -t {PgCtlWaitSeconds} start", s_pgCtlTimeout, cancellationToken); @@ -1272,13 +1309,24 @@ server is down (this method only runs when nothing is listening), so nothing hol if (exitCode != 0) { throw new InvalidOperationException( - $"pg_ctl start failed (exit code {exitCode}) for {_dataDirectory}.\n" + - $"Server log tail:\n{ReadServerLogTail()}"); + BuildStartFailureMessage(exitCode, pgCtl, _dataDirectory, ReadServerLogTail())); } _logger.LogInformation("Managed Postgres started"); } + /// + /// The pg_ctl start failure (#2186). Its Server log tail has the same trap the initdb message's + /// Output had: a loader status means pg_ctl died before it could start a postmaster, so the tail + /// reads "(no server log written)" — accurate, and completely misleading about where to look. The + /// diagnosis says which situation this is before the tail invites an operator to read a log that was + /// never going to exist. + /// + internal static string BuildStartFailureMessage(int exitCode, string exePath, string dataDirectory, string serverLogTail) + => $"pg_ctl start failed (exit code {DarlingToolExitCode.Describe(exitCode)}) for {dataDirectory}." + + DarlingToolExitCode.Diagnose(exitCode, exePath) + + $"\nServer log tail:\n{serverLogTail}"; + /// The one-time cap on the legacy pre-rotation pg.log (#1652): past this size it is /// rolled to pg.log.old (replacing any previous roll) before the next start. Two files, bounded /// forever; small files are left alone so a healthy post-rotation pg.log is never churned. @@ -2015,16 +2063,19 @@ private async Task ReconcileNetworkAsync( if (changed) { await File.WriteAllTextAsync(hbaPath, updated, cancellationToken); + var reloadPgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (reloadCode, reloadOutput) = await RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + reloadPgCtl, $"reload -D \"{_dataDirectory}\"", s_statusTimeout, cancellationToken); if (reloadCode != 0) { _logger.LogCritical( - "pg_ctl reload failed (exit {ExitCode}) after updating pg_hba.conf — the network access change may not be live: {Output}", - reloadCode, reloadOutput); + "pg_ctl reload failed (exit {ExitCode}) after updating pg_hba.conf — the network access change may not be live: {Output}{Diagnosis}", + DarlingToolExitCode.Describe(reloadCode), + DarlingToolExitCode.FormatOutput(reloadOutput, reloadCode), + DarlingToolExitCode.Diagnose(reloadCode, reloadPgCtl)); } } diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs index 9402580aa7..482da3f6be 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingStoreUpgrade.cs @@ -1539,14 +1539,18 @@ can be read rather than assumed ---- */ File.WriteAllText(passwordFile, context.Password + "\n"); DarlingFileSecurity.HardenFile(passwordFile, allowInteractiveRead: false); + var newInitDb = Path.Combine(context.NewBinDirectory, "initdb.exe"); var (initExit, initOutput) = await DarlingManagedPostgres.RunToolAsync( - Path.Combine(context.NewBinDirectory, "initdb.exe"), + newInitDb, BuildInitDbArguments(newDataDirectory, context.UserName, passwordFile, identity, context.NewMajor), s_initDbTimeout, cancellationToken); if (initExit != 0) { - throw new InvalidOperationException($"initdb of the new cluster failed (exit {initExit}):\n{initOutput}"); + throw new InvalidOperationException( + $"initdb of the new cluster failed (exit {DarlingToolExitCode.Describe(initExit)}):" + + DarlingToolExitCode.Diagnose(initExit, newInitDb) + + $"\n{DarlingToolExitCode.FormatOutput(initOutput, initExit)}"); } step = "conf-new-cluster"; @@ -1560,8 +1564,9 @@ can be read rather than assumed ---- */ step = "pg_upgrade-check"; var environment = BuildLibpqCredentialEnvironment(context.Password); + var pgUpgrade = Path.Combine(context.NewBinDirectory, "pg_upgrade.exe"); var checkExit = await DarlingManagedPostgres.RunDetachingToolAsync( - Path.Combine(context.NewBinDirectory, "pg_upgrade.exe"), + pgUpgrade, BuildPgUpgradeArguments( context.OldBinDirectory, context.NewBinDirectory, context.DataDirectory, newDataDirectory, context.UserName, mode, checkOnly: true, jobs: 1, QuiesceTimescaleServerOptions), @@ -1571,8 +1576,16 @@ can be read rather than assumed ---- */ parent); if (checkExit != 0) { + /* "The clusters are not compatible" is pg_upgrade's verdict, and only pg_upgrade can reach it. + A Windows status means pg_upgrade never ran, so the compatibility claim would be invented + (#2186) — the same wrong-blame the pg_ctl status message carried. */ + var checkDiagnosis = DarlingToolExitCode.Diagnose(checkExit, pgUpgrade); throw new InvalidOperationException( - $"pg_upgrade --check failed (exit {checkExit}) — the clusters are not compatible and NOTHING has been changed.\n{ReadPgUpgradeLogTail(newDataDirectory)}"); + $"pg_upgrade --check failed (exit {DarlingToolExitCode.Describe(checkExit)})" + + (checkDiagnosis.Length == 0 + ? " — the clusters are not compatible and NOTHING has been changed." + : ". NOTHING has been changed." + checkDiagnosis) + + $"\n{ReadPgUpgradeLogTail(newDataDirectory)}"); } step = "pg_upgrade"; @@ -1585,7 +1598,7 @@ can be read rather than assumed ---- */ becomes the complaint. */ const int jobs = 1; var upgradeExit = await DarlingManagedPostgres.RunDetachingToolAsync( - Path.Combine(context.NewBinDirectory, "pg_upgrade.exe"), + pgUpgrade, BuildPgUpgradeArguments( context.OldBinDirectory, context.NewBinDirectory, context.DataDirectory, newDataDirectory, context.UserName, mode, checkOnly: false, jobs, QuiesceTimescaleServerOptions), @@ -1596,7 +1609,9 @@ becomes the complaint. */ if (upgradeExit != 0) { throw new InvalidOperationException( - $"pg_upgrade failed (exit {upgradeExit}).\n{ReadPgUpgradeLogTail(newDataDirectory)}"); + $"pg_upgrade failed (exit {DarlingToolExitCode.Describe(upgradeExit)})." + + DarlingToolExitCode.Diagnose(upgradeExit, pgUpgrade) + + $"\n{ReadPgUpgradeLogTail(newDataDirectory)}"); } /* ---- 7. swap the directories so the configured path holds the upgraded cluster. The @@ -1772,8 +1787,9 @@ private async Task StartClusterAsync(string binDirectory, string dataDirectory, Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(Path.GetFullPath(dataDirectory)))!, DarlingManagedPostgres.ServerLogFileName); + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var exitCode = await DarlingManagedPostgres.RunDetachingToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"-D \"{dataDirectory}\" -o \"-p {port} -c listen_addresses=127.0.0.1\" -l \"{serverLog}\" -w -t 120 start", TimeSpan.FromMinutes(5), cancellationToken); @@ -1781,21 +1797,25 @@ private async Task StartClusterAsync(string binDirectory, string dataDirectory, if (exitCode != 0) { throw new InvalidOperationException( - $"could not start the cluster in {dataDirectory} with the runtime at {binDirectory} (pg_ctl exit {exitCode})"); + $"could not start the cluster in {dataDirectory} with the runtime at {binDirectory} (pg_ctl exit {DarlingToolExitCode.Describe(exitCode)})" + + DarlingToolExitCode.Diagnose(exitCode, pgCtl)); } } private async Task StopClusterAsync(string binDirectory, string dataDirectory, CancellationToken cancellationToken) { + var pgCtl = Path.Combine(binDirectory, "pg_ctl.exe"); var (exitCode, output) = await DarlingManagedPostgres.RunToolAsync( - Path.Combine(binDirectory, "pg_ctl.exe"), + pgCtl, $"stop -D \"{dataDirectory}\" -m fast -w -t 120", s_toolTimeout, cancellationToken); if (exitCode != 0) { - throw new InvalidOperationException($"could not cleanly stop the cluster in {dataDirectory} (pg_ctl exit {exitCode}): {output}"); + throw new InvalidOperationException( + $"could not cleanly stop the cluster in {dataDirectory} (pg_ctl exit {DarlingToolExitCode.Describe(exitCode)}): {DarlingToolExitCode.FormatOutput(output, exitCode)}" + + DarlingToolExitCode.Diagnose(exitCode, pgCtl)); } } @@ -2230,7 +2250,8 @@ private async Task RunAnalyzeInStagesAsync( { _logger.LogWarning( "Post-upgrade analyze reported exit {ExitCode} ({Output}). The store is fully usable; autovacuum will build the remaining statistics.", - exitCode, output); + DarlingToolExitCode.Describe(exitCode), + DarlingToolExitCode.FormatOutput(output, exitCode)); } } catch (OperationCanceledException) diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingToolExitCode.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingToolExitCode.cs new file mode 100644 index 0000000000..7ec36fbfb7 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingToolExitCode.cs @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +namespace PerformanceMonitor.Darling.Service; + +/// +/// Turns a bundled-Postgres tool's exit code into words (#2186) — the shared vocabulary every +/// managed-store process failure is reported in. +/// +/// The bug this exists for, twice from the field: a managed bootstrap failed with +/// initdb failed (exit code -1073741515) for C:\ProgramData\PerformanceMonitorDarling\pg. Output: +/// and nothing more. -1073741515 is 0xC0000135, STATUS_DLL_NOT_FOUND: Windows killed +/// the process in the LOADER, before a line of its own code ran. Which is also why Output: was +/// empty and always will be for this class of failure — the one field an operator reads is guaranteed +/// blank exactly when the failure is a load failure, so it reads as "no information available" rather +/// than "this is a loader failure". The operator's attention then went to the follow-on +/// missing-credential message and darling.json, neither of which was the fault. +/// +/// Why a decoder and not one more format string: the sibling case already converts "the +/// binaries could not run" into words (MustRefuseUnidentifiableRuntime, #1738) and every other +/// bundled-tool failure printed the raw number. One shared decoder is what keeps the next one from +/// being the third report of the same defect. +/// +internal static class DarlingToolExitCode +{ + /// + /// NTSTATUS severity ERROR — the top two bits set. A process whose exit code lands in this range did + /// not choose it: Windows did, either in the loader or by ending a crashed process. Real programs exit + /// with small numbers (initdb 1, pg_ctl 3/4), so the range is a clean discriminator rather than a + /// heuristic, and treating the WHOLE range as "Windows killed it" is what keeps an unlisted status + /// from being as opaque as -1073741515 was. + /// + private const uint NtStatusErrorFloor = 0xC0000000; + + /// + /// The statuses worth naming. Loader statuses (the ones that mean the image or a dependency never + /// loaded) carry Loader = true and earn the full two-causes-two-checks diagnosis; the rest are + /// named so the number stops being a mystery, and get the shorter paragraph their kind deserves. + /// + private static readonly Dictionary s_knownStatuses = new() + { + [0xC0000135] = ("STATUS_DLL_NOT_FOUND", StatusKind.Loader), + [0xC0000139] = ("STATUS_ENTRYPOINT_NOT_FOUND", StatusKind.Loader), + [0xC000007B] = ("STATUS_INVALID_IMAGE_FORMAT", StatusKind.Loader), + [0xC0000142] = ("STATUS_DLL_INIT_FAILED", StatusKind.Loader), + [0xC0000022] = ("STATUS_ACCESS_DENIED", StatusKind.Loader), + [0xC0000005] = ("STATUS_ACCESS_VIOLATION", StatusKind.Crash), + [0xC0000374] = ("STATUS_HEAP_CORRUPTION", StatusKind.Crash), + [0xC0000409] = ("STATUS_STACK_BUFFER_OVERRUN", StatusKind.Crash), + [0xC000013A] = ("STATUS_CONTROL_C_EXIT", StatusKind.Terminated), + }; + + private enum StatusKind + { + Loader, + Crash, + Terminated, + } + + /// + /// The exit code as an operator should read it: the bare number for a tool's own status (initdb's 1, + /// pg_ctl status's 3), and the number PLUS its hex and NTSTATUS name when Windows set it. + /// + internal static string Describe(int exitCode) + { + var status = unchecked((uint)exitCode); + if (status < NtStatusErrorFloor) + { + return exitCode.ToString(CultureInfo.InvariantCulture); + } + + /* No parentheses of its own: every call site already sits inside one ("exit code {...}"), and + nesting them produced "(exit code -1073741515 (0xC0000135 STATUS_DLL_NOT_FOUND))". The "=" form + is also how the field report itself explained the number. */ + var hex = "0x" + status.ToString("X8", CultureInfo.InvariantCulture); + return s_knownStatuses.TryGetValue(status, out var known) + ? $"{exitCode.ToString(CultureInfo.InvariantCulture)} = {hex} {known.Name}" + : $"{exitCode.ToString(CultureInfo.InvariantCulture)} = {hex}, an unnamed Windows status rather than the program's own exit code"; + } + + /// + /// The paragraph that follows the failure line: what Windows did, why the captured output is empty, + /// and the checks that separate the two causes. Empty string for a tool's own exit code — initdb + /// exiting 1 with a real error on stderr needs no help from here, and burying that message under + /// boilerplate would make the common failure worse to read. + /// + internal static string Diagnose(int exitCode, string exePath) + { + var status = unchecked((uint)exitCode); + if (status < NtStatusErrorFloor) + { + return string.Empty; + } + + var toolName = SafeFileName(exePath); + var kind = s_knownStatuses.TryGetValue(status, out var known) ? known.Kind : StatusKind.Terminated; + + return kind switch + { + StatusKind.Loader => LoaderDiagnosis(toolName, exePath), + StatusKind.Crash => + $"\n{toolName} started and then crashed — Windows ended it, so anything it had to say may be truncated or missing entirely. " + + "This is not a configuration problem and restarting will not clear it: capture the matching entry from Event Viewer > Windows Logs > Application " + + "(Application Error / Windows Error Reporting) together with the store's pg.log, and report it.", + _ => + $"\nWindows ended {toolName} rather than the tool exiting on its own, so its output may be empty or truncated and this is not a PostgreSQL error. " + + "Event Viewer > Windows Logs > Application, at the time of the failure, usually carries the matching entry.", + }; + } + + /// + /// The two causes and the two checks, in the order an operator should work them. Both causes are + /// specific to how this product ships PostgreSQL: the MSVC runtime is BUNDLED beside the binaries + /// (so a missing one means a partial extract, not a missing prerequisite), and the service runs as an + /// unprivileged virtual account that a user-profile install tree does not grant. + /// + private static string LoaderDiagnosis(string toolName, string exePath) + { + var binDirectory = SafeDirectoryName(exePath); + var builder = new StringBuilder(); + + builder.Append('\n').Append("Windows killed ").Append(toolName) + .Append(" in the LOADER: it never ran a line of its own code. This is a Windows failure, not a PostgreSQL one, and it is why the captured output is empty — for a loader failure an empty Output is EXPECTED, not missing information.\n"); + + builder.Append("Two causes account for nearly all of these:\n"); + builder.Append(" (1) The Microsoft Visual C++ runtime is missing from ").Append(binDirectory) + .Append(". Packaging bundles vcruntime140.dll, vcruntime140_1.dll and msvcp140.dll there so the box needs no prerequisite — ") + .Append("if any of the three is absent the install tree is a partial or damaged extract, so redeploy the package, or install the Microsoft Visual C++ 2015-2022 x64 redistributable.\n"); + builder.Append(" (2) The service account cannot read the install tree. The service runs as the virtual account NT SERVICE\\PerformanceMonitor Darling, which is neither you nor Administrators, ") + .Append("so an install under a user profile (Desktop, Downloads, anywhere below C:\\Users) is unreadable to it — reinstall to a machine-scoped path such as C:\\PerformanceMonitorDarling.\n"); + + builder.Append("Two checks that tell them apart:\n"); + builder.Append(" (a) Run \"").Append(exePath).Append("\" --version from an elevated prompt. Failing there too means (1). Succeeding points at (2) — and note that these tools re-execute themselves ") + .Append("under a restricted token that drops the Administrators group, so a tree readable only VIA Administrators still fails the real run even when it runs by hand.\n"); + builder.Append(" (b) Event Viewer > Windows Logs > Application, at the time of the failure: an Application Error or SideBySide entry usually names the exact module that could not be loaded."); + + return builder.ToString(); + } + + /// + /// The Output: field itself. Real output passes through untouched; a blank capture renders as + /// something readable instead of nothing at all, and says so explicitly when Windows is the reason + /// there was nothing to capture. + /// + internal static string FormatOutput(string? output, int exitCode) + { + if (!string.IsNullOrWhiteSpace(output)) + { + return output; + } + + return unchecked((uint)exitCode) >= NtStatusErrorFloor + ? "(none — Windows ended the process before it could write anything, which is expected for this failure rather than missing information)" + : "(none)"; + } + + /// Never let a malformed path turn a diagnostic into a second exception. + private static string SafeFileName(string exePath) + { + try + { + var name = Path.GetFileName(exePath); + return string.IsNullOrEmpty(name) ? "the tool" : name; + } + catch (ArgumentException) + { + return "the tool"; + } + } + + private static string SafeDirectoryName(string exePath) + { + try + { + var directory = Path.GetDirectoryName(exePath); + return string.IsNullOrEmpty(directory) ? "the pg-runtime\\pgsql\\bin directory" : directory; + } + catch (ArgumentException) + { + return "the pg-runtime\\pgsql\\bin directory"; + } + } +} From 959373f946aeb3e02e7fffbcaa98a81fe7a341dd Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:16:30 -0400 Subject: [PATCH 2/4] CHANGELOG entry for the bootstrap loader diagnosis (#2186) Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 5350 +++++++++++++++++++++++++------------------------- 1 file changed, 2676 insertions(+), 2674 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ff31fbaf..331495b6e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,2674 +1,2676 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -### Added - -- **A failing forced plan now alerts** ([#2157]) - when Query Store cannot reproduce a forced plan, the query silently runs on whatever the optimizer picks instead, and nothing in the product witnessed it: the only trace was a counter climbing inside Query Store. The new **Forced Plan Failing** alert fires per plan when `force_failure_count` RISES between collections, carrying the database, query and plan ids, whether the force was MANUAL or automatic plan correction, the engine's own failure reason, and how many failures are new. It is a rise and never a level, because the counter is cumulative AND travels with a restored database - alerting on the level would page forever about failures that happened on hardware you may no longer own. A counter that drops (an unforce/re-force cycle) is silence, and a plan seen for the first time waits one cycle, since 'new' is not knowable from a single observation. Severity is Warning for every rise: a failing force is not an outage, it is somebody's mitigation quietly not working. - -- **Every previously-hardcoded alert threshold is now a real setting** ([#2107], the split-out from gotqn's #2101 - "it was fine to hardcode these for development but any serious monitoring allows configuring of alert thresholds") - six new knobs ride the store control plane (V55), the Viewer's Settings window, and `get_alert_settings`/`update_alert_settings`, clamped on read like their siblings: the monitor store volume's self-alert warning percent (was 10), the Collection Stopped staleness window (was 30 minutes) and consecutive-failure fast path (was 10), the low-disk CRITICAL severity tier's percent and GB floors (were 3% / 2 GB - these grade the target-volume alert in BOTH apps, and Lite reads its pair from `settings.json` as `alert_disk_critical_free_percent` / `alert_disk_critical_free_gb`), and the analysis notification cooldown (was a hardcoded 360 in Darling while Lite always honored a configured value - the parity gap closed). MCP shape: `low_disk.critical_free_percent` / `low_disk.critical_free_gb`, a new `self_alerts` group, and `analysis.notify_cooldown_minutes`. -- **Per-database collection timing now separates server think-time from row streaming** ([#2164]) - the per-database Query Store line reports `sql:Xms = wm:Wms + open:Yms + drain:Zms`, where wm is the watermark refresh (a monitor-store round trip the timer already started before), open is everything before the first row arrives, and drain is streaming those rows to the collector. This exists because of a measurement that overturned an assumption: cutting the text budget from 64 MB to 12 MB on a production server moved 5x fewer bytes and roughly 7x fewer rows, and the batch clock did not move at all. That says the cost lives upstream of shipping - in Query Store's own aggregation before the first row - which no client-side budget or payload trimming can shorten. The blended number could not show that, so the split is now visible: a pass that is nearly all `open` needs the server-side query narrowed, and a pass that is mostly `drain` is the one a smaller budget or a shorter network path helps. -- **The two collector memory bounds are now operator knobs** ([#2164], [#2170]) - the per-database Query Store text budget (was a hardcoded 64 MB) and the fleet sweep width (was a hardcoded 4 servers) ride the store control plane (V59), the Viewer's Settings window, and the service's live reload, clamped [4,256] MB and [1,16] on read. Defaults reproduce the old constants exactly, so an upgrade changes nothing until a dial moves. Why both at once: peak transient memory is roughly the two multiplied, so an operator moving one needs the other in front of them. Lower the budget when the monitored fleet is a network hop away - the budget bounds memory, but it also sets how long one collector query holds the monitored server open draining to the client, which over a cross-region link is the tenant-visible cost (a smaller budget trades catch-up latency for shorter statements, never data, because every cut is resumable). Raise the sweep width when a large fleet queues behind 4-wide collection on a host with headroom, which is what makes the Fleet Health screen report staleness while every collector reports healthy. Narrowing the width never interrupts a running collection - the retiring permits are absorbed as bodies finish. -- **The Query Store backfill has an off switch** ([#2167]) - the #2058 backfill previously ran unconditionally, and during a fleet consolidation a freshly restored database's imported catalog put it into sustained byte-budget drains against a cross-region production primary with no way to stop it short of disabling plan capture everywhere. `config_service.query_store_backfill_enabled` (V58, default on) is read live by the service's backfill loop - flip it in the Viewer's Settings window (new checkbox beside plan capture) and the loop idles from its next cycle, no restart; re-enabling resumes exactly where the watermarks left off. Live collection is never affected. Lite gets the same control as a Settings checkbox ("Fill Query Store history gaps in the background"), read live so it takes effect without restarting the app - the store column exists on the Darling side because a headless service has no window to click. -- **The store measures its own background jobs** ([#2136], the visibility half) - the hourly #2068 self-metrics sweep now writes one row per TimescaleDB background job (object_kind `background_job`, V56 columns): last run duration, schedule interval, total runs, total failures. Why: the store's heaviest recurring work is its own job machinery - on the production 52-server store the four most expensive jobs are all the query_store_stats family (compression 157s, interval_hourly refresh 96s) - their runtimes scale SERIALLY with raw volume (the finalize hash-aggregate runs in one process), and a job that outgrows its own cadence compounds refresh lag silently. With the interval stored beside the duration, 'how close is each job to its ceiling' is one division over a 400-day series instead of archaeology - the number an onboarding wave moves first. A threshold alert on the series is the issue's next half. -- **The store alerts when its own background jobs outgrow their schedule** ([#2136], the alert half) - a new fleet-level self-alert, Store Job Over Cadence: WARNING when a job's last successful run reaches a store-backed percent of its own schedule interval (V57 knob `store_job_cadence_warn_percent`, default 25, clamped 5-100, on the Settings window and `get_alert_settings`/`update_alert_settings` under `self_alerts`), CRITICAL fixed at 100 - past that the job is still running when its next run is due, so runs back up behind each other and everything it maintains (CAGG freshness, compression, retention) falls further behind every cycle. Judged hourly from timescaledb_information.job_stats on the same connection as the compression-health check, successful runs only (a failed run's duration is not a cadence signal); standing condition with cooldown re-fires and a Store Job Cadence Recovered resolution row; the alert text points at the V56 duration series in collect.store_metrics for the trend. Calibration: the production 52-server store's worst job runs at ~7% of cadence, so the default warns at 3.5x today's ceiling but far ahead of real compounding. -- **The #2136 capacity model is now proven by a synthetic scale test, not asserted from one observation** - a live end-to-end drives a throwaway hypertable's compression job at 1x and then 4x row volume (parked policy, deterministic run_job - the #1888 discipline, so the scheduler can never race the measurement) and pins the whole loop: job runtime GROWS with volume (monotonicity, not a ratio - runner jitter owns the constant factor, the direction is the claim), the V56 telemetry series records both readings in order, and the Store Job Over Cadence alert fires its Critical tier from REAL store readings once the schedule interval is shrunk under the measured duration. - -### Changed - -- **Query Store collection stops re-shipping plan XML the store already holds** ([#2164]) - 97% of the execution-plan XML shipped in a three-hour fleet window (197,113 of 202,790 rows) was for plans the store had already held for over an hour. The #1556 dedupe lands each plan once per PASS, but it re-lands it on every pass forever, and since streaming rows is 94-97% of a pass and costs per-row LOB bytes, NOT fetching a plan is worth far more than fetching less of one - cutting the text budget from 64 MB to 12 MB moved 5.3x less text and left the clock unchanged, which is what pointed here. Collection now carries a per-database watermark on `plan_id` (monotonic within one database's Query Store) and asks only for plans above the highest one whose XML actually landed. An absent, malformed or expired watermark renders no predicate at all, so the conservative path is byte-identical to the old query, and the watermark never applies to the history backfill - backfill reads intervals OLDER than anything collected, whose plans are numbered below the watermark, so honoring it there would suppress the very plans that pass exists to fetch. Measured against four production Query Store catalogs before shipping: identical row counts, 28-51% less elapsed time, and plan XML rows down 42-45% with the watermark set at only the MEDIAN plan_id. One full refetch per database per DAY still happens by design, which is what bounds the three things a permanent watermark would get wrong: a plan whose XML is rewritten in place without a new id, recovery after somebody clears Query Store (plan_id restarts at 1, so every new plan would sort below a stale watermark), and a plan compiled before monitoring began that stays dormant through every collected window and so is never seen above the mark. - -- **A deliberately offline database alerts once instead of every cooldown** ([#2166], reported by @gotqn) - the Database State alert re-fired for as long as a database stayed deviated, so parking one OFFLINE for a month produced hundreds of identical alerts for a single intended action. OFFLINE, RESTORING, RECOVERING and STANDBY are now edge-triggered: they alert on the transition and stay quiet until the state changes. The integrity states (SUSPECT, RECOVERY_PENDING, EMERGENCY) are unchanged and keep re-firing on the cooldown, because nobody parks a database in SUSPECT and there the repetition IS the signal. The memory is persisted per database (V60), not held in RAM, so a service restart cannot re-announce every parked database - and it composes with the per-database expected-state override exactly as reported: a database you park stays silent while still alerting the moment it turns SUSPECT or comes back ONLINE. **This changes default behavior**: if you were relying on the repeat-nag to track deliberate offlines, it now goes quiet after the first alert. Darling only for now; Lite keeps its current behavior until parity lands. -- **Force-plan findings carry a machine-first verdict for MCP consumers** ([#2138]; Erik: agents will read these more than people) - analyze_server and get_analysis_findings (both apps) now emit structured_remediation alongside the copy-paste command: per target, eligible plus NAMED blockers (parameter_sensitivity_cofired, secondary_replica_evidence), the evidence numbers, join keys, and split force_sql / unforce_sql / verify_sql artifacts (verify asks both post-force questions: did the force stick, and the per-interval cost since). The blocker list comes from FactRemediation.ForcePlanBlockers - the ONE policy gate a future auto-force feature would consult, so what agents inspect today is what any later automation enforces, as a testable data contract. Computed at read time from the persisted targets (never persisted itself - one source of truth, no DTO mirror to forget). -- **Plan-regression detection scores CPU as the primary signal and gates on absolute spend** ([#2138] Phase 0, the advise-only foundation for the auto-force-plan bot) - the old score was GREATEST(cpu ratio, duration ratio), so a plan whose CPU never moved could fire on duration alone - but duration is confounded by blocking, IO waits and machine contention that no plan choice caused, exactly the false positive a plan-forcing bot must never act on. Now a CPU regression scores at its own ratio; a duration-dominant one fires only when EXTREME (>= 4x) AND corroborated by at least mild CPU worsening (>= 1.25x), scored at HALF the duration ratio so it competes honestly with CPU-detected rows. A new noise floor drops offenders whose latest plan burned under 10 CPU-seconds across the whole 14-day comparison window - a 12x ratio on a query costing 12ms per run is sampling jitter, not a finding. The regressed-queries drill-down previously kept its OWN copy of the old score and could surface rows the fact never counted; it now runs the same scoring, in both SKUs, pinned by split-signal tests that move CPU and duration independently. -- **The force-plan recommendation now warns when the regressed query is parameter-sensitive** ([#2138] gap 3) - each regressed_queries row carries a `parameter_sensitivity_cofired` flag, computed inside the drill-down with the PARAMETER_SENSITIVITY detector's own thresholds (one cached plan whose per-execution cost varies >= 10x across parameter values, same floors, same window) joined by query hash - so the flag can never claim evidence the detector would not report. A flagged target's force-plan preview gains a caution block naming the risk (forcing pins ONE shape for every parameter value; the population that preferred the other plan inherits the wrong one permanently, quietly, because a forced plan no longer recompiles away) and the gentler first levers (statistics updates; PSP optimization / Query Store hints on 2022+), and the copy-paste surface gets a compact two-line version of the same warning. Unflagged targets render byte-identically to before. This flag is also the standing gate for the future auto-force bot: a flagged target is never auto-forced. Both SKUs, pinned by live tests in both stores. - -### Fixed - -- **The store's compressed plan format is now documented** ([#2171], reported by @argpna) - execution-plan XML has been gzip-compressed in `query_plan_dim.query_plan_gz` since 3.4.0, with `query_plan_xml` nullable, and the release notes never said so. A consumer reading the store directly over SQL therefore got nothing back for anything collected by a current build, with no clue why. The store section of the Darling README now states the format (gzip, magic `1f 8b`), that `query_plan_xml IS NULL` means read the compressed column rather than "no plan", and the three practical ways to get XML back - ask the product via `get_plan_xml`, decompress client-side, or ship a UDF into your own store if your tooling is SQL-only. In-product paths were never affected; this was a contract change for direct SQL consumers that shipped silently. -- **tempdb no longer reports more than 100% used in FinOps Database Sizes** ([#2169], reported by @CatastropheOps) - the used percentage divided in-database usage by the size recorded in `sys.master_files`, which is the size set at configuration time and does not track autogrowth for tempdb. A tempdb that had grown was therefore measured against its startup size and rendered above 100%. The per-database probe now captures the file's current size in the same round trip it already makes for space-used, and the payload prefers it, so both halves of the ratio come from one snapshot; a database whose probe fails still falls back to the old source rather than vanishing from the grid. Affects the on-prem, RDS, and Managed Instance path - the Azure SQL Database path already read both numbers in-database. -- **Backing out of Custom Range no longer strands an open calendar - both apps** ([#2154], reported in #2153) - a DatePicker's calendar dropdown is a popup living outside the visual tree's visibility, so collapsing the pickers when the user switched back to a preset range left an already-open calendar floating on screen; the dropdowns now close explicitly alongside the collapse, in Lite's ServerTab and the Darling Viewer's twin alike. -- **One wedged background task can no longer stop collection - in either app** ([#2148], reported on an Azure elastic pool minutes after upgrading) - Lite's collection ladder runs its steps sequentially, and while every step's exceptions were contained, nothing bounded a HANG: one stuck task (the new Query Store backfill was the prime suspect on the reporter's timeline) silently froze every collector, and the CPU chart going blank was just where it got noticed. Every backfill slice now runs under a PER-SERVER abandonment deadline with an in-flight guard (the scheduled-analysis idiom, extracted as a reusable primitive) in BOTH apps - Lite's tick and Darling's fleet loop share the exact shape: a wedged slice is abandoned so everything else continues, quarantined to ITS server only (never relaunched on top of itself, never blocking a neighbor), and self-restoring when the stuck task actually ends; Lite's fleet-wide connection check gets the same treatment. Abandonment logs at ERROR naming the issue, and an abandoned task's LATE exception - the one that explains the wedge - is surfaced instead of discarded. The deadlines are generous multiples of a single healthy slice, so those lines are always defect signals worth reporting. -- **The retention purge retries once on a deadlock instead of wasting the cycle** ([#2143], caught live by the nightly's purge e2e) - drop_chunks can lose a deadlock to a TimescaleDB background job whose chunk locks clear within milliseconds of the abort; previously that one transient loss pushed the table to the row-by-row DELETE fallback (which can deadlock against the same partner) and burned the whole cycle. Now 40P01 earns exactly ONE immediate retry - a second deadlock in a row is standing contention, where the fallback-plus-next-cycle posture is right and a retry loop camped on a lock queue is not. Non-deadlock failures keep the original single-shot behavior. -- **Remote viewers can finally use the exact connection string `--print-viewer-connection` prints** ([#2117], diagnosed to the trust-chain layer by jonchapman-usrc) - the store's TLS certificate was a single self-signed end-entity cert with critical `CA=false` Basic Constraints - a shape the reporter's Windows environment refused as its own trust anchor under the custom-root trust Npgsql applies to `Root Certificate=...` (stock Windows CI tolerates it, per the new end-to-end pins, so the refusal is environmental - strict chain-policy hardening is the likely class - but a shape that only works on unhardened machines is not a shape to ship), with the real error swallowed behind the generic "is the Darling service running?" message (the reporter burned hours eliminating everything else; importing the same cert into the OS trust store - their workaround - keeps working). Three changes: the service now generates a REAL two-cert chain (a throwaway local CA signs the server leaf and its private key is discarded on the spot, so the distributable `root.crt` still pins exactly one server identity), the print/export verbs emit that root, and the viewer's store-unreachable message now carries the underlying error text so a chain rejection, a wrong password, a pg_hba refusal, and a dead host stop reading identically. **Existing stores are deliberately NOT auto-rotated** - operators who imported the old cert keep a working setup, and the service logs the rotation recipe (stop, delete server.crt + server.key, start, redistribute) instead. Chain validity is pinned end-to-end - a real NpgsqlConnection at VerifyFull against an in-test TLS listener - on every CI platform, and the legacy shape's per-platform behavior is recorded in every run's log so a platform change shows up in CI diffs. The reporter's third finding (relative `Root Certificate` resolving against the process working directory) was already fixed on dev by #1970. -- **`--collapse-legacy-slices` no longer dies at TimescaleDB's decompression rail on compressed chunks** ([#2105] round two, ghauan again - the previous fix got the repair four minutes in, far enough to hit `53400: tuple decompression limit exceeded`) - a store old enough to need this repair has had its compression policy running the whole time, so the repair's DELETE necessarily touches compressed chunks, and TimescaleDB caps decompression at 100k tuples per DML transaction by default. Each slice transaction now lifts the cap for itself (`SET LOCAL`, dying with the transaction) - the same rail-lift the retention purge's fallback DELETE has always done, because deliberate bulk decompression is exactly this verb's job. -- **A Query Store member whose catch-up window can't fit the command timeout now shrinks it until one does** ([#2111] promoted from reserve on box evidence - one database sat 3+ hours stuck through quiet overnight hours, its 1h window intermittently exceeding the 60s timeout every cycle) - after N consecutive live failures a database's catch-up window halves per failure toward a 15-minute floor, and the range the tighter window skips rides the SAME hole records the clamp already writes: deferred to the backfill trickle, never dropped. Success resets to full width. The backfill worker gets the mirror treatment - a server whose hour-wide slices keep dying digs in progressively narrower chunks until one fits. One shared pure policy (`AdaptiveSpan`, pinned) drives both paths in both SKUs and both engine arms, so nothing can drift on how fast it backs off. -- **`--collapse-legacy-slices` no longer dies with "Exception while reading from stream" on a store fresh off a large catch-up** ([#2105] follow-up, ghauan again) - the repair's staging aggregation ran on Npgsql's default 30-second command timeout, which a heavy day-slice blows through (the verb runs beside the live service by necessity - stopping a managed store's service stops Postgres - so collector writes and compression jobs contend for the same chunks), and an Npgsql timeout surfaces as a bare stream exception that says nothing about time. Every repair statement now runs on a 15-minute per-statement timeout: generous because the slice is doing real work, bounded because the slice transaction holds chunk locks the compression policy also wants. The dry-run survey gets the same treatment, and the repair remains idempotent and resumable exactly as the failure message promises. -- **Query Store catch-up can no longer spiral a big database into permanent timeout** ([#2102], found dogfooding the use1 migration) - the live path's incremental window was `(watermark, now)` with only a 24h clamp, and the watermark only advances when a cycle SUCCEEDS. The per-database query aggregates, window-functions, and sorts its WHOLE window before `TOP` or the client byte budget can bound anything - a row cap is not a cost cap - so one missed 60s cycle (a flush burst, a CPU blip, a migration cutover) widened the next window, which cost more and timed out again, growing without bound below a clamp that sat far above the tipping point. On the field evidence the big tenants wedged at 0.5-6.5h stale and re-ran the same doomed query every cycle forever while small databases on the same servers stayed current - and the backfill worker's slices had the same latent flaw one layer down, querying a hole's full remaining range in one statement. Catch-up now trickles the way it was always meant to: the clamp is one hour (the envelope the fleet proves every day under Query Store's 900s flush cadence - and it slides forward with now, so recovery is immediate no matter how stale the watermark got), the skipped range is recorded as a hole exactly as before, and every backfill slice windows at most one hour at a time, with an empty chunk SHRINKING the persisted ceiling past the quiet hour instead of wrongly declaring the range complete (a quiet chunk on a derived-boundary tail converts the remainder to a hole record, because MIN over stored rows cannot walk through quiet space). Both SKUs, both the SQL Server and Azure SQL DB arms. -- **Multi-incident alerts render as labeled, self-contained units instead of one flat fact list** ([#2108], reported with the diagnosis by gotqn) - when one alert covered several deadlock fingerprints, the Teams card serialized every victim's fields first and every fingerprint's fields after, in one undifferentiated facts[] block - there was no way to tell which victim belonged to which fingerprint, and downstream automation could not split the card into per-incident work items. Two changes, one per layer where the association was lost: each fingerprinted deadlock is now ONE self-contained item (its Database, Victim SQL, Processes, Dedup Key, Involved Objects, and occurrence count together, headed "Deadlock N of M"), with the standalone victim items kept only for deadlocks the fingerprint cannot see (no parseable lock objects - they would otherwise vanish); and the Teams payload gives every fields-carrying item its OWN sections[] entry, titled by the item's heading and carrying only that item's facts, on every alert type - advice prose and remediation-T-SQL hints stay folded into the lead section, because they are commentary on the whole alert. One compatibility note for payload consumers: automation keyed specifically on sections[0].facts[] containing every fact should iterate sections[] instead. -- **Every database-scoped alert exposes a discrete Database fact** ([#2109], also gotqn) - only Blocking Detected and Long-Running Query carried { "name": "Database" } in their webhook facts; Deadlocks named databases only inside the involved-object strings, Version Store (PVS) only in the item heading, and Database State and the two AG database alerts (Suspended / Sync Fell Behind) only in prose - so routing or tagging by database meant parsing display text. Now: deadlock items and incidents carry the distinct databases parsed from the graph's own process list (comma-separated when a deadlock spans databases), PVS items lead with the database, Database State fires with a structured context (Database / Current State / Expected State), and the AG database alerts carry Database / Availability Group / Replica (plus Suspend Reason where it applies) through one shared builder in both SKUs, so the fact names cannot drift. All additive - no existing fact changes. -- **Query Store backfill yields to the live path on contended replicas** ([#2111], found validating #2102 on the prod monitor fleet) - the hour-chunked catch-up recovered most wedged databases immediately, but servers where the backfill worker ran a slice every tick alongside the live sweep stayed in a failure churn: both paths scan the same QS internal tables, the replicas are often MAXDOP-1, and the live query that normally finishes in seconds died at the command timeout behind the slice's scan - recovery-phase contention, self-inflicted, and on some servers the slices themselves timed out so their holes never drained. The backfill worker now skips a server's slice whenever that server's live query_store collection failed within the last 10 minutes (two poll cycles - "failing NOW"), in both SKUs, judged by one shared policy so the workers cannot drift. This is the class doc's own contract - backfill can be slow forever without delaying collection - enforced at the moment it matters: the hole waits, live recovers, backfill resumes. The signal is server-grain on purpose (any database's live failure vouches for the whole replica being contended) and in-memory on purpose (a restart forgetting it costs one slice racing one cycle, once). -- **Version stamps are single-sourced from ``** ([#2113], reported by SalmanRajwani) - the 3.4.0 release bumped `` in each app project but left the hand-pinned `AssemblyVersion` / `FileVersion` / `InformationalVersion` at 3.3.0, so the 3.4.0 packages installed binaries whose FILE metadata reports 3.3.0.0. The code was genuinely 3.4.0, but the lie was not cosmetic: the in-app update check compares the entry assembly's version (the stale pin) against the latest release tag, so a user already ON 3.4.0 would be told an update is available forever. Four hand-maintained copies of one fact is a release-day trap. The three derived properties are now deleted and derive from `` at build time (with the CI source-revision suffix suppressed so InformationalVersion stays a clean semver); a release bump is now ONE line per project. -- **Lite no longer crashes with an uncatchable stack overflow on Queries > Query Store by Duration** ([#2114], diagnosed nearly end-to-end by SalmanRajwani - WER excerpt, module analysis, and the exact XAML candidate) - #1980 ported the Query Store grid's inline View Plan button into Lite still carrying the Darling Viewer's `DarkButton` style key, which Lite never defines. A missing StaticResource inside a DataGrid cell template is not a cosmetic miss: realizing the template throws `XamlParseException` during measure, WPF re-attempts realization on every layout pass, and the recursion kills the process with `0xc00000fd` - uncatchable, unloggable, the moment the grid renders. The button now uses Lite's default chrome, and a new hygiene test scans both apps' XAML trees so a StaticResource key referenced in one app but defined only in the other can never ship again. -- **Upgrading a 3.3.0-era Darling store to 3.4.0 no longer fails the migration ladder** ([#2119], field report by ghauan on #2105) - migration rung 51 is assembled at runtime from the live view generator, which since #2069 emits the `query_plan_gz` column that rung 54 adds - so a store at or below V50 replayed rung 51 referencing a column three rungs before it existed, the ladder halted with `42703`, and the service stopped. Nothing was harmed (each rung runs in its own transaction) but the upgrade could not complete. Rung 51 now pre-adds the column with V54's own idempotent ALTERs (rung 54's copy no-ops after it), the retry just works on an already-failed store, and new ladder pins make the generated-rung replay hazard - a later column teaching a generator new SQL that an earlier rung re-emits - fail in CI instead of on an operator's store. -- **The upgrade path itself is now release-gated** ([#2119] follow-up) - the 3.4.0 ladder failure escaped because every pre-release check ran either a fresh store (full generator, no ladder) or the dogfood box (which walks each rung in the era it ships); no check pointed new binaries at a store a RELEASED build had made, the one path that replays old rungs and the one path users take. Two gates now: a committed fixture of the previous release's ladder exactly as that release resolved it (generators frozen at the tag), which `MigrationUpgradeLadderLiveTests` builds on scratch Postgres and climbs with the current ladder on every CI run - verified two-sided at birth (fails with the exact field 42703 on the pre-fix build, climbs V39->V54 clean on the fixed one, including from a mid-failure retry) - and a release-cut step that boots the previous release's container image and then the candidate on the SAME volume (docs/releasing.md). -- **Store Disk Pressure no longer re-notifies every cooldown while free space sits at an unchanged level** ([#2101], reported with the diagnosis by gotqn) - the self-alert re-fired on the plain cooldown, so a store volume parked at 7.3% free produced an identical CRITICAL notification every ~15 minutes for as long as the condition stood - one static condition, dozens of notifications. It now runs behind the same worsening gate the target-server volume alert has had since the #754 follow-up: fire once on entry, re-fire only when free% drops at least a full point below the last-alerted level (still cooldown-limited), one resolution row on recovery - which also clears the watermark, so a volume that oscillates around the threshold cannot go permanently silent. Deliberately NOT applied to the state-only self-alerts (Collection Stopped, Agent Not Running, Capture Down): those have no level to worsen, and their per-cooldown "still broken" reminder is wanted. The reporter's second ask - exposing the hardcoded self-alert thresholds as store-backed settings - is real but is its own design decision (control-plane knobs, Viewer rows, MCP settings groups) and stays tracked on the issue. -- **Compression Job Stuck no longer false-alarms on a job it caught mid-run** - measured live on TimescaleDB 2.x: from the moment the scheduler picks up a due job until its run completes, `job_stats.next_start` reads `-infinity` with `job_status = 'Running'`, and the real next start is only computed at completion. The detector's first arm treated `-infinity` unconditionally as "the scheduler will never run it again", so any healthy compression run the self-alert check happened to sample got flagged as stuck, alerted, and "self-healed" with a pointless re-arm - the transient stuck-then-self-healed alert pairs the field has been shrugging off were this false positive, and the stuck-detector live test's CI flake was the same race (it re-arms with `next_start => now()` and then read a single snapshot while the run it had just triggered was still executing). A RUNNING job's `-infinity` now defers to the elapsed-bound arm, which is what actually distinguishes a hung run from a healthy one - a genuinely dead job (`-infinity`, not running) still alerts exactly as before, and a hung run still trips the bound. -- **The Job History tab speaks display names** ([#2126], asked by ghauan) - both the Server filter dropdown and the Server column showed the raw collected server name while every other tab shows the operator's alias, so a fleet navigated by aliases turned into a memory quiz on exactly the tab an operator visits during an incident. Both readers (job history and the Agent status header) now resolve through the servers registry - the alias when one exists, the raw name otherwise - so the filter, the column, the per-column filter popup, and the CSV export all speak the same names as the rest of the viewer, and the Agent roll-up sorts by them. Lite's Job History tab had the same gap through a different mechanism (review catch): Lite's display-name concept lives at the CONFIG layer, not in DuckDB (the stored servers.display_name column is unpopulated by design), so the shell now passes a server_id-to-alias snapshot into the tab Overview-style and rows swap in the alias on every refresh - a server no longer in config keeps its raw collected name, the durable-record case. -- **The long-query completion XE session actually gets created now** ([#2129], from ghauan's field report on #2061 - they enabled the collector on two servers and the Long Queries tab stayed empty forever) - the session DDL SET a customizable attribute `collect_object_name` on `sqlserver.rpc_completed`, and no such attribute exists on that event on ANY version (it belongs to `sp_statement_completed`) - `object_name` is one of rpc_completed's DEFAULT data fields, collected with no SET at all. So the CREATE failed on every server, the session never existed, and the reconcile's follow-up START surfaced as the confusing second error ('Cannot alter the event session... does not exist'). Never caught in dogfood because the collector ships OFF by design, and the DDL test pin asserted the wrong claim, so CI enforced the bug. The SET is gone (the reader already shreds the default field generically - no reader or table change), and the pin now asserts the attribute is ABSENT, with the story attached. Anyone who flipped the collector on before this fix: it starts working on the next reconcile tick after upgrading, no re-toggle needed. -- **`--collapse-legacy-slices` narrows its slice instead of dying when a day does not fit the statement timeout** ([#2105] round three, ghauan once more - with the decompression rail lifted, the run made it ~15 minutes in and died at the NEW wall: a day-wide stage aggregation on a store carrying 60k split intervals blows through the 15-minute per-statement timeout, and the operator got the same bare stream exception) - the verb's fixed day-per-slice loop now runs the same adaptive schedule the Query Store backfill worker shipped this week (`AdaptiveSpan`, 24h base): a failed slice halves the window and retries the SAME start (announced with a [RETRY] line naming the error, so narrowing reads as progress rather than a hang), a completed slice resets to full width, and only a slice that fails at the ~22-minute floor gives up to the existing idempotent re-run message. Healthy stores still repair in a handful of day-wide slices - the narrowing costs nothing until a slice actually fails. -- **Query Store collection no longer has a fixed cost that big catalogs cannot pay** ([#2133], the actual root cause under the whole catch-up saga - #2102's death spiral, #2111's yield, and #2125's adaptive shrink were all mitigating it) - the collector joined its slice aggregate straight into the query_store_plan/query/text catalog TVFs, handing the optimizer nothing but fixed-guess cardinalities, and the plan it picked re-materialized a TVF per probe: on an 82k-plan catalog that was a fixed 30-second-plus cost that NO catch-up window width could reduce - which is exactly why the fleet's big databases (echo, oak, Surge, spruce, insa...) pinned at the 15-minute shrink floor and never converged while their smaller neighbors on the same servers stayed current. Bisected live: the aggregate alone ran in 81 ms and each TVF scanned bare in ~300 ms, yet aggregate-JOIN-plan could not finish in 30 s, hinted or not. The payload now STAGES the aggregate in a temp table and joins FROM it - real row counts instead of guesses, each TVF scanned exactly once, sp_QuickieStore's architecture for the same reason - and the old LOOP JOIN hint is gone for good (looping from the temp into the TVFs is the same per-probe re-materialization by another name). The interval pre-filter also resolves ids from the tiny interval catalog now instead of scanning runtime_stats itself (20 ms vs 426 ms, same id set). Measured end to end on the wedged field store: the full 55-column batch with plan capture completed a one-hour backlog in 21.3 s where the old shape never finished inside 60; the staged core is 524 ms. Same batch = one result set, TOP WITH TIES / derived-watermark / byte-budget semantics unchanged, both SKUs, both engine arms, live and backfill. -- **`--collapse-legacy-slices` runs ~40% fewer window scans and narrates its progress** (#2105 operator feedback - ghauan's successful run 'took some time', all of it against a silent console) - each slice bracketed its work with two window-wide COUNT(*) scans purely to compute the removed-row figure, and on the catch-up-bloated stores this verb exists for, one such scan measures ~12 seconds (hash-aggregate spill to temp disk plus a backward index scan over the uncompressed hot chunk) - paid twice more per slice for bookkeeping. The removed count now derives from the DELETE and INSERT statements' own affected-row counts (deleted minus reinserted IS the net removal, same number by construction), and every completed slice prints an [OK] line with its window, removal count, and percent-of-span - so a long backlog reads as visible progress instead of a hang. - -## [3.4.0] - 2026-08-06 - -### Important - -- **Darling keeps hourly-grain history 90 days instead of 21, and existing stores are moved to the new horizon automatically** ([#1937], also closes [#1939]) - the viewer offers month-plus windows, and at 21 days a 30-day view could never render at hourly grain: the range either emptied out or dropped to daily resolution partway through, no matter how long the store had been collecting. Retention on the five hourly rollups is now 90 days, covering quarter-scale windows with room, and the read routing follows the horizon by derivation rather than by a second hand-maintained number - the shipped defect was exactly that pair drifting (the router's hourly ceiling was an independent hardcoded 20 days, so raising retention alone would have changed nothing an operator could see). On upgrade, stores that already carry the 21-day policies are converged to 90 automatically on the next service start, preserving each policy's armed-or-held state and its next scheduled run - the convergence cannot trigger a purge. **Two honest limits.** Hourly history already purged under the old horizon is gone: raw reaches back only 4 days, so the hourly tier regrows forward from upgrade day and takes about ten weeks to fill the new window - during that regrowth, a window the hourly tier cannot yet cover COMPLETELY is served at daily grain instead of partially at hourly with the older weeks silently blank ([#1939]), so month views stay whole the entire time. And the hourly tier's steady-state footprint grows roughly 4x (compressed) to hold the deeper window. - -- **Lite repairs its stored Query Store history once, on the first launch after upgrading - expect a few extra seconds and a slightly larger archive** ([#1912]) - builds before [#1907] stored Query Store's flushed and still-in-memory slice of one interval as two separate rows. They are ADDITIVE, so every read that keeps one row per interval reported a FRACTION of that interval's work - on the live evidence, 8 executions where 94 was true. [#1907] fixed the collection side and made the read deterministic; it could not touch what was already stored. The first launch of this version collapses those rows into the single row the collector would write today, in the hot store **and in the parquet archive**, and records that it has done so - it runs once, not on every start. On a real archive the Query Store files totalled ~46 MB and the largest (533,109 rows) rewrote in under a second, so "a few extra seconds" is the honest expectation rather than a warning. - - **Your archive may get slightly BIGGER even though rows were removed, and that is not a fault.** Parquet size is dominated by how well each column compresses within a row group, not by row count, so re-encoding a file can cost more than the rows it dropped - measured at 28.0 MB becoming 31.6 MB while 22,760 rows were removed. The repair keeps the archive's own ZSTD compression; it is the re-layout, not a codec change, and the data is both correct and fewer rows than before. - - **Nothing is at risk if it is interrupted.** Each archive file is rewritten beside the original and swapped in only after its row count AND its total execution count both check out - the second is the invariant the collapse's own arithmetic guarantees, and it is what would catch a bad aggregate that still produced the right number of rows. A file that fails verification is left exactly as it was, logged, and retried on the next launch; the other files still get repaired, so one bad file cannot block the rest. No backup copies are kept, because verify-before-promote is the safety. Until a file is repaired it keeps being read normally, with [#1907]'s deterministic tie-break resolving it, so a half-finished repair is a delay and never a corruption. Lite always starts, even if the repair cannot run at all. - -- **OPERATOR ACTION, Darling only: run `--collapse-legacy-slices` PROMPTLY after upgrading, and the sooner the better** ([#1912]) - the same repair Lite does for itself, on the store where Darling keeps the same rows. It is deliberately a verb rather than an automatic step, for the reason `--backfill-rollups` is ([#1759]): it rewrites stored rows and re-materializes rollups, so it runs when a person decides to, and `--dry-run` reports exactly what it would change without changing anything. - - ``` - PerformanceMonitor.Darling.Service.exe --collapse-legacy-slices --dry-run - PerformanceMonitor.Darling.Service.exe --collapse-legacy-slices - ``` - - **Timing is the whole point, and it is unusual enough to be worth stating plainly: this verb gets LESS useful every day you wait.** It can only repair Query Store rows that are still in the raw tier, which holds a few days. Run it right after upgrading and it fixes the recent history you are actually looking at, and the rollups built from it. Run it a week later and there is nothing left in raw from the affected period, so it will correctly report nothing to do while the numbers stay wrong. **What it does not reach is permanent** - the daily rollups are kept indefinitely and cannot be rebuilt from raw that retention has already dropped, so Query Store execution counts and count-weighted totals for the pre-upgrade period stay UNDERSTATED for the life of that store. There is no ordering of these operations that reaches further back; this is a disclosure, not a step you can take later. Safe to re-run at any time - a second pass finds nothing, because the signature it matches cannot occur in rows collected since [#1907]. Applies to every Darling store including **DARLING01**. - -- **Version Store (PVS) pressure alert in both apps** ([#1984], the alerting follow-up [#1951] deferred) - a new alert (default on) fires when an ADR database's persistent version store reaches a share of the database's own data files - 40% by default, warning meaningfully before the "close to 50% of the database size" that Microsoft's troubleshooting guide calls large. Percent-of-database rather than absolute size because a shipped absolute guess is workload-specific and would page half a fleet (the `ag_redo_queue_alert_kb` precedent shipped OFF at 0 for exactly that reason); the ratio is also what both FinOps grids already compute as PVS % of Database, so the alert and the grid can never tell different stories. A second knob, the **GB floor** (default 1 GB), is an AND qualifier rather than the volume alert's either-breach-fires OR: a 10 MB database at 60% is six megabytes, and nobody should be paged for six megabytes - 0 removes the floor, and a percent of 0 disables the check outright since percent is the alert's only trigger. One alert per server names the worst (highest-share) database with up to five breaching databases in the context, each carrying its PVS size, data-file denominator, aborted-transaction count, whether the aborted-version cleaner is mid-run (Microsoft's start-time-without-end-time shape), and the aborted/active transaction-id lag - presented as the gap itself, never a verdict, for the same reason the grids refused to invent a threshold Microsoft does not document. No severity tier either, for the same reason. **Re-fires only on a fresh or WORSENING breach** (5 percentage points over the last-alerted level): measured on a live rig, PVS space stays allocated even after the pinning transaction clears and the cleaner runs to completion, so a plain per-cooldown level check would re-notify for hours after the incident ended - the same standing-condition treatment the volume alert got, with the direction flipped because a version store worsens by rising. Fires through the shared cross-app alert path, so Lite and Darling evaluate identically from their stored `pvs_stats` (hourly), with the same cooldown, mute, alert-history, tray, and email plumbing, and a resolved notice when every version store drops back under. Darling's knobs ride the store control plane (V48 migration; the Viewer gains the Settings row, its schema gate a V48 rung, and `get_alert_settings`/`update_alert_settings` a `pvs` group); Lite's live in `settings.json` (`alert_pvs_*`) with its Settings row. Validated end-to-end against a Docker SQL2022 rig carrying real ADR pressure - a pinned-cleanup workload grown to 78% PVS-of-database - through the real collector, store, read-adapter, and engine path. - -- **Lite's data now lives OUTSIDE the install directory - and on any version older than this one, upgrading with `Setup.exe` destroys it** ([#1832]) - Lite kept everything in `%LOCALAPPDATA%\PerformanceMonitorLite`, which is also the installer's own install root. Re-running `Setup.exe` over an existing install renames that folder aside and deletes it, so an installer-based upgrade took the DuckDB store, the Parquet archive, the logs, and `settings.json` with it, in one move, before any code from the new build ran. **In-app updates were always safe** - Help > About replaces the `current\` subfolder in place and never touches the data sitting beside it - which is precisely why the loss looked random instead of reproducible. Two things always survived and still do: the monitored-server list (`%ProgramData%\PerformanceMonitorLite\config\servers.json`, machine-wide) and every password and webhook URL in Windows Credential Manager. - - **The one-time caveat: reaching THIS version via `Setup.exe` still loses your data.** The installer deletes the old directory before the new build starts, so no fix shipping inside the new build can prevent it - the code that would migrate your data has not run yet when the data is deleted. Upgrade **in place** instead and nothing is lost: **Help > About** downloads and applies the update, or extract the portable ZIP over your existing copy. From this version forward `Setup.exe` is safe, because the data is no longer where the installer writes. - - Data now lives in the sibling `%LOCALAPPDATA%\PerformanceMonitorLite-Data`, and the first start MOVES `config\`, `archive\`, `logs\`, `monitor.duckdb` (plus its WAL) and `alert_state.json` there from the old location automatically - a same-volume rename, so a multi-GB store transfers instantly. Nothing already in the new location is overwritten, nothing anywhere is deleted, and a `DATA-MOVED.txt` signpost is left behind pointing at the new path. The move is per-artifact rather than all-or-nothing, so a run interrupted by a locked store file finishes itself on the next start rather than stranding the store in a folder the next installer run would delete. The updater's own `Update.exe`, `current\` and `packages\` share that old folder and are deliberately left where they are. - -### Added - -- **Stable releases now ship the Linux artifacts** - before this, the linux-x64 service tar.gz and the container image existed only at nightly quality (tagged `:nightly`), so the compose quickstart pointed RELEASED users at a nightly image and a stable tag had no Linux download at all. The release pipeline's Linux job now uploads `PerformanceMonitorDarling-linux-x64-.tar.gz` plus `SHA256SUMS-linux.txt` to the release and pushes the container image to ghcr tagged both `:` and `:latest`. Linux artifacts are not SignPath-signed (container and GPG signing are edition-gated there); instead the released image carries a **keyless Sigstore signature** — verify with `cosign verify ghcr.io/erikdarlingdata/performancemonitor-darling: --certificate-identity-regexp 'github.com/erikdarlingdata/PerformanceMonitor' --certificate-oidc-issuer https://token.actions.githubusercontent.com` — and the tarball carries **GitHub SLSA provenance** (`gh attestation verify -R erikdarlingdata/PerformanceMonitor`) beside its SHA-256 checksums. Both prove the exact bytes were built by this repository's release workflow, and both were verified end-to-end from a client machine against a signed nightly before shipping. - -- **OPTIONAL, Darling only, for stores that collected before this release: `--recompress-plan-dim` converts the plan dimension's existing text rows to the gzip form new plans already use** ([#2076]) - the gzip change ([#2069]) applies to plans written from V54 on; everything collected earlier stays lz4-TOASTed text until the dimension GC happens to retire it, and a STABLE plan's row never retires (every sighting refreshes its watermark), so on a long-running store a permanent tail keeps paying ~15 KB per plan instead of ~10 KB. This verb is the operator-paced version of the rewrite the migration deliberately refused to do implicitly (the `--collapse-legacy-slices` rationale: rewriting stored rows runs when a person decides, with `--dry-run` first - which here also measures the real ratio on a sample of YOUR store's plans before anything is written). **No outage**: the service stays up and collecting throughout; the conversion runs in 1,000-row batches, each its own transaction, and an interrupted run resumes from wherever it stopped because the fetch predicate is the resume point. Every row's gzip bytes are round-trip verified against the original text before the text is nulled - a row that fails verification keeps its text untouched and is reported. The content digest and the GC watermark are both deliberately untouched: identity was always computed over the uncompressed text, and recompression is not a sighting, so dedup, fact-row references, presence flags, and retention timing all come out exactly as they went in. **The run ends by actually returning the space to the volume**: converting rewrites rows, so on its own the file would keep its high-water mark with the freed space merely reusable inside the relation - invisible to df and to your monitoring. So when the conversion CONVERGES with zero failures, the verb finishes with a `VACUUM FULL` of the dimension, rewriting it to its live content - preflighted against free disk on the store's own volume (refused with numbers when it will not fit, and the conversion stays committed either way), disclosed as the one step that takes an EXCLUSIVE lock - measured on the production reference store: ~46 minutes of lock for a 174 GB dimension (7.1M plans, rewritten to 77 GB), with collection freshness degraded for the duration and fully recovered within five minutes after; the service does not stop, and smaller dimensions scale down proportionally. `--no-vacuum-full` skips it for operators who want the lock window scheduled separately; `--vacuum-full` compacts a store that was already converted earlier (validated on the production 52-replica store: 6.58M plans, 885 GB of raw XML, 4.5 hours, 14.4x, zero verification failures). New installs never need this verb: every plan they ever store is gzip from the first collection. - -- **The store measures itself: an hourly self-metrics sweep records every hypertable's size and compression, the payload dimension tables, and the whole store into `collect.store_metrics`, with a `get_store_metrics` MCP/REST read for capacity forecasting** ([#2068]) - forecasting the monitor store's growth required ad-hoc archaeology over the TimescaleDB chunk catalog, and the evidence kept evaporating: raw chunks age out in 4 days, so when compressed daily ingest jumped ~3x around Aug 1 (620 MB/day to ~2.3 GB/day) nothing recorded it - the jump was only reconstructible at all because the chunk catalog still happened to hold both eras. The store's single biggest forecasting term was also invisible to every surface: on the production 52-replica store (147 GB), the `query_plan_dim` payload dimension alone was **101 GB - 69%** - a deliberately PLAIN table no hypertable-shaped read ever counts. The service now sweeps its own store hourly (the disk-check/compression-check fleet-level idiom): one row per hypertable - total bytes, pre/post-compression bytes (compression measured excellent but previously only measurable by hand: 6-36x), chunk count - from the TimescaleDB catalog, skipped silently on plain-PostgreSQL stores; one row per payload dimension (`pg_total_relation_size`, which counts the TOAST the plan XML actually lives in, plus the exact row count); and one whole-store summary row (`pg_database_size` plus the enabled-server count, so the per-server ingest rate - the number onboarding N primaries multiplies - is derivable from the stored series alone). The series lands in a new PLAIN table the compression/retention machinery can never recurse onto (not a collector table, not a hypertable - pinned by test), bounded by the sweep's own 400-day DELETE instead of policy machinery (~30 narrow rows/hour - nothing), and `get_store_metrics` (MCP + `/api/read`, `days_back` default 30) returns the latest snapshot per object plus a settled daily series with the whole-store growth and derived per-server rate. Darling V53; the viewer's schema gate knows the rung, so a fully-migrated store is never refused. - -- **New execution plans are stored gzip-compressed, shrinking the store's single largest object by a projected further ~35% with no change to what is captured** ([#2069]) - the self-metrics sweep ([#2068]) put a number on the storage picture: on the production 52-replica store, the `query_plan_dim` payload dimension was **101 GB of a 147 GB store (69%)** - 6.5 million distinct plans whose XML PostgreSQL's lz4 TOAST was already compressing at a measured 9.2x. A bake-off on that store's own live content (not synthetic plans) measured application-side gzip at **14.0x against lz4's 8.9x on the identical sample**, so the dimension's steady state projects from ~101 GB to roughly 64 GB. gzip rather than zstd because PostgreSQL 18's TOAST offers only pglz and lz4 (verified against the shipped binary, not the docs) and .NET ships gzip natively; zstd is tracked separately ([#2071]). The service now compresses each distinct plan ONCE at write time into a new `query_plan_gz` bytea column, leaving the dimension's text column NULL on new rows. The content digest is still computed over the UNCOMPRESSED text, so a plan first seen before this version and seen again after lands on the SAME dimension row - dedup, fact-row digests, and every has-plan presence flag survive the change untouched. **No rewrite, no migration downtime, no peak-disk spike**: existing text rows are left exactly where they are, every reader resolves text-else-gz, and the dimension GC retires unreferenced text rows on its normal multi-day horizon (~9 days measured on the production store), so the store converts itself as it turns over. Every product surface follows both forms - the viewer's plan opens, the MCP plan tools (`get_plan_xml`, the analyzers), the FinOps plan actions, and the actual-plan command's stored-plan resolution. **The one honest loss is raw SQL against the store**: `v_query_stats.query_plan_xml` is NULL for plans written since this version - PostgreSQL cannot gunzip in SQL, so the view carries the bytes in `query_plan_gz` instead, and an ad-hoc psql consumer that wants the XML must pull those bytes through any standard gzip tool (they are an ordinary RFC 1952 gzip member, recoverable with `gunzip` alone). Darling V54; the viewer's schema gate knows the rung, so a fully-migrated store is never refused. - -- **The web fleet gains tags: coloured pills, a group-by-tag tree, and tag-name search** ([#2020], stage 2b-ii, completing the issue) - the tag concept the Darling Viewer and Lite already have reaches the read-only web dashboard on all three counts the issue asked for. Each `/api/fleet` card carries its server's tags, rendered as coloured **pills** below the status line; the fleet page's existing **search box now matches tag names** as well as the display/instance name (via the same rule the desktop `ServerOverviewFilter` uses - so `prod` finds both `sql-prod-01` and everything tagged Production); and a **Group by tag** toggle rearranges the cards into a collapsible **tag tree** (nested tags → Untagged), the read-only web twin of the desktop sidebar - built from the full tag forest `/api/fleet` now returns, so an organisational parent tag with no directly-tagged servers still nests its children correctly, and mirroring `FleetView`'s collapse-reveal (collapsing a tag hides its whole subtree) and multi-tag duplication. The toggle and collapsed groups persist client-side. The pills use the stored `#RRGGBB` colour, falling back to a neutral pill for an uncoloured tag, with the same fixed neutral + light label the desktop `TagColorBrushes` use, so a pill reads identically whatever the theme - and no colour palette is resolved server-side (the headless service builds for Linux and stays free of the WPF `PerformanceMonitor.Ui` assembly). **Display-only, matching the web seat's read-only philosophy**: tags are created and assigned in the Viewer or Lite; the web only shows them. Delivered through the shared fleet reader, so the per-server tags also appear on every `get_fleet_overview` card an MCP client reads - a bounded `server_tag_map`→`server_tags` join keyed on `server_id` plus the tag forest, needing no new grant (the read-only role already SELECTs the config-schema tag tables). Completes the tag rollout across all three surfaces ([#2020]); the earlier stages brought tags to the Viewer (2a) and Lite (2b-i). - -- **Darling runs on Linux: a compose file for the whole stack, a systemd walkthrough, and the store connection string as a secret reference** ([#1804] stage 4, completing the distribution the issue asked for) - \`Darling/compose/\` carries the deployment markallisongit asked for: the service container paired with the official \`timescale/timescaledb\` image, healthchecked, on a named volume, with every secret delivered as a compose \`secrets:\` file that \`darling.json\` references - including the WHOLE \`postgres.connectionString\`, which now also takes an \`env:\`/\`file:\` reference (resolved once at the parse seam; the password lives inside the string, so per-field indirection could not reach it, and a literal string is byte-for-byte unaffected). The sample config ships secret-free by construction. The Darling README gains the Linux section: the compose quickstart, the three rules that bite (secrets never in the file; fresh store volume per deployment because the control plane is store-authoritative after the first seed; file permissions are the operator's on Linux - the container boundary or chmod stands in for the Windows ACL lockdown), and the systemd + bring-your-own-PostgreSQL unit for shape two, with the \`libgssapi-krb5-2\` dependency called out. Validated live before merging: the checked-in compose file stood up both containers on a fresh volume, the service resolved the store through the secret-file reference, collectors flowed, the web token->cookie gate and MCP bearer gate answered correctly through the mapped ports, and the logs carried zero secrets and zero plaintext warnings. The Viewer stays Windows; Linux hosts read the web dashboard. - -- **Enabling a default-OFF collector fleet-wide actually takes effect now, and the schedule editor stops showing it as already-enabled** ([#2064], root-caused from ghauan's [#2061] report) - two bugs that compounded into one very confusing symptom. First, the editor's baseline schedule hardcoded every collector as ENABLED instead of reading each collector's own shipped state, so `long_query_completions` (which ships OFF by design - it creates an XE session on your server) appeared **checked** at default scope while the feature was off. Second, the fleet-scope save's sparse-row filter compared frequency and retention to the code defaults but tested enabled-ness as bare truth rather than against the collector's DEFAULT enabled state - so ticking that box at fleet scope, with frequency and retention left alone, wrote **no row at all**: the save reported success, the store held nothing, and the Long Queries tab correctly said the trace was off. Server-scope saves always worked (they write a full snapshot), which is exactly what made the report read as a caching problem rather than a persistence one. Both are one-line fixes; the test that should have caught the first one asserted every collector was enabled, which is how the hardcode survived - it now pins each collector's own default, and a regression test covers enabling a default-OFF collector at fleet scope. - -- **The Viewer's Settings dialog keeps Save and Close pinned below the scroll** ([#2063], requested by ghauan) - the action buttons scrolled with the settings list, so a change made near the top of a long dialog meant scrolling to the bottom to save it. The bar now sits outside the scroll region, always visible - the Collector Schedules dialog's always-visible-actions shape, applied to Settings. - -- **Findings now keep their evidence: the drill-down behind every persisted finding survives read-back, and `get_analysis_findings` can return it** ([#2060]) - a finding could say *"13 plans showed the pattern"* while no surface could enumerate them: the drill-down (the specific parameter-sensitive plans, spill queries, blocking chains) was ephemeral, built on the write path and gone on read-back, so a triaging agent had to re-derive the list the engine had already computed - the #2054 fleet-chain triage did exactly that. The fix is the `remediation_action_json` pattern applied to the evidence itself: a **capped** copy persists beside the built action (first 10 rows per section - collectors emit worst-first, so the head is the signal - and a 64 KB total bound, with an explicit `_truncation_note` naming anything dropped, never a silent cap) and deserializes back into the finding. Both MCP `get_analysis_findings` tools and the REST route gain `include_drilldown` (default off - the envelope is already dense; the summary usually suffices), returning the chain's latest occurrence's evidence rows. Darling V52 / Lite analysis-schema v5 add the nullable column; no backfill - findings persisted before the upgrade honestly return no drill-down and age out with finding retention. `analyze_server` (live) is unchanged and still returns the full uncapped drill-down. - -- **Darling backfills the Query Store history the live path never takes — newest-first, on its own tick, never past the raw tier's horizon** ([#2022], phase 2 of [#1960]) - phase 1 made the LIVE path hole-free, but two bounded windows still discard history by design: first contact takes only the trailing 60 minutes of a ~30-day catalog, and post-outage catch-up is clamped to 24h (the [#1556] incident fix) as a logged hole. One worker now fills both. The first-contact tail needs NO stored state: the backfill ceiling is derived exactly like the live watermark - MIN(last_execution_time) over the rows already stored - because both of phase 1's bounded cuts complete the boundary tie group, so each newest-first slice ships strictly below the stored floor and the write itself advances the boundary; a pre-existing store whose history already reaches the horizon marks itself done without shipping a row, which is what makes deploying this to a long-running fleet a no-op. Clamp holes are interior gaps MIN/MAX cannot see, so the runner records them at the moment the clamp fires ((raw watermark, clamped floor), merged WIDER on a repeat outage) under the worker's own `collector_state` rows - deliberately not the definition's StateKeys, so query_store still declares none. Slices reuse the live path's everything: the same payload body with the window and direction flipped (`> @floor_time AND < @ceiling_time`, DESC - strict bounds are resumable for the same tie-group reason the live path's strict `>` is), the same 64 MB per-database byte budget, the same COPY writer. Backfilled rows carry a BACKDATED collection_time (the slice ceiling) so they land in time buckets beside their own activity - and that is exactly why the worker refuses to dig below the raw tier's read horizon (derived, not hand-maintained - the [#1937] rule): inside it every hourly CAGG's 3-day start_offset re-materializes the touched buckets and the 4-day raw retention cannot immediately drop them; deeper backfill interacts with rollup routing and is [#2022]'s explicitly staged next decision, not an accident of this one. Runs one slice per server per 5-minute tick on its own loop (the command-plane precedent), sequentially - sequence is the fleet-wide concurrency bound - without ever touching the sweep gate, so backfill can be slow forever without delaying collection. The live watermark cannot see backfilled rows by construction (MAX ignores older values), satisfying [#1960]'s never-race constraint. Azure SQL DB targets ride the same state model on per-database connections ([#2058] - the window travels as command parameters, since Azure rejects the on-prem `sys.sp_executesql` nesting, and the clamp-hole recording gained its twin on the Azure per-database branch); Lite runs the same worker on its own terms ([#2058], completing the backfill across both SKUs): the stored contract (state identity, hole codec) is shared so the two products can never disagree on what a hole row means, while Lite's HORIZON is its resolved query_store retention (per-server schedule, default 30 days) rather than Darling's 3-day raw tier - Lite has no CAGGs or tiered retention to respect, and a backfilled row's backdated collection_time ages through retention and the parquet archive on the same clock as live rows, so digging the full retention depth is safe by construction. One slice per server per 5-minute tick on Lite's background IfDue ladder, clamp holes recorded at both of its watermark sites, Azure targets included from day one via the same shared per-database query. - -- **The "server silenced" muted-bell reaches Lite's sidebar too** ([#2031], the Lite half) - a whole-server alert silence already worked in Lite (right-click a server's tab badge → Silence), but nothing showed that a server WAS silenced, so a muted server looked identical to a healthy-quiet one and was easy to forget. Lite's sidebar rows **and its Overview cards** now show the same subtle greyed **bell-with-slash** the Darling Viewer and web fleet gained in [#2031]'s first half - a vector glyph (never an emoji) beside the status dot, visible only while the server is silenced, with an "Alerts silenced for this server" tooltip. (The bell rides both the sidebar and the NOC-style Overview grid, so a silenced server reads the same on either surface - the web already shows it on its fleet cards.) It rides the existing 30-second status poll (an in-memory read of the persisted silence set - no store hit) and every list refresh, and the Silence/Unsilence handlers flip it instantly rather than waiting for the next tick. Display-only, exactly like the other surfaces: silencing and unsilencing stay where they were, on the tab badge's menu (which already greys out the inapplicable one). Completes [#2031]; the Viewer and web halves shipped separately. - -- **The Darling service builds for Linux on every PR and ships a nightly container image** ([#1804] stage 3) - the collector service is cross-platform .NET on purpose, but nothing PROVED it: the linux-x64 publish and the container image built at release time or never. A checked-in \`Darling/Dockerfile\` builds from source (sdk stage publish into the official aspnet runtime image - which sets the \`DOTNET_RUNNING_IN_CONTAINER\` marker the stage-2 bind gate keys on, and gains \`libgssapi-krb5-2\`, which Microsoft.Data.SqlClient probes at connect time on Linux even for SQL auth: found the hard way when the container smoke's first SQL connect failed on the missing library). CI gains a path-filtered \`darling-linux\` job answering exactly two questions on every Darling PR - does the service still publish for linux-x64, and does the image still build - and the nightly gains a linux job that uploads \`PerformanceMonitorDarling-linux-x64-*.tar.gz\` (its own checksum file; the Windows job owns SHA256SUMS.txt) and pushes \`ghcr.io/erikdarlingdata/performancemonitor-darling:nightly\`. The whole path was validated end-to-end locally first: the containerized service against dockerized TimescaleDB and SQL Server 2022 ran 25 collectors green with an \`env:\`-referenced SQL password, the web login/token/cookie gate and the MCP bearer gate answering correctly through mapped ports - a run that also caught a config-validation warning still claiming the network block is IGNORED in containers while the host correctly exposed it (fixed), and proved the store-authoritative control plane overrides the file on a reused store exactly as documented. The bundled pg-runtime is deliberately absent from the linux artifact: compose pairs the service with the official timescale/timescaledb image, and managed mode stays Windows. - -- **Network exposure works in a container: the bind ladder's managed-mode gate extends to \`managed OR containerized\`** ([#1804] stage 2) - under \`postgres.managed = false\` the shared bind ladder refuses web/MCP network exposure and degrades to loopback-only ('your own reverse proxy governs BYO exposure') - the right rule on a host, and dead-on-arrival inside a container, where compose port mapping cannot reach a loopback bind and the mapping itself IS the boundary the reverse-proxy rule was standing in for. The pure decision table gains one input: \`inContainer\` (the official .NET images' \`DOTNET_RUNNING_IN_CONTAINER\` marker, read by the callers and passed in so the ladder stays pure), and the managed requirement relaxes for it - NOTHING ELSE does: the bearer token and the valid same-family \`allowFrom\` CIDR are required identically, a containerized not-exposed block skips the now-false 'network.* is ignored' notice, and the uncontained BYO rule is byte-for-byte unchanged, all pinned by new decision-table tests. - -- **Secrets without DPAPI: every plaintext secret slot also takes an \`env:NAME\` or \`file:/path\` reference** ([#1804] stage 1 - the largest piece of the Linux/compose distribution, and useful to Windows BYO-store shops today) - the DPAPI fields (\`encryptedPassword\`, \`encryptedToken\`) are Windows-only by nature, and the only alternative was a literal secret sitting in \`darling.json\`, warned on every use. A monitored server's \`password\`, the new \`smtp.password\` (before this, SMTP had ONLY the DPAPI field - non-Windows hosts had no email-alerting path at all), and the mcp/web \`network.token\` slots now all accept a reference: \`env:\` reads the named environment variable, \`file:\` reads the file's trimmed contents (compose \`secrets:\` mounts end with a newline, and a newline inside a password is never what the operator meant). A reference is not a secret in the config file, which is the whole point - it does not trip the plaintext warnings - and a missing or empty target is a configuration error naming both the setting and the target, never a silent empty secret. The prefixes match case-sensitively and only at the start, so an exotic literal is expressible via a file. DPAPI stays preferred and byte-for-byte unchanged on Windows. - -- **A headless \`--test\` flag on the Darling Viewer: the #1954 connection self-test without the UI** ([#2005], the half #1954 deferred) - the connect-failure overlay's Run Self-Test button runs a six-layer ladder (config parse, DNS, TCP, TLS, authentication, a real query) that pins WHICH layer between the viewer and the store is broken, but it needs a clickable window - useless for scripted diagnostics over an SSM session, which is exactly where the monitoring box gets debugged. \`PerformanceMonitor.Darling.Viewer.exe --test [--config ]\` now resolves the config exactly like startup (explicit path, then DARLING_CONFIG, then the conventional locations), prints the #1966 provenance block (which file, which rule picked it, the non-secret connection summary with the #1970 certificate anchoring) and the full ladder report to the console AND the viewer log, and exits 0 when no layer failed / 1 otherwise - never a window, never the single-instance dance, so a diagnostics run can't surface or fight a running viewer. The WPF wrinkle the deferral was about is handled honestly: a GUI-subsystem binary has no console, so the flag attaches to the launching shell's (allocating one when double-clicked), rebinds the null-device standard streams, and documents the one inherited limit - an interactive shell doesn't wait for a GUI process, so scripts sequence on the exit code via \`Start-Process -Wait\`. The pure contracts (flag detection, the config-path rule, the exit-code rule) are pinned by tests; the console dance itself rides the box dogfood. - -- **\`get_pvs_stats\`: the ADR persistent version store gets a browsable MCP reader on both hosts, plus the web mirror** ([#2029], the sibling of [#2028]'s parity audit) - PVS was reachable over MCP only indirectly (the alert-settings knob group and the custom-view compose measures), which an agent scanning tool names for "what is my version store doing" would never find. The tool tells the same story the FinOps grid, the [#2018] trend chart, and the [#1984] pressure alert tell, from the same reads: per-database PVS size with percent-of-database computed from the SAME data-file denominator every other surface uses (so none can disagree), online-index version store size, aborted-transaction counts, version-cleaner run state in Microsoft's shape (a start time without an end time means mid-run), and the oldest active/aborted transaction ids - presented as the gap itself, never a verdict, for the same reason the grids refuse to invent a threshold Microsoft does not document. The trend is an opt-in knob (\`trend_hours_back\`, default 0 = snapshot-only) returning the [#2018] chart's exact top-5-by-newest-size window with per-point percentages; both reads and both semantic pins (newest-collection snapshot; top-5-at-newest with same-row denominators) are held by tests. Same name, shape, and params on Lite's MCP host over its existing FinOps reads, and mirrored at \`/api/read/get_pvs_stats\`. - -- **Automatic plan correction is finally agent-readable: a \`get_plan_corrections\` tool on both MCP hosts, the web read mirror, and custom-view measures** ([#2028], found auditing collector-surface parity) - \`plan_correction\` was the ONE collected table with no agent-readable path at all: no reader tool, and absent from the custom-view MeasureCatalog, so "did APC flip or unforce a plan on this server last night?" - a first-class explanation for sudden plan-shape changes - had no answer even though both desktop apps render the data. One tool now returns both layers the collector captures in each row: the windowed FORCE_LAST_GOOD_PLAN recommendation/action rows (state, reason, score, estimated gain, regressed-vs-last-good plan ids and their execution/CPU stats, who initiated an execute/revert and when, the query text), and the newest per-database automatic-tuning enablement snapshot (desired vs actual state with the engine's reason - the "is it on and actually working" check). The two reads mirror the Viewer's grids exactly, including the semantic split a naive read gets wrong: a database with nothing to recommend lands an enablement-only row whose recommendation fields are NULL - the recommendations read drops those, the tuning read keeps exactly one per database at the newest capture, and both rules are pinned by tests. Same name, same shape, same params on Lite's MCP host (over its archive-union view) and on Darling's, plus the \`/api/read/get_plan_corrections\` web mirror. Custom views gain the table too: \`plan_correction_captures\` (count of captured recommendation rows - honestly named, since a long-lived recommendation lands one row per collection cycle) with database and recommendation-state dimensions, so "APC activity by state over time" is one panel away. - -- **A silenced server now LOOKS silenced - a muted-bell indicator beside the status dot, and Silence/Unsilence are mutually exclusive** ([#2031], proposed from the field - the Viewer and web halves; the Lite sidebar half follows the in-flight [#2020] sidebar refactor so the two don't collide) - a whole-server alert silence kept working invisibly: nothing on any surface said "this server is muted," so a silenced server was indistinguishable from a healthy-quiet one and it was easy to forget a mute and miss real alerts. The Viewer's sidebar rows now show a greyed vector bell-with-slash (never an emoji) immediately right of the status dot while a whole-server silence is active - refreshed on the same poll cadence as the alert badge (so a silence created from another seat or over MCP surfaces within a tick) and flipped instantly by the row's own Silence/Unsilence. Those two menu items are now mutually exclusive the way Lite's already were: the inapplicable one is DISABLED with a tooltip saying why, never hidden (the [#2011] disabled-with-reason idiom), driven from the polled state so opening the menu costs no store read - and the click handlers still re-check live rules, so a stale flag degrades to a status-bar note, never a wrong write. The web fleet gets the same honesty read-only: \`/api/fleet\` and \`get_fleet_overview\` now carry \`is_silenced\` - computed in SQL as the exact mirror of the Viewer's whole-server-silence predicate (server-scoped case-insensitively on the displayed name, enabled, unexpired, and with every narrowing pattern NULL, so a metric/database/query mute never renders the whole-server bell; a new pin holds all five NULL guards) - and the fleet cards render the same masked-vector bell beside the dot. Display-only on the web seat by design: silencing stays with the Viewer and MCP. - -- **The Query Store grid gains the inline plan button its sibling grids always had** ([#1980], from the [#1949] census) - Query Stats and Procedure Stats both surface stored plans inline; Query Store required the context menu or a double-click into the history window. Both apps' Query Store grids now carry a **View** button per row that opens the STORED plan in the Plan Viewer through the exact path the history window's View Plan uses - Lite's rows already carry the plan text (button gated on its presence); the Darling viewer fetches by the row's database/query_id/plan_id from the store. No live-server hit on either app, and a row whose plan was never captured gets the same "Plan Not Found" treatment the sibling buttons show. - -- **A PVS trend chart on the FinOps Version Store tab, in both apps** ([#1984] part 2, completing the issue) - the PVS grid answers "what is my version store doing right now"; it never answered "when did it start growing", which is the actual story in the silent-disk-eater case. Above the grid, both apps now chart the last 7 days of hourly \`pvs_stats\` points for the **top-5 databases by current PVS size** - one line per database, with each legend label carrying that database's latest %-of-database (computed per point from the same data-file denominator the grid uses, so the two surfaces cannot disagree). The chart hides entirely on servers with no PVS history rather than rendering a dead axis, and the top-5 cap is what keeps a 90-day-retention series readable on a many-database instance - the grid below still lists every database. The [#1951]-deferred alert half shipped earlier as the PVS pressure alert; this closes the visibility half. - - -- **The Procedure Stats comparison grid gains the text column its two siblings always had** ([#1981], from the [#1949] census) - the Query Stats and Query Store comparison grids both carry query text; the Procedure Stats comparison carried only the name, in both apps. Each row now shows a **representative statement** captured from inside the procedure - the latest `query_stats` text sharing the module's normalized `sql_handle`, the same join the [#1568] module attribution relies on - because `procedure_stats` stores no text of its own. The column header says "Statement (representative)" rather than pretending to be the definition: it is one statement the plan cache happened to hold, there to orient (“which proc is this again?”) and to copy from, not to diff procedure versions with. Symmetric in Lite and the Darling viewer, resolved through the [#1767] payload dimension on Darling. - -- **Lite's sidebar groups the fleet into its tag tree, with inline assign and tag CRUD** ([#2020], stage 2b-i-b) - the tags stage 2b-i added to Lite could be edited and shown as Overview pills, but the sidebar was still a flat server list. It now renders the same **Favorites → tags → Untagged** tree the Darling Viewer's sidebar does: a server carrying any number of tags appears under each, tags nest (up to four levels, indented), favourites float to a pinned group at the top so collapsing a tag can never hide a starred server, and an Untagged group always accounts for the rest of the fleet. Fully opt-in — with no tags defined the sidebar stays the plain flat list it always was, so nothing changes until the first tag exists. Groups expand/collapse from a disclosure chevron and that state **persists across restarts** (a new `collapsed_fleet_groups` list in `settings.json`, keyed by group so a tag rename keeps its state). Assign inline from a server row's right-click **Assign Tags** checkable submenu (indented by nesting), do tag CRUD from a group header's right-click (**New Tag / New Child Tag / Rename / Delete**, disabled on the Favorites/Untagged pseudo-groups), or open **Manage Tags** for the bulk editor as before. Built on a WPF-free `FleetView` projection — the Lite twin of the Viewer's — pinned by `LiteFleetViewTests` (opt-in flat default, Favorites/nesting/Untagged, collapse hiding descendants, multi-tag duplication, selection skipping headers, and cycle / dangling-parent safety), so the tree's rules are asserted without a window. The sidebar `ListView` became a `ListBox` rendering a mixed header+server projection via a template selector; every "what servers exist?" consumer already read the server manager rather than the bound list, so the projection changes only what the sidebar shows. The Overview search box's placeholder and tooltip now say **server name / tag** rather than just server name, since stage 2b-i already made that box match tag names too. Stage 2b-i-b of [#2020]; web parity is 2b-ii. - -- **Server tags come to Lite: a Manage Tags editor, colour, coloured pills, and search-by-tag on the Overview** ([#2020], stage 2b-i) - Lite gains the tag concept the Darling Viewer already has. A new **Manage Tags** window (New Tag / New Child / Rename / **Colour…** / Delete, plus a per-tag server-assignment checklist) organises the fleet into a hierarchical tag tree stored in DuckDB — `server_tags` + `server_tag_map`, the twin of the Viewer's Postgres tables including the `#RRGGBB` colour. A new tag is auto-assigned a colour from the shared, theme-safe palette (rotated by tag id, so it's stable across re-creation) and can be recoloured or cleared from the same swatch picker the Viewer uses. Each server's tags render as coloured **pills** on its Overview card (neutral when uncoloured, hidden when a server has none), and the existing Overview **search now matches tag names** as well as server names, via the shared `ServerOverviewFilter` — so `prod` finds both `sql-prod-01` and everything tagged Production. Because Lite is single-connection with the user's own credentials, tag editing is simply enabled (there is no read-only seat to gate, unlike the Viewer). The two tables are created idempotently on next launch like every other Lite config table, so an upgraded store gains them with no migration step and no risk. Stage 2b-i of [#2020]; the sidebar tree-grouping drill-down follows as its own PR (2b-i-b), and web parity is 2b-ii. - -- **Tags get colour, and a server's tags show as coloured pills on the Overview cards (Viewer)** ([#2008], stage 2a) - server tags could organise the sidebar tree but were colourless and invisible on the Overview. Each tag now carries an optional colour: a new tag is auto-assigned one from a fixed, theme-safe palette (rotated by tag id, so the choice is stable and reproducible - re-creating the same tags gives the same colours), and any tag can be recoloured, or set back to no colour, from a swatch picker in **Manage Tags** (a new **Colour…** button; the tag tree shows each tag's swatch). A server's tags then render as coloured pills on its card in the Overview, so DEV/PROD/TEST is visible at a glance rather than buried in the sidebar. The palette is a fixed set chosen to read under a constant light label in both the light and dark themes, so a pill looks the same whichever theme is active; a tag with no colour renders as a neutral pill. Editing tags stays on the admin seat exactly as before - the colour controls ride the same read-only gate ([#2011]) as the rest of the tag editor, so a read-only viewer seat sees them inert. Darling stores migrate to **schema v50** (one nullable `colour` column on `config.server_tags`, added additively with no backfill - existing tags stay neutral until touched) - automatic on the next service start. Stage 2a of [#2008]; the search box shipped in stage 1, and Lite/web tag parity (2b) follows as its own PR. - -- **Find servers fast: a live search box on the server list across all three surfaces** ([#2008], stage 1 of the proposal) - as a monitored fleet grows past a screenful there was no way to *find* a server; the Viewer sidebar, Lite's Overview, and the web fleet page each had at most a sort control. Each now carries a search box that filters the list live as you type - a case-insensitive substring of the server name, and in the Viewer (the surface where tags exist) of any tag assigned to it, so `prod` finds both `sql-prod-01` and every server tagged Production. The match rule is one shared, unit-tested helper (`ServerOverviewFilter`) that both desktop apps call and the web viewer mirrors in JavaScript, so the three surfaces filter identically. In the Viewer it lands on the `FleetView.Rebuild` seam reserved for exactly this - the single projection point - so grouping, favourites, and selection all follow with no consumer changes: tag groups whose whole subtree has no match drop out, a match inside a collapsed group is revealed (every group renders expanded while a search is active) with the persisted collapse state restored untouched when the box is cleared, and the sidebar count reads "N of total" while filtered. No schema change and no new store reads - filtering is an in-memory pass over the fleet already loaded. Stage 1 of [#2008]; tag colour + pills and Lite/web tag parity follow as separate PRs. - -- **The Viewer's connect-failure overlay gains "Run self-test" - a layered probe that names WHICH layer broke instead of one collapsed error** ([#1954], completing the field-feedback issue whose path-provenance and parse-summary halves shipped in [#1966]) - a failed store connection walks six layers separately: config parse, DNS, TCP reach, TLS, authentication, and the schema-version gate, each reporting pass/fail/skip with prose and elapsed time, and the verdict names the FIRST failed layer. That makes "firewall problem" diagnosable by elimination from the Viewer's own seat: DNS passed, TCP did not. TLS and authentication share one network exchange, so they are attributed by exception shape (a pinned, pure classification table: the PostgreSQL auth SQLSTATEs mean the handshake already succeeded; a certificate failure means auth was never reached). The probe is honest about what it cannot prove - under \`SSL Mode=Prefer\` a successful open does not demonstrate a TLS handshake (the mode falls back to plaintext), so that reports as skipped-with-caveat rather than a hollow pass; the least-privilege viewer role cannot read the owner-only version table, so the schema layer falls back to a privilege-free catalog probe (\`to_regclass\`, NOT \`information_schema\`, which hides unprivileged tables and misread a healthy store as not-a-Darling-store in live testing) and reports "store detected, version unreadable by this role" instead of a false failure. Results append to the failure overlay's details block (re-runs replace, not stack) and go to the log, flushed. The probe core is shared, never throws, and every failure mode was validated LIVE against a real store: happy path, wrong password (28P01), wrong port, unknown host, unknown database (3D000), not-a-store, least-privilege role, and garbage connection string all attribute to the right layer. A headless \`--test\` flag for scripted diagnostics is deferred to its own issue. - -- **You can see what automatic plan correction is doing, including the queries the engine is working on right now** ([#1952]) - when `FORCE_LAST_GOOD_PLAN` is on, SQL Server quietly detects regressed queries, forces the last known good plan, verifies the result, and backs the change out when it did not help. None of that was visible in the product, even though the engine is effectively handing you a list of which queries regressed and by how much. A new `plan_correction` collector captures both halves in both apps: per-database enablement from `sys.database_automatic_tuning_options` (a new **Automatic Tuning** grid under Configuration) and the engine's live recommendation set from `sys.dm_db_tuning_recommendations` (a new **Plan Corrections** grid beside the Query Store surfaces). The recommendation's details JSON is shredded into typed columns - query id, regressed and last-good plan ids, execution counts, average CPU per plan, estimated gain, and the engine's own state (`Active`, `Verifying`, `Success`, `Reverted`, `Expired`) - and the regressed query's TEXT is resolved through Query Store at collection time, so a stored row explains itself without a join. SQL Server 2017+ and Azure; automatic tuning is an Enterprise-edition feature, and on Standard the enablement row reports `NOT_SUPPORTED` rather than going quiet. - - **Collecting this is the only way to keep it.** The DMV is documented as holding recommendations *only until the instance restarts* - a restart erases the set, and a restart during a verification silently unforces the plan the engine had just forced. A cycle that captured a `Verifying` row is the sole surviving record that any of it happened, which is why this runs on a cadence rather than as a one-shot config snapshot. - - Two smaller things worth knowing. The enablement grid is at its most useful when desired and actual state DISAGREE: ask for `FORCE_LAST_GOOD_PLAN` in a database whose Query Store is off and you get desired `ON`, actual `OFF`, reason `QUERY_STORE_OFF` - so every database is enumerated rather than only the Query-Store-enabled ones, because filtering would drop exactly the rows worth looking at. And a recommendation whose plans have already aged out of Query Store still appears, with its text blank, instead of disappearing. -- **Both apps now collect and show the ADR persistent version store, so a database that grows for no visible reason finally has an explanation** ([#1951]) - Accelerated Database Recovery keeps its version store INSIDE the user database. When a long-running or aborted transaction pins version cleanup, the data files grow and every size surface in the product reported the growth without ever saying what it was. A new hourly collector reads `sys.dm_tran_persistent_version_store_stats` (SQL Server 2019+, and ALWAYS collected on Azure SQL Database, where Microsoft documents ADR as always enabled), one row per database, kept 90 days to sit on the same time axis as database sizes. A new **FinOps > Version Store (PVS)** tab in Lite and the Darling Viewer puts PVS size next to the database's own data-file size and the share it has taken, alongside the aborted-transaction count, the cleanup state, and the skipped-page counters that say WHY cleanup is not reclaiming. ADR enablement is collected as inventory too, so "ADR is on and quiet" is visible rather than inferred from a blank. - - **Sizes are OFF-ROW versions only, and every surface says so.** Microsoft documents `persistent_version_store_size_kb` as excluding versions stored in-row; ADR uses both, so this number structurally understates total version space and a workload whose versions fit in-row can read near zero while still paying ADR's cost. It is labelled "PVS Off-Row MB" rather than "PVS Size" for that reason. - - **What is deliberately NOT here, because it does not work.** Microsoft's published PVS diagnostic query resolves the transaction holding cleanup back, and the session running a long snapshot scan, with two `LEFT JOIN`s. Both were reproduced against a live ADR database on SQL Server 2025 with a genuinely open transaction and a genuinely open snapshot scan, and **both returned NULL**: `oldest_active_transaction_id` is an internal sequence number in a different ID space than `sys.dm_tran_database_transactions.transaction_id` (66881 against 5694862 in the same reading), and `min_transaction_timestamp` is a cleanup low-water mark rather than any live transaction's sequence number (62903 against 62919). Shipping those columns would have meant three permanently blank fields under headers promising "the transaction holding cleanup back" - worse than absent, because a blank reads as "nothing is holding it". What ships instead is the comparison Microsoft's own troubleshooting text calls for and which needs no join at all: the oldest aborted transaction against the oldest active one, surfaced as an **Aborted Lag** number rather than a yes/no verdict. Microsoft's rule is "much lower ... and the count is large", and neither "much" nor "large" has a documented threshold on what are dense internal sequence numbers - so the grid shows the gap, the aborted count, and the skipped-page counters, and leaves the judgement where Microsoft leaves it. - -- **The Darling Viewer says which darling.json it read, and what it parsed out of it** ([#1954]) - a viewer that would not connect gave a generic failure over a log that named neither the config file it chose nor the values it got from it, so "it read the wrong file" and "it read the right file and a value in it is wrong" produced identical evidence. Startup now logs the resolution: which of the four rules won (an explicit command-line path, `DARLING_CONFIG`, `darling.json` beside the viewer, or the service root one level up), the absolute path that rule produced, and whether that file exists - emitted BEFORE the load is attempted, so a missing or unparseable file still says where the viewer looked. Once it loads, a non-secret parse summary follows: host, port, username, database, SSL mode, search path, whether the connection string was read verbatim or DERIVED from `postgres.managed` (on a managed install nothing reads `postgres.connectionString` at all, which is worth knowing before you spend an afternoon editing it), and the certificate - the value as written, **the absolute path Npgsql will actually open**, and whether that file is there. - - The certificate line is the sharp one. The documented bring-your-own connection string carries a bare `Root Certificate=server.crt`, and a relative path there resolves against the process WORKING DIRECTORY - not the viewer's install directory and not darling.json's directory - so the same viewer launched from a shortcut and launched from a shell look for the certificate in different places, and verify-full fails in one of them with nothing said about where it looked. - - **The same block is rendered in the connection-failure window**, under a Copy details button, so a field operator answers both questions without going to find a log; it reaches the log too, flushed immediately rather than on the buffered writer's next tick, because a viewer stuck on that screen usually gets killed first. - - **The password cannot appear in it, structurally rather than carefully.** The summary is built from an allowlist of connection-string properties read off a parsed builder - no path through it copies the caller's string into its output, so a keyword an operator adds later cannot leak by being forgotten - and a string that will not parse reports only the exception TYPE, because Npgsql's parse errors can quote the fragment they choked on and that fragment can be the credential half. A test feeds a live password through and asserts it appears nowhere in the summary or in the composed block. - - The **layered connection self-test** the same issue asks for - DNS, TCP, TLS, auth and the schema gate, reporting which layer failed - is deliberately not in this change; [#1954] stays open for it. - -- **Darling hands you a working remote Viewer setup instead of a documentation exercise: `--export-viewer-config`** ([#1953]) - getting a remote Viewer connected was the hardest part of a real field setup, and every step of it was work the service could already do for you: copy JSON out of the docs, hand-merge `--print-viewer-connection`'s terminal output into it, save the certificate somewhere the connection string agrees with, and discover by trial that the VIEWER's `darling.json` wants `"managed": false` even though the SERVER runs `"managed": true`. Run on the service host, the new verb writes the viewer machine's whole folder - a complete `darling.json` with the resolved host, port, role credential, verify-full TLS and `"managed": false` already set; the store's `server.crt` beside it; and a `README.txt` documenting every field, including the valid `Root Certificate=` values (a bare name resolves against the folder holding `darling.json` - see [#1970] below - and an absolute path is used as written). The JSON carries the same documentation as comments IN the file, which the Viewer's parser skips, so the export is also the reference the docs were not. Copy the three files next to the Viewer executable and start the Viewer - nothing to edit. (Keeping them somewhere else and pointing `DARLING_CONFIG` at them works too, with nothing to edit either way: since [#1970] a bare `Root Certificate` name anchors to the folder holding `darling.json`, wherever that folder is.) The folder lands beside the service's `darling.json` by default, or in a directory you name. The `managed` confusion dissolves structurally: nobody writes that file by hand anymore. - - The exported `darling.json` **holds a live database password**, because that is what the Viewer authenticates with. The verb says so, naming the file, BEFORE it writes it, then ACLs the file to SYSTEM + Administrators + the running account + INTERACTIVE (dropping any inherited `BUILTIN\Users` read) and **confirms** the result - if the secret is still readable by ordinary users it says so loudly and exits non-zero, so a script cannot read "exported" as "protected". A missing credential or certificate is refused with the exact missing file and the one action that produces it, rather than exporting a folder that cannot connect. - - It also refuses, rather than destroys: exporting into the service's **own** config directory would have overwritten the service's `darling.json` with the viewer's - taking every monitored server, every encrypted password and the MCP/web tokens with it, none of which exist anywhere else - and the install directory is the obvious place an operator would put a handoff folder. It likewise refuses a destination holding a `darling.json` this verb did not write, a destination that is a junction or symlink, and any argument it does not recognize; re-exporting over its **own** previous output (the documented step after a credential or certificate rotation) still just works. - -- **`--print-viewer-connection` warns BEFORE it prints, not after** ([#1953]) - the field report watched a live database password scroll into their terminal and only THEN read the advice to redirect it somewhere safer, which is a warning arriving after the damage. Every line of STDERR guidance - including the "the certificate does not exist yet" note, which used to trail the payload - is now emitted ahead of the STDOUT payload, verified on a real console rather than assumed from stream semantics. - -- **The PagerDuty channel can route through a proxy** ([#1945]) - it was the only webhook channel that could not, and its endpoints are fixed public URLs, which makes a proxy MORE necessary in locked-down networks, not less. A pagerduty_proxy setting follows the sibling channels' exact shape in both apps (plain pref, not a secret), with a V43 store migration and a matching viewer probe sentinel on the Darling side. - -- **PagerDuty is a first-class alert channel in Lite and Darling** (contributed by @ianwalkeruk, [#1943]) - native Events API v2 integration alongside Teams, Slack, and the generic webhook: severity maps onto PagerDuty's four levels, repeated alerts for one ongoing incident correlate into a single PagerDuty alert via a dedup key derived from the same incident fingerprint the notification cooldown uses, and both US and EU data centers are supported. The routing key is stored as a bearer secret - Windows Credential Manager in Lite, the viewer-REVOKEd secret column tier in Darling's store (V42) - never in a plaintext settings file. Remediation T-SQL is never inlined in the page, matching every other webhook channel. - -- **Collection Health now says whether a collector that has come back empty all week had anything to collect** ([#1852]) - [#1837] shipped the qualifier that says HOW MUCH of the window an enumerated collector spent empty (`enumeration yielded 0 items ... (all 96 runs)`), and deliberately stopped there: it could not tell a server that legitimately has no user databases - which must stay quiet, and stays HEALTHY - from one that has databases and is enumerating none of them, which is the case worth a look. The three collectors that enumerate user databases (`query_store`, `index_object_stats`, `database_scoped_config`) now read that note as `... (all 96 runs, target has user databases)` when the store has seen user databases on the same server inside the same window. **No band changes and no new status** - the row still reads HEALTHY, because a filter that excluded everything or a feature nobody enabled are legitimate reasons to enumerate nothing, and a monitoring tool that reddens on them gets ignored. All four Collection Health surfaces inherit it from the one shared formatter (Lite's grid, the Darling Viewer's grid, both `get_collection_health` MCP tools, and the web dashboard's table behind them), and the MCP payload also carries the raw `target_has_user_databases` so a caller diagnosing an empty collector gets a boolean rather than having to read it out of a sentence. - - **Silence beats a false alarm, in four separate ways.** The qualifier appears only when the note is the empty-enumeration one (a probe-failure note already names its own cause), only when EVERY run in the window carried it (a sometimes-quiet collector is normal), only for a collector that actually enumerates databases, and only when there is inventory to go on. The inventory is one uncorrelated `EXISTS` against `database_size_stats` - the size collector rather than `database_config` because it runs on the scheduled loop where `database_config` runs on connect and can age out on a long-running install, because it is indexed on `(server_id, collection_time)` where `database_config` has no index at all, and because it reads `sys.master_files`, so it still sees databases the monitoring login cannot ENTER, which is exactly the fault being diagnosed. It screens on `database_id > 4`, since the size collector takes every ONLINE database and a bare row check would be true on every server alive. **Its window is the health read's own seven days**: an inventory that aged out is not evidence about the target today, so it yields no qualifier rather than a stale claim - as does an install whose size collector is disabled, or a server that has not collected yet. - - One byte-portable query shape on both stores as before, verified by running it against a real DuckDB and a real PostgreSQL 18.4 + TimescaleDB. The subquery is uncorrelated, so Postgres evaluates it ONCE per query as an InitPlan rather than per row, on the index `database_size_stats` already carries. Measured over a 200-server / 4-million-row store with 3.4 million size rows: on a target that HAS user databases the probe costs **0.068 ms and 4 buffers out of the query's 20,043**, and the read as a whole is indistinguishable from before (medians 98 ms and 83 ms over ten warm runs each, run-to-run spread wider than the difference). The worst case is the legitimately-empty target, where the probe scans that server's whole window and rejects every row: **7-12 ms warm** against a read that costs ~100 ms without it. Both are inside the per-server budget; the fleet read pays neither. **The Darling Viewer's FLEET rollup carries no inventory at all**, holding [#1863]'s line: it groups `server_id` INTO its result so there is no single server to probe, its only caller counts bands, and with the note text already blank there a real value could not change one rendered character - a truthful fleet version would be a cross-collector join across every enabled server on a query the status bar re-runs on every refresh. Availability-Group collectors get no equivalent mapping, for cause: none of them enumerates today, so the mapping could never fire, and the Darling store has no `v_ag_replica_states` view to join - tracked as [#1867]. - -- **A blocking alert that fires on total blocked WAIT TIME, not just how many sessions are blocked** ([#1839]) - the existing gate counts blocked-process events in a rolling window, which cannot tell one session blocked for an hour from one blocked for a second: the count is 1 either way. `blockingWaitSecondsThreshold` (Lite: Settings > Alerts, "Total blocked wait"; Darling: `darling.json` + the Viewer's Settings window) sums `wait_time_ms` across the rows of the LATEST blocking snapshot and fires when that total reaches the threshold. **Defaults to 0 = off in both SKUs**, so nothing starts alerting because you took an update. It is a SNAPSHOT sum rather than a windowed one on purpose - the alert answers "how much aggregate blocked wait is happening right now", and summing a window would accumulate blocking that has already ended and could never clear while the window still held it. Unlike the count gate's rolling-window edge trigger it is LEVEL-triggered, the shape the High CPU check already uses: it re-fires every cooldown while the wait stays above the threshold and resolves with "Blocking Wait Cleared" when it drops below, so standing blocking keeps announcing itself instead of going quiet after one alert. It reports under its own "Blocking Wait Time" metric rather than reusing "Blocking Detected", so mute rules, cooldowns, and history rows for the two gates never tangle - but it carries the same blocked-process incident content, built from the rows the sweep already fetched for the count gate rather than a second query. A STALE snapshot neither fires it nor holds it active ([#1812]'s rule, at the blocking collector's cadence): a stopped collector leaves a "latest" snapshot that would otherwise read as NOW, and a level-triggered alert on frozen rows would re-fire forever - staleness resolves rather than latching, the opposite of the running-jobs treatment, because for a level condition the worse failure is staying latched on data of unknown age. The value travels as a real number ([#1830]), never parsed back out of the display text. Both gates stay under the one `blockingEnabled` toggle. Darling stores migrate to **schema v40** (one additive `NOT NULL DEFAULT 0` column) - automatic, no operator action. Two adjacent corrections ride along: DuckDB widens `SUM(BIGINT)` to HUGEINT, which the .NET driver hands back as a `BigInteger` that `Convert.ToInt64` cannot convert, so the read is cast in SQL on both stores (caught by the new round-trip test, and it would have thrown at alert-check time); and Lite's MCP `get_alert_settings` had been reporting the blocking COUNT under the key `threshold_seconds` - a name copied from the deprecated Dashboard, whose blocking threshold genuinely is seconds - which is now `count_threshold` beside the new `wait_threshold_seconds` (the spelling is Darling's existing `count_threshold`, per the review of [#1840]). -- **A database-state alert that fires when a monitored database leaves its expected state** ([#1986]) - `database_config` captures `state_desc` only at server-load, so a database going OFFLINE / SUSPECT / RECOVERY_PENDING / RESTORING AFTER onboarding was invisible. A new per-minute `database_states` collector captures `sys.databases.state_desc` as a time series, and a new shared-engine alert fires when a database's current state DEVIATES from its expected state in the two most recent collections (a two-sample rule, so a restart's brief RECOVERING / RECOVERY_PENDING transient does not false-fire). The expected state is captured automatically on first observation (a baseline), so a log-shipping secondary - modelled at its effective `STANDBY` state rather than the `RESTORING` it flickers through on every log restore - baselines at `STANDBY` and stays quiet, and it is user-editable per database - change it to accept a new normal, or set `(ignore)` to opt a database out. **A critical first observation does NOT baseline itself:** a database first seen SUSPECT / RECOVERY_PENDING / EMERGENCY alerts immediately and stays pending rather than learning the bad state as expected, which would otherwise go quiet through the outage and then fire on RECOVERY. Alerts are graded CRITICAL for those three states and WARNING otherwise, are per-database (each fires, cools down, and recovers independently), and flow through the existing mute rules and delivery channels plus the web viewer's alert history. A **Configure...** button in each desktop app's Settings > Alerts opens a per-database editor (current state, expected state, re-baseline, ignore), and the MCP `get_alert_settings` / `update_alert_settings` expose the master toggle. Enabled by default. Darling stores migrate to **schema v49** (the `database_states` collector table, the `config.database_state_expected` control table, and the master toggle) - automatic, no operator action; Lite creates the equivalent DuckDB tables on next start. - -### Changed - -- **A bounded Query Store cycle now DEFERS its backlog instead of dropping it - collection ships oldest-first from the watermark and resumes exactly where it stopped** ([#1960] phase 1) - the [#1556] per-database bounds (the 50k-row `TOP` backstop and the client text-byte budget) kept one busy database from ballooning the process, but they were hole-makers: rows were read newest-first, the watermark derives from the newest row STORED, so everything a bounded cycle did not reach fell permanently behind the strict `> @cutoff_time` and was logged as "oldest rows dropped." The read now ships FORWARD from the watermark (`ORDER BY last_execution_time ASC`), which turns the same derived watermark into an exact resume point: a cycle cut by either bound leaves the boundary at the newest row it actually shipped, and the next cycle continues from there - a bound now costs catch-up latency, never data. Two tie-group guards make the boundary exact rather than approximate, because the raw rows are COPY-appended (no upsert) and the cutoff comparison is strict: the SQL `TOP` becomes `WITH TIES` so the server can never split a group of rows sharing the boundary `last_execution_time`, and the client budget finishes the same tie group before stopping (tied rows are adjacent under the ASC order, so the overshoot is one group). With cuts now hole-free, the text budget drops 256 MB → 64 MB: the July field catalog (~1.33 GB fresh) converges in ~21 bounded cycles instead of one unbounded pull, and the worst-case transient under the 4-wide sweep shrinks from ~1 GB to ~256 MB. The bounded-cycle WARNING in both apps now reports what a cycle actually did - megabytes shipped and the boundary reached - so a long catch-up reads as steady progress rather than a mystery, the same observability contract the 24h clamp already had. The plan-XML dedupe keeps its newest-interval-carries-the-plan rule within each cycle's window, so during a multi-cycle catch-up a plan's XML can trail its interval rows by a cycle and then lands; steady-state cycles are unaffected. The 60-minute first-contact window and the 24h catch-up clamp are unchanged - backfilling the ~30 days both leave behind is phase 2's newest-first worker, tracked separately. - -- **Analysis findings re-notify only when they WORSEN or RESOLVE-and-return — a standing story no longer re-fires on every cooldown expiry** ([#2054]) - the first weekday of fleet-scale severity data made the problem concrete: ~46 of a day's 50 analysis alerts were the IDENTICAL story chain sitting exactly at the 1.50 notify threshold, once per server across a 52-server fleet, and the one server that reached 1.80 looked identical in the stream. That is one ambient workload truth, not 45 incidents - and re-firing it at every cooldown expiry is what buried the outlier. Each notification bucket now remembers the severity it last notified at: re-notification requires the story to worsen by 0.25 (chosen from the live evidence - it separates the 1.50 ambient from the 1.80 outlier exactly), still spaced by the cooldown so a climbing story cannot e-mail every analysis cycle (a band-jumping escalation already gets a fresh per-hash CRITICAL bucket and fires immediately, unchanged). A story that stops firing for 2x the cooldown is forgotten and re-alerts as fresh on recurrence - the same fresh-or-worsening standing-condition treatment the PVS-pressure alert shipped with ([#1984]), pointed at the analysis stream. Buckets are pruned by last-SEEN rather than last-notified, so a persisting story cannot age back into freshness while it keeps firing. One honest limit: the alert log persists no severity, so a restart seeds surviving buckets at the notify threshold - the first post-restart re-notify needs threshold + 0.25, and a chain that was already above threshold may notify once more than strictly necessary. The issue's other two options (threshold raise, fleet-dedup delivery) stay open as complements. - -- **Top-queries reads now say when a hash group merged different statements, instead of labeling a blend with one arbitrary text** ([#2012], stage 1 - found during live production triage) - \`query_hash\` is a SHAPE hash: \`INSERT INTO #t EXEC \` statements naming DIFFERENT callee procedures share one (reproduced on SQL Server 2022 - two wrappers calling two different procs, one hash), and ad-hoc literal variants collapse too. Both apps' \`get_top_queries_by_cpu\` group by \`query_hash\`, so colliding callers' stats were SUMMED into one line labeled with whichever caller's text the latest-text lookup happened to grab - during real triage that mislabeled per-tenant attribution twice in one day, and only hand-checking \`OBJECT_DEFINITION\` caught it. Each group now carries \`distinct_texts\` (counted over the [#1767] content digest already on every Darling row, and the inline text in Lite - effectively free), and when it exceeds one, a \`text_note\` says the shown text is one representative and to attribute per-caller work via \`sql_handle\`/\`OBJECT_DEFINITION\` before naming a caller. Zero means only rows predating the text dimension (unknown, ages out with raw retention). Per-row storage was never wrong - the collector captures each row's own statement text - and stage 2 ([#2012]) captures the host-object identity at collection so proc-hosted callees can be grouped apart outright. - -- **Top-queries reads now split same-hash statements by their hosting procedure, so INSERT...EXEC callers stop blending outright** ([#2012] stage 2, completing the issue; Darling V51 / Lite v52 store migrations) - the collector resolves each statement's host object on the monitored server (`sys.dm_exec_sql_text.objectid` to `schema.object`, NULL for ad-hoc/prepared text) and stores it on every row as `host_object_name`, appended last so every existing ordinal stays put; nullable, no backfill - pre-upgrade rows read as unknown host and age out with raw retention. Every hash-grouped top-queries surface - both apps' `get_top_queries_by_cpu` MCP tools (which now return `host_object`), the REST route riding the same reader, and both apps' Top Queries grids - groups by (database, `query_hash`, host object) and constrains the representative-text lookup to the group's OWN rows, which was the actual mis-attribution mechanism: a hash shared across callers could label one caller's numbers with another caller's text. Ad-hoc behavior is untouched (NULL hosts group as one, so literal variants still collapse), and `distinct_texts`/`text_note` remain as stage 1 shipped them, now flagging only ad-hoc blends and pre-upgrade history. The grids' Module column prefers the collection-time host object over the [#1568] `sql_handle` stitch - it is resolved at the source, so it holds even when the module has aged out of the procedure-stats cache the stitch depends on. Darling's V51 rebuilds the [#1767] resolving `v_query_stats` from the generator rather than a passthrough (the tripwire test for exactly that silent regression did its job in review), V38's generated body now pre-adds every query_stats payload column so a store upgrading from ANY older version survives the ladder in order, and the Viewer's schema gate moves to V51 since its reads name the new column. The comparison and FinOps 80/20 surfaces deliberately keep hash-level grouping: they are aggregate trending reads, and splitting their keys would false-flag NEW/GONE on every proc-hosted query across the upgrade boundary. - -- **The orphaned CPU/IO baseline aggregates are retired — every store drops them on the next service start** ([#2007], the cleanup [#1995] deferred) - when the CPU and IO anomaly arms moved to reading the raw hypertables (medians cannot be computed from the aggregates' sufficient statistics), \`cpu_utilization_baseline\` and \`file_io_baseline\` lost their only reader — but they stayed registered, kept materializing on their hourly schedules, and kept holding storage on every store. Fresh stores no longer create them, and a startup sweep (the same shape as the reshape drops) removes BOTH implementations an upgraded store can carry: the continuous aggregate on TimescaleDB stores (its refresh/retention policies drop with it) and the plain fallback view on plain-PostgreSQL stores — discriminated the same way the fallback-drop guard does, because a continuous aggregate is also a \`relkind='v'\` view and the two need different DROP verbs. The sweep runs in the worker's ungated fallback block so every store shape is cleaned, no-ops on fresh stores, is idempotent, and is failure-isolated per relation. No schema migration is involved. Validated live in all three states an upgraded store presents: the CAGG shape (policies confirmed gone with it), the plain-view shape, and already-gone. The remaining seven baseline aggregates are untouched. - -- **Both MCP hosts move to ModelContextProtocol SDK 2.0 (the MCP 2026-07-28 specification)** ([#1990]) - taken deliberately rather than riding a routine bump, because the 1.x-to-2.x major changes HTTP transport behavior and a grouped auto-bump had already demonstrated the failure mode (half-applied across projects, poisoning every lockfile - [#1822]). The version moves in every referencing project in one commit (shared Common, Lite, the Darling service, the deprecated Dashboard) with all lockfiles regenerated together under the CI-pinned SDK. **No code changes were needed**: 2.0's headline behavior switch - stateless-by-default HTTP - is a no-op here because both hosts have run \`Stateless = true\` explicitly since [#1074] (clients that don't echo \`Mcp-Session-Id\` otherwise connect but list zero tools), and the deprecated-API audit (Roots/Sampling/Logging) found no usage in either host or the shared tool plumbing. The Host-header/bearer/CIDR guards install ahead of \`MapMcp\` as pipeline middleware, so 2.0's discovery-first endpoint surface routes through the same checks - verified live against the SDK 2.0 host: initialize/tools-list/tools-call succeed statelessly and a non-loopback Host is still rejected before any handler. The SDK 2.0 surface was additionally validated end-to-end on Performance Studio's identical hosting stack before this migration. With the major taken, the Dependabot ignore rule that deferred \`ModelContextProtocol*\` majors to this issue is removed, so future SDK patches and minors resume flowing through the weekly group. - -- **`get_analysis_findings` now returns one entry per diagnostic chain instead of one per engine cycle - a 24h read shrinks ~28x** ([#2000], found dogfooding the MCP surface on the 52-replica production monitor) - the analysis engine re-persists the same stories every cycle, so a multi-hour read returned each chain dozens of times, every occurrence re-carrying the full advice prose and copy-paste remediation command: measured fleet-wide, 21,623 rows in 24 hours collapse to 774 distinct (story, incident) chains - 27.9x mean duplication, 39x on the worst server, ~257KB for a single server's 10-hour read. Both MCP twins (and Darling's web API bridge) now group by `story_path_hash` + `incident_id` and return the chain's LATEST occurrence - value-bearing advice is frozen at analysis time, so latest is current truth - plus the stats that replace the collapsed timeline: `occurrences`, `first_seen`, `last_seen`, and `peak_severity`, which matter because severity genuinely moves within a chain (275 of the 774 measured groups spread more than 0.1). `finding_count` counts deduplicated chains and a new `total_occurrences` reports the raw rows, so the compression is visible rather than silent. The read also stops being quietly WRONG at depth: the store read's row cap of 100 covered only the ~5 most recent cycles of a 24h request and silently dropped the rest, so the findings tools now pass a window-covering cap sized for the deepest legal read - occurrence stats computed over a truncated read would lie about `first_seen` - and a read that ever FILLS that cap says so in a `truncation_note` instead of under-reporting silently. The store keeps every occurrence; nothing changes for viewer timelines or what is persisted. - -- **Memory-pressure anomalies no longer fire on healthy servers with young baselines** ([#1996], found dogfooding the robust-baselines nightly on the 52-replica production monitor) - every server on the fleet reported the same self-contradicting finding, "Memory pressure spiked to 100% - 0σ above its 100% baseline for this time of week", 1,279 times across two eras. The mechanism, proven against the store rather than guessed: memory's absolute-fallback bar sat at 95%, BELOW the metric's healthy operating value - a warmed-up SQL Server holds total ≈ target = 100% by design, and the fleet's median is exactly 100.0 on every server - so whenever a baseline bucket was too young to trust (403 of 406 firing buckets had exactly 2 distinct days, one under the Full tier's floor of 3, with ~82 samples each), the untrustworthy-baseline path fired the absolute bar on completely normal behavior and displayed the honest-but-absurd 0σ. The bar moves to 101: total EXCEEDING target is genuine over-target pressure and still fires; total sitting at target stays silent however thin the baseline. Pre-existing (identical findings before the robust upgrade), young-store-shaped (it would have faded as buckets crossed the day floor, then returned with every new server), and exactly the class of thing the dogfood loop exists to catch - the finding volume was the single largest noise source on the MCP surface, 31 of the busiest tenant's latest 100 findings. - -- **CPU and I/O latency reach robust-baseline parity in Darling** ([#1743] follow-up) - phase 1 shipped with a stated asymmetry: Darling's CPU and I/O baselines read sum/sumsq rollups that can reconstruct mean and stddev but structurally cannot produce a median, so those two families degraded to the classical gate while Lite (which reads raw grain locally) got robust statistics on all nine non-event metrics. The obvious fix was wrong twice before it was right, and both wrongs were caught by measurement rather than shipped: a 4-day raw window would satisfy sample counts but never the distinct-day trust floors (measured on the production store: zero of 5,044 full buckets trustworthy), which would have REGRESSED those families to the absolute-fallback bar - and the "4 days of raw supply" premise itself turned out not to apply here, because cpu_utilization and file_io_stats carry their own 30-DAY service-side retention (1-minute cadence collectors; verified against the production store, where both tables hold the store's full life, compressed after a day, with no TimescaleDB retention policy). So both arms now read their raw hypertables at Lite's exact grain through the same robust scaffold - the mean/stddev they produce are the SAME per-sample statistics the rollups reconstructed, plus the median/MAD the rollups could not - validated on the production fleet's busiest tenant at 227 ms for the full window, with the tier ladder behaving correctly at every store age (42 full buckets already trustworthy at 16 days, every hour-only sentinel trustworthy). The supply pins moved with the reasons: the nine aggregate-fed families keep their pin, the two raw reads are pinned to their exact tables, and a NEW pin makes the load-bearing invariant a red build instead of a quiet degradation - if either collector's retention ever drops below the 30-day baseline window, the build fails naming #1757. The retired rollup aggregates stay registered for upgrade compatibility; removing them is separate cleanup. - -- **The anomaly engine judges deviations against median/MAD instead of mean/stddev, and its confidence is honest** ([#1743] phase 1) - the classical baseline is self-poisoning: every burst a server ever ran through inflates its mean and stddev, and the inflation MASKS the next real deviation. Measured twice over before a line changed. On a HammerDB store, weeks of load tests left the mean 17x the median and the stddev at 11,994 against a MAD of 11 - a genuine 26x workload surge registered 0.0 classical sigmas (the engine ran and could not see it) while the modified z-score read 99.2. And on 52 production replicas, the realistic form: the fleet's busiest tenant's own history inflated its stddev enough that a real sustained 2-3x Friday-evening surge read 1.4-2.0 classical sigmas - invisible at any sane threshold - while the modified-z read 3.5-4.7 and fired. At the SAME 3.5 cutoff over 24 hours of fleet samples, the robust statistic traded +284 genuine catches for 7 misses. Both apps' baseline providers now compute median and MAD alongside mean and stddev, EXACTLY at every tier - medians cannot be pooled from per-bucket medians, so the hour-only and flat tiers come from the SQL itself (GROUPING SETS; DuckDB's native `median()`/`mad()` in Lite, `percentile_cont` twice in Darling's Postgres, validated to-the-digit against the production store's independently-computed numbers) - and the shared gate judges the modified z-score at 3.5, or 5.0 for the heavy-tailed families (waits, query duration), where the fleet sweep showed 3.5 runs hot. The wait-profile detector's ratio trigger is REPLACED by the modified-z: measured at every cutoff swept, the ratio caught nothing the modified-z missed (strict containment) while missing the masked-surge class entirely - and the scorer grades those facts off the same statistic, because a catch the ratio floor zeroes at scoring is not a catch. Everything defensive is unchanged and load-bearing: the magnitude floors and absolute-fallback bars apply exactly as before (fleet-measured, MAD collapsed ONLY on idle-box CPU, precisely where the bounded-metric floor clamps), the trust gate and fallback complementarity are untouched, and a bucket without robust statistics - Darling's CPU and I/O latency read sum/sumsq rollups that cannot produce a median (their raw-window variants are follow-up work; Lite reads raw grain locally and gets robust statistics on all nine non-event metrics) - degrades to the classical gate rather than judging against zeroed fields. Fact confidence is now derived from baseline quality (tier and sample density; an untrustworthy baseline scores zero) instead of the hardcoded 1.0 shipped since [#1606], and every anomaly fact carries its baseline median and MAD so a reader can see the frame the verdict was made in. The calibration datasets ride in the regression suite verbatim - the HammerDB blindness case, the production surge, the floor-composition cases - so the thresholds stay measurements, not folklore. EWMA and changepoint detection remain phased behind this, per the issue. - -- **The MCP docs now lead with the boundary instead of the warning** - the networked-MCP section of the Darling guide opened with everything a token-holder could do, under a "blast radius" heading, and read like an AI client had the run of your SQL Servers. That was never what the code does: no MCP tool runs SQL an AI client wrote against a monitored server, the only live-server contact (the analysis plan fetch and the onboarding connection probe) runs the product's own fixed read-only queries under the same least-privilege monitoring login the collectors use, and every write-capable tool changes the monitor's own configuration under the carved-down `mcp` store role. The facts and every piece of wire/TLS guidance are unchanged - the sections now state what is structurally impossible first, then what the token actually gates, and the root README's MCP section carries the same boundary up front. -- **Query text and query plan columns sit at the FRONT of every query grid now, right of the time column, in both apps and the web UI** ([#1949]) - on a query screen the text and the plan are the thing you opened the screen to see, and on every one of these grids they were the last columns in the row, behind a horizontal scroll past a dozen numeric columns. The worst case was Query Store, where the query text was column **55 of 55**. An audit walked all 134 grids in Lite and the Darling Viewer, column by column: 36 carry query text and/or a plan button, **not one of them was already correct**, and 34 moved. The rule is uniform - time orients the row, then the payload, then the numbers - so Query Text and Query Plan now follow `Collected` on Active Queries, `Creation Time` on Top Queries, `First Execution` on Query Store, and the `Plan` button follows the time block in all three history windows. The blocking and deadlock grids move their statement text the same way, and the XML artifact button moves with it, because on those two screens the report XML is the drill-in that a plan button is everywhere else. Grids with no time column anchor on the identity column they are ranked by instead (`Database`, or `Score` + `Database` on FinOps High Impact). The Darling service's own web server page gets the same move: `query_text` was index 8 of 9 in the active-queries table and now follows `collection_time` directly. - - Column ORDER only. No grid's default sort, binding, filter button, tooltip, or plan-button wiring changed, and the CSV/clipboard export follows grid order automatically, so it picks the new order up for free. The two `Definition` columns on the FinOps index-analysis grids are deliberately untouched - that is index DDL, not collected query text. - - **A structural test now pins it in both apps**, which is the actual reason the columns drifted to the back for years: the eight existing `Pins*ColumnOrder` tests all assert the collector's WRITE payload into DuckDB or PostgreSQL, and **nothing anywhere asserted a grid's display order**. The new pins parse the XAML for every grid on the move list and assert the payload columns follow that grid's recorded anchor immediately, so pushing one back fails by name and prints the whole column list. A companion check asserts the 16 query grids that exist in both apps carry the *same* column sequence, so the two front ends cannot fork outside the pinned window either - the drift this issue was cleaning up in the first place. - -- **Setting up a remote Darling Viewer is its own section in the docs now, and it leads with the verb that does it for you** ([#1955]) - the viewer-only path was a sub-sub-section of the LAN-endpoints material, so the person whose entire goal was pointing a viewer at a service someone else runs had to read service-side setup to find their three steps, and what they found first was the manual path: paste a connection string, save a PEM, hope. **Connect a Remote Viewer** is now top-level and table-of-contents visible, written for a machine with nothing installed on it, and it leads with `--export-viewer-config` - run the verb, copy the folder, start the Viewer. Manual configuration is demoted to a fallback subsection for the case where you want the connection string itself. The LAN section keeps the service-side store endpoint and points at the new section for the viewer side. - - The `Root Certificate=` field now has a reference form, which is what [#1953] asked for at the doc level: what a bare name, a relative subpath, and an absolute path each resolve against (the folder holding the `darling.json` the viewer read, since [#1970]), where the service writes `server.crt` and what rotates it, and what a bring-your-own-PostgreSQL setup points the field at instead - its own CA or self-signed server certificate, the same file `psql` takes as `sslrootcert`. - -- **The SQL Server grant block is documented in ONE place, and it now includes `ALTER SETTINGS`** ([#1955]) - the Darling operator guide and the root README carried near-identical copies of the monitoring-login grants, which is how they went stale: **both** were missing `ALTER SETTINGS`, while Darling's own table listed it as the grant behind the `blocked process threshold (s)` bootstrap. `DarlingXeSessions` really does run `sp_configure` + `RECONFIGURE`, so a login built from either block could not do it, and blocked process reports stayed empty for a reason the docs described but never granted. The root README's block is now the authoritative copy with `ALTER SETTINGS` added; the Darling guide points at it and keeps only what is Darling-specific - the per-grant "what breaks without it" table, and the integrated-auth note that the grants belong to the account the SERVICE runs as, not the one that ran `--test-connection`. - - On the PostgreSQL side, the least-privilege role table understated what `provision-roles.sql` builds. `viewer` does not get plain "SELECT on both schemas": it reads all of `collect`, but on `config` the secret columns of `config_monitored_servers` / `config_command` / `config_notification` are carved out fail-closed, and the role runs under `statement_timeout = 15s`. Both are now in the table, along with the fact that `admin` *can* read those columns because the Settings window needs them. Verified against a live store rather than by eyeball: the shipped script run as documented against a freshly migrated PostgreSQL 18.4 store, then probed as each role - `viewer` reads every collector view and is denied every secret column (seven at time of probing, including `pagerduty_routing_key` - the carve is fail-closed, so a future secret column joins the denial without a script change), every collect write, every config write except its one `custom_views` grant, and DDL; `admin` reads the secret columns, writes the config tables, and is still denied collect writes and DDL. - -- **Deadlocks collection defaults to the 5-minute tier instead of every minute, in both apps** ([#1963]) - field decomposition on a 52-replica fleet showed the read costs ~273 ms of FIXED `dm_xe_session_targets` serialization no matter how empty the ring buffer is (measured at 284 BYTES of content), so the query has nothing left to optimize and cadence is the only lever on its overhead. The ring buffer retains events between polls and the watermark catches up, so a slower cadence keeps the same deadlock data - the trade is detection latency, and a deadlock is forensic by the time anyone reads it: the victim rolled back at the moment it happened, minutes before any cadence would have shown it. The new default sits beside the other event-buffer readers (`system_health_events`, `default_trace_events`) rather than inventing a one-off tier; the schedule-editor presets map it the way they map those siblings (Aggressive 2 / Balanced 5 / Low-Impact 15), and a test now pins the ruled value so a quiet edit cannot regress the decision. **Who picks it up when:** Darling stores with no per-collector override row get the new cadence automatically on upgrade; Lite installs keep their stored (user-editable) schedule, so an existing install moves when you apply a preset, edit the collector's schedule, or reset to defaults. The one loss scenario is a deadlock storm big enough to cycle the ring buffer between polls - the buffer's own capacity bounds that exposure at any cadence, and a storm that big is visible everywhere else in the product. -- **Alt mnemonics for the three checkboxes that live on tab content: `Auto-refresh` in both apps, `All Databases` in Lite's FinOps tab** ([#1878]) - [#1847]/[#1860] gave dialogs their access keys and deliberately stopped at the main window, because a `UserControl` hosted in a `TabItem` gets no access-key scope of its own: these three share the main window's scope with the shell around them, where both apps carried exactly **zero** keys. So `Alt+A` toggles Auto-refresh on the selected server tab, and `Alt+D` the FinOps index-analysis scope. The letters differ deliberately even though those two tabs can never be on screen together - relying on that would make the choice correct only until someone moves a control. - - **The interesting part is that duplicate keys across per-server tabs are safe, and it was measured rather than assumed.** Lite opens one `ServerTab` per connected server and holds them all, so ten open servers means ten `_Auto-refresh` checkboxes as objects - and a scope holding duplicate keys makes WPF CYCLE FOCUS instead of activating, which would be worse than no mnemonic at all. A `TabControl` realizes only the SELECTED tab's content, so exactly one is ever in the visual tree: verified by walking the tree with two tabs carrying the same key (one found, either way round), and confirmed once against a real shown window where the unselected tab's checkbox reported `IsLoaded=false` while the selected one reported true - that unload is what unregisters the key. The shipped guard is the structural form, which needs no desktop session, and it fails if a future hosting change ever realizes both. A collision audit over the whole main-window scope ships with it, honoring `__` escapes and excluding `ContextMenu`/`ToolTip`/`Popup` contents (their own scopes - that exclusion is load-bearing rather than defensive, since Lite's FinOps tab alone carries 20 context-menu keys and zero window-scope ones). **The shell's own key budget is untouched**: a future menu bar or sidebar pass still has every letter except A and D. -- **Azure SQL Managed Instance now collects Query Store plan type and replica role, instead of storing NULL placeholders on every instance** ([#1886]) - both columns are gated on SQL Server 2022+, resolved from the collector's live `PRODUCTVERSION` probe. MI reports major **12** (EngineEdition 8), so that gate could never fire there: every MI row stored `NULL` for `plan_type` and the `nvarchar(1)` NULL placeholder for `replica_role`, on every instance, regardless of what its engine actually supported. [#1848] and [#1872] flipped Azure SQL Database on for the same reason and **deliberately left MI alone** - correctly, because the justification does not transfer. Azure SQL Database is evergreen, so "Azure means newest" is a sound claim about the fleet; **MI is not**. Its feature set follows a per-instance update policy, so an instance on an older policy genuinely might not have the catalog, and getting that wrong is not a missing column - a column that does not bind fails the whole SELECT for that database, and MI takes the on-prem enumeration path where that means lost collection. - - So MI needed a stricter bar than Azure did, and [#1886] set it explicitly: the catalog must be present on the **oldest update policy still in support**, not merely on whatever instance was to hand. Measured 2026-07-31 on a GPv2 Gen5 4-vCore Managed Instance (westus3, provisioned for the run and torn down after) reporting `SERVERPROPERTY('ProductUpdateType') = 'CU'` - the conservative SQL Server 2022 policy rather than Always-up-to-date, which is exactly that oldest-supported case. `OBJECT_ID('sys.query_store_replicas')` = -660, `COL_LENGTH('sys.query_store_runtime_stats', 'replica_group_id')` = 8, `COL_LENGTH('sys.query_store_plan', 'plan_type_desc')` = 120 - and the two replica values re-answered from a **user-database context through `[db].sys.sp_executesql`**, the collector's actual MI mechanism, since binding in `master` would not have settled a gate that fires on the per-database path. Both gates flipped on that one session rather than provisioning MI twice. - - **One difference worth knowing before it alarms somebody:** a standalone MI's `sys.query_store_replicas` is an **empty enumeration**, where Azure SQL Database's is a static 4-row roles table even with no replicas present. That is harmless and the `LEFT JOIN` is why - the same shape that keeps a SQL Server 2022 standalone (whose view is also empty) collecting normally. `replica_role` simply reads NULL on such an instance, which is the honest state rather than an invented one. Nothing changes for box SQL Server below 2022, which still gets both NULL placeholders and never references the replicas catalog. The gate stays edition-based rather than probing each instance for the capability directly; that more general design is filed as [#1934]. - -- **Bring-your-own PostgreSQL now documents how to size TimescaleDB's background workers, and that the pool is shared by the whole cluster** ([#1899]) - managed mode writes `timescaledb.max_background_workers` and `max_worker_processes` on every start, derived from the live hypertable count; a bring-your-own store gets neither, and PostgreSQL's stock `max_worker_processes = 8` cannot launch the compression, retention and continuous-aggregate policies at all. Nothing said so, so an unmanaged store could sit quietly uncompressed with the reason visible only in the postmaster log. `Darling/README.md` gains a **Background workers** section carrying the formula (`timescaledb.max_background_workers = hypertables + 2`, `max_worker_processes = 3 + that + 8` - 41 and 52 today), the note that both are RESTART-only, and the fact that the `+ 2` is TimescaleDB's own two built-in jobs rather than slack, so a migrated store holds exactly one job per worker. - - **The part that is not in the formula: that pool is CLUSTER-wide while the derivation is PER-STORE.** Managed mode puts one store on one cluster so the two coincide, but N Darling stores (or any other TimescaleDB database) sharing a cluster need both numbers multiplied by N - each database with the extension loaded permanently holds a scheduler slot, so the sharing starts before any policy fires. The symptoms are documented from measurement rather than theory: `out of background workers` (TimescaleDB's pool) or `failed to start a background worker` (PostgreSQL's) in the postmaster log, with the job skipped and retried on its next schedule. **Occasional ones are benign and are called that**, so nobody chases a warning that costs nothing; persistent ones mean compression is falling behind a 1-day policy on a store that compresses ~16.7x on perfmon and ~6.4x on `query_stats`, which is the number that makes the consequence concrete. `timescaledb_information.job_stats` is given as the check that settles which case you are in. Pointers land where an operator actually is when it matters - the Prerequisites bullet, the `postgres.connectionString` row, a Troubleshooting entry keyed on the log line, and the bring-your-own block of `darling.sample.json` itself. - -- **The Settings windows' section-local buttons have Alt mnemonics, and the three identically-labelled webhook test buttons now say which channel they test** ([#1856]) - [#1847] gave both Settings windows mnemonics on the two footer buttons only and left the 13 section-local ones bare, because three of them read the identical "Send Test Notification" (Teams, Slack, generic webhook) and are visible together on one scrolling page, so distinct keys needed label edits rather than underscores. Those three are now **"Send Test to Teams" / "Send Test to Slack" / "Send Test to Webhook"**, which keeps the "Send Test..." prefix "Send Test Email" already set and is shorter than the label it replaces, so nothing re-wraps. Every section button in both windows then took a key: **15 access keys per Settings window, zero duplicates**, with Lite and the Viewer identical on every button the two share. **Five buttons have their `Content` reassigned from code-behind**, which is why a XAML-only underscore would have worked exactly once per launch - every restored state string now carries the underscore too. Pause/Resume is the one that needed a decision rather than a mnemonic: its label swaps between "Pause Collection" and "Resume Collection", and a key that moved with the state (Alt+P becoming Alt+R) is a worse affordance than none, so the key sits on **"Co_llection", the word both states share**, and Alt+L means the same thing in both. The transient "Sending..." / "Pausing..." strings deliberately carry no key, because the button is disabled while they show and an access key cannot activate a disabled button. The Viewer's two identical "Auto" buttons (MCP port, web port) keep their label and take **different keys, Alt+A and Alt+O**: each sits beside its own Port box under its own section heading, so unlike the test buttons they are not ambiguous on screen, and padding them out would be noise. The deprecated Dashboard's Settings window is deliberately untouched - it takes bug fixes, not enhancements. - -- **Alt mnemonics reach the context menus and the dialog checkboxes and radio buttons [#1847] scoped out** ([#1860]) - [#1847] was deliberately buttons-only. Two other kinds of control parse access keys and had none. **Every grid context menu in both apps** - 38 menus, 223 items, plus the code-built chart, tray, alert-badge and tag-assign menus - now carries a consistent set: `Copy _Cell`, `Copy _Row`, `Copy _All Rows`, `_Export to CSV...`, `_View Stored Plan`, `View Cac_hed Plan`, `_Get Actual Plan (re-run)` and the rest, identical between Lite and the Viewer wherever a menu exists in both. A `ContextMenu` is its own key scope, so the window's buttons are never in contention and the key works without Alt once the menu is open - the one place that mattered is Manage Servers, where `_Edit` keeps E and Export takes `Ex_port to CSV...` instead. **44 checkboxes and radio buttons across eight dialogs** took keys as well, including the ones that ARE the dialog's decision: the auth-mode radios in both Edit Credential Profile dialogs, the credential-source, auth-mode, encryption and per-server option controls in both Add Server and Add Multiple Servers dialogs, and "Use default schedule" in both Collector Schedule editors. Labels are byte-identical between the SKUs there, so parity was free, and a control's key is the same in every dialog that offers it (Managed Identity is always Alt+M, SQL Server Authentication always Alt+Q). Only static literals were touched - a data-bound label is [#1857]'s territory, not a mnemonic site. The full audit re-ran over both apps afterwards: **373 access keys across 59 windows and user controls, zero duplicates**. In-tab checkboxes that live in the main window's key scope rather than a dialog are tracked as [#1878], because the two main windows carry no keys at all today and the first one added there sets the convention for the whole shell. - - **One live defect the rename surfaced, and fixed:** the Viewer located two of its tag menu items by comparing `MenuItem.Header` against the display string, so adding a mnemonic silently broke them - the tag-only actions would have stayed enabled on the Favorites and Untagged pseudo-groups, and the "Assign Tags" submenu would have stopped populating. Both now match on a `Tag` discriminator, because behavior should never hang on a label a translator or a mnemonic can move. - -- **Alert Detail has a Close button, and the plan and graph viewer windows close on Esc** ([#1858]) - [#1847] put `IsCancel` on every cancel/close button in both apps, which left Alert Detail as the one dialog it could not cover: it has no close button at all, its only button is a per-section Copy inside an `ItemsControl` template, and hanging `IsCancel` on that would have made Esc copy something. It is opened straight off an Alert History row, so it is a dialog people open and close repeatedly, and it took Alt+F4 or the title-bar X. Both apps' Alert Detail windows now carry a real **`_Close` in the footer with `IsCancel`**, which matches every sibling dialog and fixes the discoverability gap as well as the keyboard one. The chromeless viewer windows could not take that treatment - the hosted control fills them edge to edge - so **Lite's Plan Viewer and the shared Graph Viewer** get a window-level Esc handler instead. It is wired to the **bubbling `KeyDown` rather than `PreviewKeyDown` on purpose**: a hosted control that wants Esc for itself (a column filter popup clearing its value) marks the event handled and never reaches the window, where a preview handler would have taken Esc away from it. The shared `GraphViewerWindow` was not in [#1847]'s count because it lives in `PerformanceMonitor.Ui` rather than in either app's folder, and it is the Viewer's twin of Lite's Plan Viewer - both apps float plans and block-chain and deadlock graphs through it. **40 windows across the two apps and the shared UI library, 38 of which now close on Esc**; the two that do not are the two main windows, where Esc must not close the application. - -- **Enter no longer submits three Darling Viewer dialogs that their Lite twins leave alone** ([#1859]) - `IsDefault="True"` makes Enter press a button from anywhere in the window, and it was set in the Viewer and not in the equivalent Lite dialog, so the same dialog in the two SKUs answered Enter differently. [#1828] had already declined to introduce Enter-to-submit to Add Server on the grounds that Enter from a half-filled entry form is a surprise rather than an affordance, so convergence went that way: **`IsDefault` is removed from Add Multiple Servers' Add and the mute-rule editor's Save**, both multi-field entry forms, and both now match their Lite twins. **Manage Tags' Close loses it too** - it also carries `IsCancel`, so Esc still closes the window, but the window has a server filter box where Enter would have closed it mid-filter, the same surprise in a window with no Lite twin to drift from. **Tag Name's OK keeps it**: one text field and an OK/Cancel pair is a confirm prompt, which is exactly where Enter-to-submit is correct. Nothing was added to Lite - the only dialog on the confirm-prompt side of the line is Viewer-only, so there was no Lite twin to bring up to it. `IsDefault` now appears exactly once in either app. - -- **Every dialog in Lite and the Darling Viewer now has Alt mnemonics on its action buttons, and Esc closes it** ([#1847]) - [#1828] gave the two Add Server dialogs `_Test Connection` / `_Save` / `_Cancel` because the reporter had gone looking for Alt keys and found none; a repo-wide check during that fix confirmed those were the ONLY dialogs in either app with mnemonics. The sweep finishes the job: 32 dialog files - 17 distinct dialogs, most of which exist in both SKUs (About, Add Multiple Servers, Collection Log, Collector Schedules, Edit/Manage Credential Profiles, Excluded Databases, Manage Servers, Manage Tags, Mute Rules and its editor, the three history windows, Settings, Tag Name, Wait Drill-Down) - gained mnemonics on their action buttons, and 19 windows gained `IsCancel` on their cancel/close button, so 34 of the 37 dialogs across the two apps now close on Esc where 15 did before. Keys are audited for collisions: two buttons in one window sharing a key degrades Alt+key from "press this button" to "cycle focus", which is worse than no mnemonic at all, so the audit runs over all 38 windows in both apps and reports 95 access keys with zero duplicates. `IsDefault` is deliberately NOT added anywhere - Enter-to-submit from a half-filled form is a behavior change, not a keyboard affordance ([#1828]'s decision, kept). Three button shapes are deliberately skipped for cause: per-row buttons inside a `DataGridTemplateColumn` or `ItemsControl` template (the history windows' Download, Alert Detail's Copy) would materialize N instances of the same access key, the column-header filter glyphs in Wait Drill-Down have no text to mark, and the Settings windows' section-local buttons are tracked separately ([#1856]) - three of them carry the identical label "Send Test Notification", so distinct keys need label edits rather than underscores. Three adjacent gaps the sweep surfaced are filed rather than folded in: names containing underscores render wrong wherever a checkbox binds `Content` to a database or server name ([#1857]), Alert Detail has no close button for Esc to hang on ([#1858]), and `IsDefault` is set on Viewer dialogs but not their Lite twins ([#1859]). Esc safety was verified per handler against the WPF source rather than assumed: a cancel Click handler runs BEFORE `IsCancel`'s own action, and `Window.OnDialogCancelCommand` sets `DialogResult` only when the window was shown with `ShowDialog()`, so the pattern is inert rather than throwing on the non-modal windows - and it is the same "IsCancel plus a plain `Close()` handler" combination four windows in this repo already shipped. - -- **The Long Running Query filter checkboxes stopped eating an underscore out of the wait and procedure names they name** ([#1847], found by the collision audit above) - WPF reads a single `_` in a control's Content as an access-key marker, which both removes the character from the rendered label and claims an Alt key. Four checkboxes in each app's Settings window are labelled with raw SQL identifiers, so "Exclude SP_SERVER_DIAGNOSTICS" had been rendering as "Exclude SPSERVER_DIAGNOSTICS" (and silently binding Alt+S), with `BROKER_RECEIVE_WAITFOR`, `XE_LIVE_TARGET_TVF`, and `sp_MScdc_capture_job` mangled the same way. The underscores are now doubled, which is WPF's escape for a literal one - the labels read correctly and claim no keys, which is also what freed Alt+S for Settings' new Save. - -- **Azure SQL Database now attributes its Query Store rows to a replica role instead of storing a NULL** ([#1872]) - [#1844] added `replica_role` so an Availability Group primary's Top Queries could not silently blend secondary-replica workload into the primary's own numbers, and [#1836] deliberately gated it OFF for Azure SQL DB. That gate was about bind SAFETY, not taste: `sys.query_store_runtime_stats.replica_group_id` is documented for "SQL Server (Starting with SQL Server 2022 (16.x))" with no Azure SQL Database in its applies-to note, on a page whose sibling columns name Azure explicitly when they mean it; and Query Store for secondary replicas is documented as unavailable on the Hyperscale service tier, with the docs silent on whether the view and column still BIND there. **A column that does not bind is not a NULL** - it fails the whole payload for that database, and Azure collects per database, so on Azure that would have meant every database going dark rather than one column. The gate's own comment named the exact evidence that would lift it, and that evidence is now in hand from live databases on **both** service tiers it worried about ([#1848] General Purpose, [#1872] Hyperscale, same engine banner and EngineEdition on each): `OBJECT_ID('sys.query_store_replicas')` and `COL_LENGTH('sys.query_store_runtime_stats', 'replica_group_id')` both return non-NULL on each tier, the view binds and holds the same four role rows box SQL Server has, and - the decisive check - the full 55-column payload composed with the attribution ON executed clean on Hyperscale, binding all 55 columns and returning `replica_role = Primary`. A per-tier gate was considered and rejected as unbuildable: Hyperscale reports the same engine edition as General Purpose, so the collector cannot tell them apart when it builds the query, and now does not need to. **Nothing changes on the read side** - the column already existed and every dedup key and rollup that touches Query Store already partitions on `replica_role`, so an Azure server's rows go from one NULL group to one `Primary` group and the grids render identically. **Managed Instance is deliberately unchanged** and keeps the pure version gate, for the same reason `plan_type_desc` gives: MI under-reports its product version too, but its feature set follows a per-instance update policy rather than being evergreen, so "Azure means 2022+" is not sound there. Riding along, a stale comment correction found during the same validation ([#1871]): `BuildPayloadBody`'s summary said the two execution paths select "53 reader ordinals" when the real count is 55 - the class-level doc and the reader itself already agreed on 55, and the count had simply not been updated when [#1841] added the interval-identity pair. It matters because that one comment is the drift guard between the two paths' column sets, so a wrong number sits in exactly the place a future audit would trust. - -- **CI: the version-bump check survives the deprecated/ layout transition** ([#1821]) - the check compares the PR's Dashboard.csproj version against MAIN's copy, but read main's copy only at the NEW deprecated/ path - which does not exist on main until the v3.3.0 promotion itself lands, so the release PR failed the check on a path error rather than a version verdict. The main-side read now falls back to the pre-move location; removable once main carries the new layout. - -### Fixed - -- **`--encrypt-password` and `--configure-network` explain themselves on a non-interactive console instead of silently doing nothing** ([#2097], reported with the diagnosis by gotqn) - in the PowerShell ISE, remote/PSRemoting sessions, and some integrated terminals, stdin is not an interactive console: `ReadLine()` returns null immediately, and the prompts and errors these verbs wrote to STDERR are not surfaced at all in those hosts - so the very first setup step (encrypting the SQL password) read as a hung or broken tool. Both verbs now write an actionable explanation to STDOUT (the one stream every host shows) naming the cause and the ways forward - run from a real console, or pipe the value: `Read-Host -Prompt 'password' | PerformanceMonitor.Darling.Service.exe --encrypt-password` - and the wizard now tells EOF apart from an explicit quit (guidance + exit 1 vs the quiet \"No changes made.\" + exit 0). - -- **Linux: `add_servers` accepts `env:`/`file:` secret references, unblocking onboarding on compose deployments** ([#2087], found during the 3.4.0 release smoke) - the MCP/web onboarding path refused every SQL-auth password off-Windows ("Storing a SQL-auth password requires Windows (DPAPI)"), which dead-ended the DESIGNED way to add servers to a running Linux deployment: the control plane is store-authoritative after first seed, so darling.json edits do not add servers. A reference is a pointer, not a secret - the secret stays in the mounted file or environment variable, the #1804 contract - so references are now stored verbatim in the encrypted-password slot and resolved at connect time (a reference can never be confused with a DPAPI blob). Literal passwords still require Windows, and the refusal now says exactly what to pass instead. - -- **An already-hardened config backup no longer errors on every service start** ([#2093], reported by ghauan) - the startup sweep that closes the ordinary-users-can-read exposure on `darling.json.bak-*` files re-attempted its ACL rewrite unconditionally, and the rewrite needs OWNERSHIP - which the error message's own icacls remediation does not transfer. So an operator who followed the instructions fixed the exposure and still got the ERROR line every start, about a file that was already secure (the independent READABLE-by-Users CRITICAL check was silent - the honest witness). The sweep now checks whether the exposure is already closed and skips the file entirely when it is; the CRITICAL check stays as the witness for the still-exposed case. - -- **Self-alerts and PVS alerts now carry their real severity in Teams/Slack/PagerDuty/webhook payloads instead of rendering INFO-blue** ([#2090], diagnosed precisely by gotqn) - every Darling self-alert (Collection Stopped, Agent Not Running, Capture Down, Store Disk Pressure, Store Runtime Upgrade, Compression Job Stuck) fires with an explicit severity at its site, and that severity rode the outcome into the log line - then died, because the channel builders read only the CONTEXT's severity override and self-alerts deliver with no context. The deliverer now folds the outcome's severity into the context once, upstream of every channel (an explicit override from a context builder still wins), and the context serializes into alert history so replays keep it too. The severity map also gains backstop arms for all six self-alerts plus Version Store (PVS) - the render path for history replays, which have no context by nature - and a TRIPWIRE test now enumerates every fired metric name against the map, because this exact gap (the #1136 fall-through) has now shipped five separate times and nothing previously forced a new alert's author to visit the map. The machine-readable category/severity payload fields from [#2090] follow separately. - -- **`--configure-network` no longer corrupts its edit (and then refuses to write) when a postgres/mcp/web section's last member is a string with no trailing comma** ([#2073], reported with the root cause by gotqn) - the wizard synthesizes the separating comma "after the previous member's value", but its backward scan for that value was code-only, and a string value is entirely string-literal region (quotes included) - so on a minimal config like `"dataDirectory": "E:\\pg"` the scan skipped the whole value, landed on the member's COLON, and spliced the comma there (`"dataDirectory":, "..."`). The write-time parse gate caught the damage every time and aborted with "the edited darling.json did not parse ... No changes were written", so no config was ever corrupted on disk - but LAN exposure could not be configured through the wizard for any file of that shape, and the shipped sample never trips it (its sections' last members carry trailing commas, which the scan finds first). The scan is now member-end-aware: a string-literal hit is always a real value's closing quote (quoted runs inside comments classify as comment), so the comma lands after the value. Regression fixture is the reporter's exact minimal-config repro. - -- **Lite's two server-delete doors now run the same deep cleanup - Manage Servers' Delete no longer leaves a removed server's runtime state behind for a re-add to resurrect** ([#2033], found during the [#2026] review) - a server's id is a deterministic hash of its storage name, so a removed-then-re-added server gets its old id BACK, and every piece of hash-keyed runtime state left behind resurrects with it. The sidebar context menu's Remove cleaned up properly (collection health, the [#1696] AG edge state whose staleness pages a phantom failover, and tag assignments); Manage Servers' Delete removed only the registry entry and left all three - it could not even reach the services that hold them. The cleanup is now ONE MainWindow helper both doors call: the sidebar path inlines it, and Manage Servers receives it as a delete callback, run before the registry delete (so the storage-name hash still derives from the intact connection) and never allowed to block the delete itself. The dialog-close health sweep stays as the belt. - -- **A transient config read failure at boot no longer silently kills the web dashboard and MCP server for the whole process lifetime** ([#2038], investigating a field report of both dying across a service restart) - each host front-loaded one \`darling.json\` read whose failure stood that host down until the next manual restart, logged only at DEBUG, which default logging never shows: one anti-virus pass holding the file mid-restart was enough to take out both HTTP surfaces while the collectors kept running, with nothing in the log saying why. The load now lives inside each host's supervisor loop on the existing 30-second failed-start backoff, logged at WARNING with the retry cadence - a transient failure self-heals in seconds, and a persistent one says so on every attempt instead of never. Once loaded, the config is held for the process lifetime exactly as before: the network-exposure block stays restart-only by design. - -- **Removed servers leave the web dashboard instead of lingering as ghosts** ([#2030], reported from the field) - the reload sync that mirrors the desired config onto the observed server registry (`collect.servers`) was an inner join on the config row, so it could flip a DISABLED server's observed flag but never a DELETED one's: a server removed from monitoring kept its last `is_enabled` - almost always TRUE - forever, and every reader scoped to `WHERE is_enabled` (the web dashboard's `/api/fleet` cards, `get_fleet_overview`, the fleet Collection-Health aggregate, FinOps) listed the ghost indefinitely, dataless. The sync now has a second half: an observed row with NO desired row flips to disabled. Deliberately a flag flip, never a row delete - the registry row anchors whatever collected history retention has not yet aged out, and a re-added server (same storage name, so the same server_id) flips back on the next reload and resumes under its old identity and history. Runs on every control-plane reload, so existing stores clean up their accumulated ghosts on the first reload after upgrading, with a log line saying how many. - -- **Add Multiple Servers accepts SQL Server's own \`host,port\` syntax instead of eating the port as a display name** ([#2027], reported from the field) - the paste grammar splits comma lines into \`server[, display name][, database]\`, so \`sql01,2433\` parsed as server "sql01" with display name "2433" and connected to the DEFAULT port - silently wrong - and the four-field \`sql01,2433, Prod, master\` was rejected as too many fields, making non-default-port fleets impossible to bulk-add at all. On a comma line, a second field that is a pure number in the valid port range (1-65535) is now recognized as a port riding the server name: it folds back into the server field and the rest shift left, so all of \`sql01,2433\`, \`sql01,2433, Prod POS\`, and \`sql01,2433, Prod POS, master\` parse the way SQL Server tooling reads them. Out-of-range or non-numeric lookalikes ("0", "65536", "24x3") stay display names, and tab-separated spreadsheet pastes are untouched - their first cell always carried \`host,port\` intact, and the tab form remains the escape for the one thing the rule costs, a display name that is only digits. Fixed once in the shared parser, so Lite and the Darling viewer stay grammar-identical; both dialogs' hint text now states the port form, and the mirrored grammar pins in both test suites cover the fold, its boundaries, and the tradeoff by name. - -- **The Viewer's tag editors now honor the read-only seat instead of failing silently** (from [#2008]'s audit) - on a provisioned store the least-privilege `viewer` role deliberately has no write on the tag tables (its ONLY write is `config.custom_views`), but the tag menus never gated on the read-only probe the way every other write surface does - so "Assign Tags", create/rename/delete, and the bulk Manage Tags editor all LOOKED functional, and every change quietly died as a permission error behind a status-bar line. The tag editors now disable on a read-only seat with a tooltip saying why ("this seat is connected with the read-only viewer role - tag editing needs the admin connection"), the same `IsReadOnly` gate the other write affordances use. The role model is unchanged on purpose: tags are shared fleet configuration, and a read-only seat stays read-only. - -- **The Darling Viewer looks for a relative `Root Certificate` beside `darling.json`, not beside wherever it happened to be launched from** ([#1970]) - a bring-your-own `postgres.connectionString` was handed to Npgsql verbatim, and Npgsql resolves a relative certificate path against the process **working directory**. That is not necessarily the folder holding `darling.json` and the certificate exported beside it: a desktop shortcut's "Start in", or launching from Explorer's last-used folder, silently changed which file a `SSL Mode=VerifyFull` connection pinned. So the working directory was the trust anchor for the pin - and launched with a working directory an attacker can write, a planted `server.crt` becomes the pinned root, which with LAN position is interception of the store session and the credential inside it. A relative or bare `Root Certificate` is now resolved against the directory of the `darling.json` the viewer actually read, so the exported folder works wherever it is copied and however the viewer is started. An absolute path is used exactly as written, and the keyword is never ADDED to a connection string that did not carry it - a connection that was not pinning is not made to pin. - - **The export docs lost a caveat rather than gaining one.** `--export-viewer-config` emits a bare `Root Certificate=server.crt` (the service host cannot know the viewer machine's absolute path), and its `darling.json` comments, its `README.txt` and the Darling README all told you to hand-edit that value to a full path if you kept the folder anywhere other than beside the Viewer executable. That edit is now unnecessary: point `DARLING_CONFIG` at the exported `darling.json` and the `server.crt` beside it resolves with nothing changed. Those three texts now document the anchor instead of the workaround, because a documented workaround that is no longer true is worse than none - it gets you to hardcode a path that breaks the next time the folder moves. - - One resolver does both jobs. The startup diagnostics added in [#1954] promise "the absolute path Npgsql will actually open", and the connection string is rewritten before Npgsql sees it; both call the same code, so the path the block names cannot become a second opinion about the path the connection uses. - -- **A failed `--export-viewer-config` no longer leaves a half-finished handoff folder behind without saying so** ([#1973]) - a write that died part-way through the export removed the secret-bearing `darling.json` and stopped there, so if the failure hit `server.crt` or `README.txt` the files written before it stayed on disk. Since each write deletes the previous export's copy before recreating it, the folder could be left as a mix of new and stale files with no config to go with them - and the error named none of it, which reads as "nothing was written" when something was. A failure now sweeps all three files (the cert and the README carry no secret and are useless without the config), and the error names each one either as removed or, if it could not be removed, as left behind with the reason. The `darling.json` line still says what it says today - that it held a live password - because a removal an operator might have to redo themselves is the one that matters most. Each removal is independent, so one that fails hides neither the original error nor the other two. - -- **A Lite test suite that failed about one run in six, on a race between the tests rather than anything in the product** ([#1965]) - `McpAlertSettingsKeyTests` asserts that Lite reports `cpu.mode` in Darling's lowercase vocabulary, and it proves that at RUNTIME, by setting `App.AlertCpuMode` and serializing the real MCP payload. `App.AlertCpuMode` is a process-global static, xUnit runs separate test classes in parallel, and two other classes write it: `LiteAlertForwardingTests` directly, and `AlertSettingsCredentialLoadTests` through `App.LoadAlertSettings`, which rewrites the entire alert block rather than just the webhook keys it was reasoning about. A foreign write of `Total` landing between the set and the assert failed the SqlOnly case with `Expected: "sql" / Actual: "total"`. Reproduced deliberately at **7 failures in 12 runs** of the two classes together, against 0 in 22 after the fix. All four classes that touch these statics now share one xUnit collection so they cannot interleave - the same serialization idiom the SMTP/webhook pair already used, widened and renamed from `app-webhook-statics` to `app-alert-statics` to match what it actually covers, rather than a new mechanism invented for this. No assertion was weakened. - - The stale comment was load-bearing, which is why the third participant went unnoticed: `LiteAlertForwardingTests` documented itself as "the ONLY test class that mutates the App.Alert\* statics", so the two classes that arrived later each reasoned about a narrower conflict than the one that existed. It now names them. The forwarding class also gained a `Dispose` that restores the statics after each test - serialization alone stops the classes overlapping but not the last test's values leaking into whatever runs next - and its reset now covers `AlertBlockingWaitSecondsThreshold`, which one of its own tests set to 745 and nothing put back. A suite that fails intermittently trains people to re-run rather than read, which is the same trust problem [#1957] was about. - -- **The Darling installer's SECURITY WARNING was right, and the files it named really were left world-readable** ([#1957], from three consecutive installs on a 24-server RDS field box) - it was reported as a false alarm, because `icacls` captured immediately before and after each install showed `darling.json` and both `.bak` copies in exactly the correct posture. The warning was honest and the installer was the thing breaking them: after applying the hardened ACL it set the service account as owner by handing `Set-Acl` a **freshly constructed** `FileSecurity` carrying nothing but that owner. `Set-Acl` applies the whole descriptor it is given, so that second call also wrote an empty, unprotected DACL - re-enabling inheritance and handing `BUILTIN\Users` read straight back on an install under `C:\`, one statement after the hardening. Measured on a scratch layout under `C:\`: immediately after the owner step the file read `AreAccessRulesProtected=False` with `BUILTIN\Users` present, all four inherited ACEs back, and every hardened ACE gone. The owner is now set on the file's **current** descriptor, so the hardened DACL rides along and the state the verification judges is the state the installer leaves behind - proven on the same layout, where a hardened file now produces no warning and a deliberately re-exposed one still does. - - **Why three releases of a red warning went unactioned, and what was actually exposed.** [#1818]'s startup sweep re-hardens the live config and every `darling.json.bak-*` sibling at each service start, which is seconds after the installer finishes - so the operator's own before/after measurements were both taken outside the window and looked clean, and only the installer disagreed with them. The exposure was therefore real but brief, between the end of the hardening block and the first service start; it is now closed at the source rather than repaired after the fact. A security warning that has cried wolf three times trains the operator to skip it, which defeats the one time it is real - the reporter's framing, and the reason this is a defect and not a cosmetic nit. The per-file verification predicate is untouched, because it was correct throughout. The install summary now states that the service re-verifies these ACLs at every start, so the two mechanisms' relationship is documented rather than coincidental. Pinned structurally against the script, since the defect lives in PowerShell's `Set-Acl` semantics where no compiler looks and the equivalent .NET call - which persists only the sections it sees modified - would have gone green against the exact code that broke the field boxes. - -- **Darling's retention summary line described a posture the store does not have** ([#1958], from the 3.3.0-nightly.20260731 field validation) - it read `(raw 4 days, hourly CAGGs 90 days; daily CAGGs kept indefinitely)`, a universal claim with three counterexamples sitting in `timescaledb_information.jobs` - the very table the docs send an operator to when they want to check it. `query_store_stats_interval_hourly` carries 7 days BY DESIGN ([#1849]'s interval-identity dedup tier, internal plumbing sized only to outlive raw so the arming gate cannot race itself, not user-facing history); its daily twin carries 10 despite the promise about dailies; and the nine baseline aggregates keep 35. The operator cross-checking hit the first one immediately and had to work out whether they had found a bug, which is the whole cost - a summary line is only worth printing if it survives being checked. The line now names every tier that has a policy and qualifies the two that are kept indefinitely, with **every number interpolated from its constant** rather than restated ([#1942]'s rule; this exact line class has drifted that way before). Newly pinned by a test that derives its expectation from the policy list itself, so a tier added on a new horizon fails until the summary mentions it - there was no pin on this line at all before, only a doc comment claiming the property. -- **default_trace_events reads the CURRENT trace file instead of the whole rollover set** ([#1962], from a field investigation on a 52-replica RDS fleet measuring 5.0x) - the collector rewrote the trace path back to the BASE file and passed max_files, so every cycle re-read all five 20 MB rollover files to return a handful of new events: fn_trace_gettable has no predicate pushdown, so the StartTime watermark filters only AFTER the whole set is materialized. Steady-state cycles now read just the current file, and fall back to the full set whenever the trace has ROLLED since the previous cycle - because post-watermark events then span the previous file and the current one. The reporting fleet measured 897 ms whole-set against 178 ms current-file on its apex box (~1.25 s average at the 6-minute tier, roughly 4.5 SQL-hours/day fleet-wide, with 11 s first-run maxima that now confine themselves to the genuine fallback path). Reproduced locally on SQL Server 2022 and 2025 with the reporter's methodology: a cutoff chosen INSIDE the current file's span, where both arms must see every qualifying event, returned identical counts and identical CHECKSUM_AGG over (EventClass, SPID, StartTime), at 4.4-4.9x. **The fallback is what keeps this lossless**, and it is deliberately conservative: a cutoff spanning a rollover returned 1,161 events from the current file against 4,497 from the full set, so a cycle that cannot prove the trace stayed put reads everything. That decision needs one new piece of per-server state - the trace file the previous cycle read - which cannot be derived from the collected rows the way the existing event_time watermark is, precisely because the cycles that most need it collect NOTHING: a server whose trace churns through files without producing curated events must still notice the rollover. It is stored per server and per collector in both apps (Lite creates the table on next start; Darling migrates to V44) and survives restarts; no stored path at all - first run, a restarted host, a store upgraded onto this build, a cycle that failed - takes the fallback, because a collector that cannot know what it missed must not assume it missed nothing. Azure SQL DB has no default trace and was already gated off; nothing changes there. On Darling the state row's `updated_at` is written as naive UTC like every other timestamp in that store, rather than being silently offset by the server's UTC zone. - -- **query_stats and procedure_stats rank BEFORE they render** ([#1959], from a field investigation on a 52-replica RDS fleet) - both collectors spliced their expensive applies (statement text, and Darling's plan-XML render) below the TOP, and the optimizer does not defer them: a captured field plan showed dm_exec_text_query_plan executing 2,434 times to keep 200 rows - 81% of the entire sweep - with 30-second command-timeout MISSES on big-cache boxes. Both queries now rank on the cheap DMV columns inside a derived table first and run the applies against the survivors only, making the deferral structural instead of hoped-for. Field-validated at 6.0x on the reporting fleet's heaviest box (median 7.7s to 1.3s, renders 2,434 to 200, 99.2% row parity with the residual explained by cache churn between paired runs); the timeout misses concentrate exactly where discarded renders scale worst and are expected to disappear. procedure_stats gets the structurally identical fix as tail insurance: free when candidates fit under the TOP (the measured daytime case), bounded when overnight batch windows blow past it. The query_stats inner TOP carries headroom (300 over 200) because the self-filter now runs post-ranking; procedure_stats needs none (all-cheap ranking, exact parity), and its plan handle round-trips from the varchar the payload already carries so no extra column threads through. The survivors-only ordering is pinned structurally in both collectors' definition tests. - -- **--collapse-legacy-slices was unreachable from the command line** ([#1912] follow-through) - the verb had a full dispatch, help text, and live-tested internals, but the #1581 startup classifier's IsKnownVerb allow-list never learned it, so the binary answered 'Unknown option' to a verb its own help advertised. Caught on the first real field deploy, not by any test, because the verb's live tests called the command layer directly and never crossed Program.Main's argument seam. The allow-list is fixed, and a reflection pin now walks every verb classifier on the class and proves IsKnownVerb reaches it - a future verb whose author forgets the list fails by name at build time instead of in the field. - -- **Chart lines no longer connect across collection gaps** ([#1944], from discussion #1936) - a monitor that was offline rendered a continuous line through the outage, which reads as data that was never collected. Every time-series line in both apps (104 series) now breaks where the gap between points exceeds three times that series' own median spacing - derived per series because collectors run at wildly different cadences (one-minute counters to daily index stats), so no constant could serve both without shattering slow series or papering over real outages on fast ones. The median rather than the mean, so the outage gaps being detected do not inflate the yardstick that detects them. Ordinary scheduler jitter, duplicate timestamps, and a single missed cycle stay connected - pinned both directions. -- **The #1912 destroy-pin live test is self-sufficient under a SearchPath-pinned rig** ([#1940]) - it builds a raw scratch hypertable with no migrations, so a DARLING_TEST_PG carrying the local-rig recipe's SearchPath=collect,config pin left no writable schema on the path and the test died 3F000 while CI's bare connection string stayed green. The test now sets its session path explicitly; proven under both connection-string flavors. - -- **Every remaining 21-day mention in code and CLI output caught up with the 90-day horizon** ([#1937]) - the retention change shipped with the constants and routing derived, but thirteen comments and one OPERATOR-FACING line (the --backfill-rollups completion guidance, which still taught the old trim timing) described the 21-day world. The CLI line now interpolates the horizon constant so it can never drift again; current-behavior comments are corrected or made horizon-neutral; historical narration that legitimately describes the old defect keeps its 21s. - -- **The Query Store slicer overlay is drawn at the hour the work RAN, so it lines up with the bars underneath it again** ([#1921]) - selecting a Query Store row overlays that item's curve on the slicer. [#1841] moved the BARS to the hour an interval ran and left the OVERLAY on the hour it was collected, so a point sat up to one Query Store interval - 60 minutes by default, 1440 at the maximum - to the RIGHT of the bar describing the very same work. The file's own comment stated the invariant this broke ("leaving the overlay raw would make the overlay disagree with the bars it annotates"): its dedup half kept holding, its placement half silently stopped. Both apps now place the overlay on `COALESCE(interval_start_time_utc, collection_time)`, the same expression the bars use, and rows collected before [#1841] tier 2 keep collection-time placement exactly as every other read handles that generation. - - **The window moved with the x-value, which is the part that would otherwise still disagree.** Filtering on collection time while plotting on the interval start disagrees at BOTH edges - an interval that started before the window but closed inside it draws a point to the left of the range, and the window's own last interval, whose closing fetch lands after the range ends, vanishes - and worse, it disagrees with the bars about whether an interval is in the window at all. The overlay now filters on the same expression it plots on, with the same `collection_time` pruning bounds the slicer already ships ([#1892]/[#1923]); the issue's own scope note said "no ceiling", which was written before that round added one. - - **Two accepted costs, stated so they are not re-reported as bugs.** The overlay can no longer show WHEN a value was observed - that was a real thing to want, and it is what option 2 would have kept - and several intervals returned by one collection cycle now spread across the x-axis instead of stacking at that cycle's instant. Spreading is the truthful picture: those intervals really did run at different times. - - **Lite's overlay also gained the dedup it never had.** It had been fed from the history-grid read, which is a deliberately raw per-collection projection ([#1845]) windowed on collection time - correct for a list of collection events, wrong for a series drawn over the bars. Moving that shared read would have silently changed the history grid too, so Lite gained a dedicated overlay read instead, matching the structure Darling has had since [#683]. One consequence is visible: an interval collected repeatedly used to draw as a rising staircase of restatements and now draws as one point at its final values, which is what the bar beside it has always shown. - -- **Darling gains `--collapse-legacy-slices`, which repairs the Query Store rows stored before the split-slice fix and re-materializes the rollups they fed** ([#1912]) - the companion to Lite's automatic repair above, for the store where the same rows live. Same collapse, same count-weighted math, same idempotence: the pre-fix signature is two or more rows sharing the entire dedup key AND `collection_time`, which no row collected since [#1907] can match, so the verb is safe to re-run and a second pass reports nothing to do. It surveys first and `--dry-run` reports real numbers while changing nothing. - - **It is bounded by raw retention, and that bound is the whole story.** Only rows still in the raw tier can be collapsed, so the repair reaches the last few days - which at upgrade time is exactly the recent history an operator is looking at, and is why **running it promptly after upgrading is what makes it worth anything**. Query Store numbers older than the raw window keep their understated counts PERMANENTLY: the daily rollups are kept indefinitely and cannot be rebuilt from raw that retention has already dropped. That is disclosed rather than worked around, beside the [#1849] boundary it shares. - - **The refresh is clamped to the rows actually collapsed, and that is a safety property rather than an optimization.** Measured on PostgreSQL 18.4 + TimescaleDB 2.28.1: a refresh whose range lies entirely within DROPPED raw chunks **destroys** the materialization there - with `force` and without - while a range spanning dropped and live chunks preserves it, which is why it took a three-way single-variable probe to see at all. Aiming at a nominal "pre-fix period" instead of at the collapsed rows' own span would therefore have blanked the 21-day hourly and the indefinitely-kept daily below raw's floor, which is the one thing [#1759]/[#1793] forbid. Rows that were collapsed are inside raw's extent by definition, so deriving the window from them cannot reach under it. The behavior is pinned by a live test, so a future TimescaleDB that makes it safe fails loudly rather than quietly widening what a mistake would cost. The shipped `--backfill-rollups` was checked for the same exposure and is safe by construction - it converges down to raw's oldest row and never refreshes below it. - -- **The retention chunk-geometry guard now reads the chunks on disk, not just the interval the catalog is configured with** ([#1925]) - [#1915] shipped the geometry half of the coverage invariant by reading each relation's CONFIGURED chunk interval. That is the whole story only for a store that never changed one. `set_chunk_time_interval`, and a change to the raw chunk interval, apply to chunks cut AFTER them - existing chunks keep the width they were cut at - so a relation that has lived through a change carries a MIX, and the configured interval describes only the newest of them. - - **Widening reads safe; narrowing was a false green.** Measured directly on PostgreSQL 18.4 / TimescaleDB 2.28.1: a table created at 1 day and widened to 7 reports `7 days` as its interval with six 1-day chunks still underneath it. Read that way, widening is conservative - the configured number is the wider, worse one. NARROWING is the opposite: the configuration returns to a small value while wide chunks sit beneath it, and a guard reading configuration alone passes a store whose real rounding it never saw. That is not hypothetical arithmetic; the test now builds exactly that store - wide chunks, then a narrowed configuration - and the pre-[#1925] reading passes it, which is what the mutation run demonstrates. - - **Judgement now runs on the WIDEST width a relation carries, configured or on disk.** Old chunks age toward the retention cutoff and eventually sit on it, and every future chunk takes the configured width, so both are candidates for the chunk that straddles the cutoff and the worst of them is what bounds the rounding. A relation with no chunks yet reports only its configured interval, so a fresh store is judged exactly as it was before - this strictly extends [#1915] rather than replacing it. A relation carrying more than one width is also refused the identical-grid argument outright, since something on two grids cannot support a superset argument whatever its widest chunk equals. - -- **Query Store execution counts stopped reporting a sliver of an interval instead of the whole thing** ([#1907]) - `sys.query_store_runtime_stats` returns the FLUSHED slice and the still-IN-MEMORY slice of one `runtime_stats_interval_id` as two separate rows, and they are ADDITIVE members of one interval rather than competing snapshots of it. The collector selected straight from the view, so both were stored - and they then shared every column of the read-side dedup key ([#1841]/[#1845]/[#1853]) **and** `collection_time`, which meant the viewer's `ROW_NUMBER() ... ORDER BY collection_time DESC` and the rollups' `last(execution_count, collection_time)` were both ordering by a value identical for both rows. The survivor was whichever the engine happened to emit first. Live on Azure SQL Database that showed as **8 executions reported where 94 was true**, on two of five rows, with a different two wrong on the next run - not a stale number but an arbitrary fraction of one. Query Store's default 900-second flush against its default 3600-second interval means one interval can hold several flushed slices, so the split is not bounded at two, and it is not an Azure peculiarity: it reproduces on box SQL Server 2022 (16.0.4255.1), where 100 executions flushed plus 25 in memory came back as two rows while `sys.dm_exec_procedure_stats` - a wholly separate source, read at the same instant - reported 125. - - **The slices are now combined at COLLECTION**, in the one payload body both the on-prem and Azure execution shapes run, grouped on exactly the natural key of the view (`plan_id`, `runtime_stats_interval_id`, `execution_type`, replica group) so one interval yields at most one row per cycle. Read-side aggregation was not an option and is not a matter of taste: it would mean teaching every consumer the difference between "same interval, later cycle" (keep the latest) and "same interval, same cycle" (add them), and a TimescaleDB continuous aggregate cannot express that at all. `execution_count` SUMs; every `avg_*` column takes the count-WEIGHTED mean, because Query Store stores an average and a count but never a total, so `avg * count` is what recovers a slice's total - a plain average of the slice averages would weight a 25-execution sliver the same as a 100-execution flush. `min_*` and `max_*` take the extreme, `first_execution_time`/`last_execution_time` the interval's own span. Verified live: the emitted collector SQL returns 125 and 70 where the raw view holds `{100, 25}` and `{60, 10}`, matching `dm_exec_procedure_stats` exactly, and the weighted mean lands on 1871.856 for slices of (1778.42 over 100) and (2245.60 over 25) - which is `(1778.42*100 + 2245.60*25) / 125` and is not the 2012.01 an unweighted average would have given. - - **The incremental cutoff had to move from a per-slice `WHERE` to an interval-grain `HAVING`, and that is load-bearing rather than tidiness.** The flushed slice is STATIC, so once the growing in-memory slice pushes the watermark past its `last_execution_time` the flushed slice stops qualifying - and a sum over the survivors is the sliver alone, the original defect with an aggregate bolted on top. `HAVING MAX(last_execution_time) > @cutoff_time` asks whether the INTERVAL saw new activity and then takes all of it, which is strictly more permissive than the predicate it replaces, so nothing that used to be collected stops being. **The fixed query is FASTER than the one it replaces**, despite the added aggregate: measured on a real 212,000-row Query Store with the full 55-column payload, 375/422/438 ms against 453/485/516 ms, because half the rows means half the `nvarchar(max)` query text and plan XML to materialize and ship. That depends on the interval pre-filter, which is a prune and not a semantic - its interval list is by construction a superset of what the `HAVING` keeps, so it cannot subtract a row - and without it the aggregate runs over the database's entire retained Query Store every cycle and costs 1203 ms instead. - - **No stored row shape changed**: the same 55 columns in the same order, the same positional writers, no migration and no storage-version bump - only the number of rows per interval. The `TOP` backstop now caps intervals rather than slices, which is also why it sits outside the aggregate: a cap falling mid-interval would emit a partial sum, which is worse than omitting the interval. The deprecated Dashboard's `collect.query_store_collector` had the identical defect against the identical view and takes the identical fix (it is a proc body change, so an upgrade re-applies it with no schema step); its grouping key carries `replica_group_id` under a 2022+/Azure gate for the same bind-safety reason the collector's does, since naming a column that does not exist in a `GROUP BY` fails the whole batch rather than yielding a NULL. - - **Rows already collected cannot be repaired, and are handled honestly rather than quietly.** All 19 read-side dedup sites across both apps now order by `collection_time DESC, execution_count DESC`, so a pre-fix tie resolves to the FLUSHED slice - the one holding the bulk of the interval's work - deterministically instead of flapping between runs. That is closest-available, not correct; the correct value is the sum, and the materialized rollups cannot be tie-broken at all because `last()` has no tie-break and the tied rows are gone once materialized. The residual, including the fact that the indefinitely-kept daily tier keeps understated counts for the pre-fix period, is tracked as [#1912]. Verified end to end against live PostgreSQL 18 + TimescaleDB 2.28.1: the corrected rollups report the hand-computed 155 for an hour of two combined intervals and the exact execution-weighted mean off the same rows, while the same store fed the pre-fix split slices reports a single slice and can never reach the 125 they add up to. -- **A live-test rig without TimescaleDB preloaded now says so, instead of failing every test with "Connection is not open"** ([#1922]) - `CREATE EXTENSION IF NOT EXISTS timescaledb` does not merely ERROR when the library is on disk but missing from `shared_preload_libraries`: it **terminates the backend**. `TimescaleSupport.TryEnableAsync` catches that and returns `false` exactly as its contract says, so the caller reads "carry on in plain-PostgreSQL mode" while the connection it handed over is dead. The `live-postgres` collection fixture then ran three more calls on it and every test in the collection died with `InvalidOperationException: Connection is not open` pointing at `RelationsAsync` - the real cause named nowhere, in neither the message nor the stack. That is the [#1794] masking signature in fixture SETUP rather than teardown, and it is precisely how a fresh local rig presents, since the bundled `pg-runtime.zip` ships the extension files while a hand-rolled `initdb` has no preload line. **Measured before and after against a deliberately unpreloaded PostgreSQL 18.4**: before, the whole collection failed on the masked error; after, the fixture survives and the single failure is its own assertion - "TimescaleDB is not enabled on DARLING_TEST_PG. The live rig is expected to have it" - with the other fifteen tests passing. - - The enable now runs on its own short-lived connection (`LiveTimescaleProbe`, deliberately shaped like [#1896]'s `LiveStoreCleanup`: the safest connection to risk is one nobody else holds), so nothing the caller owns can be destroyed by it. A second call site had the same shape and a wider blast radius - it discarded the result entirely and kept using the connection for the whole test - and is converted too. **The product was checked first and is safe, verified rather than assumed**: `DarlingWorker` opens a DEDICATED connection for the TimescaleDB block and gates every subsequent call on the returned flag, so a `false` return means nothing touches it before it is disposed. Because that safety is a property of how the block is written rather than something the compiler holds, it is now pinned - one test fails if a call moves out from under the flag, another fails if any test-side caller reuses a connection it handed to `TryEnableAsync`. `TryEnableAsync` itself gains the warning its contract was missing. Test-suite only, plus that one documentation comment - no shipped behavior changed. - -- **The force-plan recommendation was telling operators to write the one call form that errors** ([#1914]) - [#1882] disclosed that a secondary-derived recommendation forces on the PRIMARY, and told the operator how to scope it deliberately: "pass it as the fourth argument (they are extended stored procedure arguments, so the documented order matters)". That instruction came from the reference page's syntax block and had not been run. **It fails.** `sp_query_store_force_plan` is an `EXTENDED_STORED_PROCEDURE`, so `sys.system_parameters` is empty for it and its argument surface can only be established by executing it - which is what [#1914] did, on **SQL Server 2022 (16.0.4255.1) and SQL Server 2025 (17.0.4045.5)**, each form run from a freshly-unforced plan: - - - `@query_id, @plan_id, @disable_optimized_plan_forcing = 0, @replica_group_id = 1` - the documented four-argument order - **fails on both versions** with error 12463, *"Role id should be between (including) 1 and 4"*, for a role id of 1, which is in that range. The same call with that argument set to `1` succeeds. - - `@query_id, @plan_id, @replica_group_id = 1` - three named arguments, skipping the middle one - **works on both**. Named arguments are honored and may even be reversed (`@plan_id` before `@query_id` succeeds), so the doc's ordering warning does not describe named-argument behavior at all. - - The recommendation now names the three-argument form, and says explicitly not to add `@disable_optimized_plan_forcing` to reach it. Two further probe findings are in the text because they change what an operator should do: **SQL Server 2025 does not validate the group id** - `@replica_group_id = 99`, not a role at all, SUCCEEDED on a standalone server and wrote a row into `sys.query_store_plan_forcing_locations` naming replica group 99, where 2022 rejects the same call - so the id must be checked against `sys.query_store_replicas` before use, on exactly the version where the feature is GA. And **nothing protects a typo**: a misspelled `@replica_groupid` still reached the replica-group logic, and an entirely invented parameter name was accepted silently. - - **Those findings also close [#1914]'s own proposal without building it.** The issue asked whether to collect `replica_group_id` so the recommendation could emit a scoped call rather than describe one. The answer is no, and the probes are why: the value would have to be persisted, `sys.query_store_replicas` accumulates a row per (replica, role) across failovers so a persisted id can silently re-point, and 2025 accepts a wrong id **without error** - so a stale id would not fail, it would record a forcing against a replica that does not exist. An operator reading the current ids off the live server at the moment they act is the only reliable version of this, which is what the text now tells them to do. No collector column, no storage migration, no `StorageVersion` or schema bump. -- **The Dashboard's "Collection Stopped" alert history row threw away the figure the alert was raised to report** ([#1913]) - the alert computes a cause once and shows it in three places: the toast, the email, and the history row's detail text all carry "No collector has run in 47 minutes - the SQL Agent service may be stopped or the collectors are failing." The history grid's **Value** column did not. It was built separately at the fire site and reported the constant `no recent collection` on that branch - a restatement of the metric name that discarded the 47 - and on the disabled-jobs branch it reported `2 job(s) disabled`, dropping the total that is the whole difference between "some data is still landing" and "collection has completely stopped". One incident, two descriptions, and the grid had the uninformative one. It now reads `no collection in 47m` and `2 of 6 job(s) disabled`. Deliberately still compact rather than the full sentence: the Value column is 200px and its siblings are `87% (Total CPU)` and `Session #12 running 45m`; the long form already has three homes. Both forms are now produced together by the same decision that picks the branch, so they cannot describe different branches of one incident again - which is what went wrong, not merely what could. - - **The em dash from [#1846]/[#1881] is deliberately NOT propagated here, and the code now says why.** This issue was filed during [#1881] on the belief that the Dashboard's grid renders `0.00` for state-only metrics the way Lite's and Darling's did. It does not, and it cannot: the Dashboard's alert history is **string-typed end to end** - the fire site passes display text, `JsonAlertHistoryStore` keeps it as a string and discards `AlertHistoryRecord`'s numeric slots entirely, and the grid binds that string. There is no double on the path, so there was never a `0.00`. The em dash exists in the other two apps because their `current_value` columns are NOT NULL **doubles**, so a metric whose value is a state ("Blocking and Deadlock") has nowhere to put it and stores a 0; the Dashboard keeps the words and shows them. Routing it through the shared `FormatHistoryValue` would mean inventing a numeric column in order to **replace readable text with a dash** - a regression, not a parity fix. `AlertMetricClassifier` now documents that boundary at the members it applies to, and a test suite pins the string-typed contract so a future change that starts persisting the numerics has to decide, deliberately, what the grid should then render. -- **The other half of the retention coverage invariant - chunk GEOMETRY - is now guarded against a real TimescaleDB** ([#1915]) - [#1905] proved every retention consumer's horizon outlives its source's. That is necessary and not sufficient. `drop_chunks` removes only chunks whose WHOLE range is past the cutoff, so every relation retains up to one chunk interval MORE than its horizon says, and by a different amount on each side. Get the geometry wrong and a source ends up holding history its consumer never kept, with both horizons perfectly ordered - which the gate measures as a coverage shortfall and, since [#1877], answers by STOPPING a purge that was running correctly on a healthy store, in a way that never self-releases. - - **A live test now walks every pair and sorts it into the only two arguments that make one safe.** IDENTICAL GRIDS: TimescaleDB derives chunk boundaries deterministically from the epoch, so two relations sharing a chunk interval share boundaries, and since the consumer's cutoff is older it keeps a superset of the source's chunks. ARITHMETIC: when the grids differ there is no superset to appeal to, so the consumer's horizon must reach the source's horizon PLUS one of the source's chunks. Both are load-bearing and neither implies the other - the corrected Query Store chain is carried only by the first (10-day chunks on both sides, where the arithmetic would demand 17 days against the 10 it has), and the raw tiers only by the second. The counts on each side are pinned, so a walk that quietly stopped judging fails instead of passing over nothing, and the test finishes by re-chunking an aggregate finer than its source and requiring its own judgement to refuse it. - - **The engine rule this rests on was measured, and it is not the one that gets repeated.** A materialization hypertable's chunk interval is **10x the RAW hypertable's** - 1 day of raw gives 10 days, 7 gives 70 - uniformly, at every depth of the hierarchy and regardless of bucket width. It is NOT "10x the bucket width" (a 1-hour aggregate would then be 10 hours; it is not) and it does NOT compound down a chain (the three-level day-grain daily reads the same multiple as its parents, not ten times theirs). [#1915] was filed describing it as a flat 10 days and prescribing "the consumer's chunk interval must be at least the source's"; building it showed that predicate is wrong in both directions, since the pair it most needs to protect has EQUAL chunks and is safe for a reason that comparison cannot express. Verified on PostgreSQL 18.4 / TimescaleDB 2.28.1 by moving the raw chunk interval and re-reading every relation. One residual is filed at [#1925]: the check reads the CONFIGURED interval, so a future release that NARROWED one would leave older, wider chunks on disk it cannot see - nothing narrows today, and widening is the direction it already judges conservatively. - -- **The retention ordering that keeps purges from stopping is now derived from the policy list, not asserted against whichever pairs happened to exist** ([#1905]) - every retention policy waits on its consumers covering it, so a consumer that expired FIRST could never cover its source and would hold that source's purge forever. [#1877] raised what an inversion costs: a positively measured shortfall now RE-HOLDS a policy that is already armed, so an inverted pair would stop a purge on a completely healthy store - and because holding lets the source grow deeper, the gap widens instead of closing and it would never self-release. That is a latch, and the thing standing between it and the fleet was three hand-written comparisons over the pairs that existed the day they were written. A policy added to a tier that has none today - the composer-grain dailies are kept indefinitely and carry none - would have been green in CI. - - **The list is now a declaration rather than a local, and the invariant is a walk over it.** `EnsureRetentionPoliciesAsync` built its policy list inside the method, so nothing outside could enumerate it. It is now `TimescaleSupport.RetentionPolicies`, declared beside the coverage map it derives from and iterated by the sweep, and the ordering test walks every (source, consumer) pair: a consumer with its own policy must outlive the source, a consumer with NO policy is kept indefinitely and always satisfies, and a consumer that IS the source is the [#1757] leaf rule, where what has to hold is that the tier outlives the 30-day baseline window its real consumer reads. Pairs actually compared are counted against a floor, so a walk that quietly stopped judging - a mistyped consumer name is enough - fails instead of passing over nothing. - - **The [#1784] guard was re-pointed deliberately, and watched red on both sides of the move.** That guard is what proves the three coverage-gated raw tiers are DERIVED from `RawTierCoverage` rather than hand-copied beside it, which no behavioural test can see because a correct copy produces an identical policy set. It read the method body; it now reads the field's initializer, with comments stripped so prose quoting a relation name is not mistaken for code. Re-hardcoding the raw block turns it red before the move and after it, which is the only way to know a re-pointed structural guard still points at something. It also now asserts the sweep still iterates the hoisted list, so the property cannot become true of a field nothing reads, and a value check stands behind it for the case where someone defeats the structural guard and copies the rows WRONG. - - **Two twin pins that were missing turned up while generalizing.** Each horizon exists twice - the interval literal the policy is created with, and a `TimeSpan` the router and gates compare against - and a disagreement drops on one number while everything else reasons about another. `RawRetentionSpan` and `HourlyRetentionSpan` were pinned, `IntervalDailyRetentionSpan` had a hand-written pin, and `IntervalRetentionSpan` and `BaselineRetentionSpan` had none at all despite doc comments claiming otherwise. All five are pinned as a set now, and the doc comments say where. The other half of the safety argument is engine GEOMETRY rather than our own constants - materialization chunks are never finer than their source's, so a consumer never retains less than its source at a chunk boundary - and that half is still unguarded, filed at [#1915] because it needs the aggregates actually built where this walk is a pure unit test. - -- **Alert history stored the PostgreSQL major version as an alert's "current value", and other numbers lifted out of prose** ([#1881]) - the history stores fall back to parsing a number out of an alert's display text when the producer supplies none, and that parser scans to the first digit ANYWHERE in the string rather than to a leading number - which is right for the decorated case it was built for (`87% (Total CPU)` must yield 87) and wrong for any producer that passes a sentence. So `Store Runtime Upgrade` recorded **18** as its current value and **17** as its threshold, because its text reads "PostgreSQL 18"; `Collection Stopped` recorded a run count on one branch and elapsed minutes on the other, two different units under one metric name; `Compression Job Stuck` recorded minutes-stuck when a run had hung and nothing when the scheduler had simply stopped firing; `Server Unreachable` recorded whatever number the driver put in its error message (a TCP port, an error code); and `AG Sync Fell Behind` recorded lag seconds, redo-queue kilobytes, or - on an Availability Group somebody named `Sales2024` - part of the name. [#1846] fixed the clean half of this (metrics whose text has no digit store a 0 sentinel, and the grids render an em dash for those) and could not touch these, because its dash is gated on a stored 0. - - **Each metric now states its own answer instead of leaving it to word order.** Every self-alert whose value is a state, a version, or a prose diagnosis passes an explicit 0 and joins the shared `AlertMetricClassifier.IsStateOnly` list, so the grids render an em dash - "a column of numbers where some of the numbers are PostgreSQL version 18 is worse than a column with dashes in it". The parameter carrying that decision is **required rather than defaulted**, so a future self-alert cannot fall into the text-parsing fallback silently the way these did. Two metrics [#1846] simply missed are fixed with them: `Capture Down` ("Blocking and Deadlock" against "session running") and `Agent Not Running` (literally "Stopped" against "Running") have stored the 0 sentinel since the day they shipped and rendered it as "0.00" ever since. - - **`Store Disk Pressure` is the one metric that keeps its number, and it is now exact.** Percent-free is a genuine measurement and a genuine 0 means a FULL volume - the single reading an operator most needs to see as a number rather than a dash - so it passes its percentage explicitly instead of relying on the parser finding it, which also means the value no longer rounds to whatever the sentence displays. It stays out of the state-only list, with a test that fails if anyone adds it. - - **Two copies of this rule became one.** The expression that resolves the stored double was written out once per store, and the alert-history grid's value formatter - the code that decides whether a row shows "0.00" or an em dash - existed twice as well, the Darling Viewer's copy carrying the comment "Copied from Lite's ... so both grids read identically". They had stayed identical by hand, which was tolerable while the dash was cosmetic and is not now that the producers guarantee the sentinel it keys on. Both are shared, which also let every metric's round trip be pinned all the way to the rendered string rather than only to the two inputs behind it - and pinned without a live store, since the store contributes nothing to the decision beyond that one call. All of this was invisible to the existing suites: each of these metrics already had tests for its name, its severity, its edge behavior and its message, and not one asserted the number. The deprecated Dashboard is deliberately untouched and still renders `0.00`: its grid never called the shared classifier, and it writes alert rows through its own path rather than the parser these two stores share, so it has a parallel version of the same defect rather than this one leaking through - filed as [#1913] so the divergence is on file rather than rediscovered by whoever next reads two grids that disagree. - -- **A force-plan recommendation driven by a read-only secondary replica's regression no longer silently recommends forcing on the primary** ([#1882]) - [#1850] made the Query Store analysis split by `replica_role`, so on an Availability Group with Query Store for secondary replicas enabled one query can legitimately regress on the primary AND on a secondary and arrive as two rows, each with its own best plan. It de-duped on (database, query_id) keeping the worst regression on ANY replica, which stopped the real contradiction it was fixing (two `sp_query_store_force_plan` calls naming one query with different plan ids) and deliberately left open which replica SHOULD win. **Checked against the docs rather than reasoned from the row shape**, as [#1850] asked: forcing IS scopeable per replica - `sp_query_store_force_plan` takes a fourth `@replica_group_id` argument - it always EXECUTES on the primary, and **omitting that argument targets the primary**, which is exactly what this codebase renders. So "worst regression anywhere wins" was not merely undecided, it was wrong: a secondary's 12x outranked a primary's 3x and produced a statement that forces on the primary using a read-only replica's evidence, with nothing on screen saying so. - - **The primary's row now wins when a query regressed on more than one replica**, and regression factor only breaks ties among rows of equal standing - which makes the evidence match the scope the statement already had. A regression seen only on a secondary is still recommended, because forcing runs on the primary either way and the finding is real, but it now renders with the replica NAMED and a disclosure saying the statement acts on the primary, plus the `sys.query_store_replicas` lookup and the documentation link for scoping it deliberately. That disclosure travels with the copy-paste SQL as well as the preview, since the copy-paste is the text somebody actually runs. `@replica_group_id` is still not emitted, for cause: the collector stores the replica ROLE (`replica_name`) and not the group id, so there is no correct value to put there - and the id would be the wrong thing to persist anyway, since that catalog view accumulates a row per (replica, role) across failovers and an id captured at analysis time can silently re-point. The back-out line gained a note for a documented asymmetry that is easy to miss because the argument has the same name on both procedures: **force defaults `@replica_group_id` to the primary, unforce defaults it to the local replica**, so a back-out run anywhere but the primary is not the inverse of the force it reads like. **Zero change on any server without replica attribution** - every standalone server, every non-AG server, everything below SQL Server 2022, and the deprecated Dashboard's own drill-down, which has no replica column at all - and the byte-for-byte render golden still pins exactly that rendering. Emitting a genuinely scoped call rather than describing how to write one needs the collector to store `replica_group_id`, which raises three questions the documentation cannot answer (whether SQL Server 2022's build of the proc even accepts the argument, whether a named-argument call may skip the one ahead of it, and whether an id that re-points across failovers is safe to persist at all) - filed as [#1914] with the probes to settle each. -- **Query Store charts no longer draw a bar outside the range you asked for, or drop the newest one** ([#1892]) - [#1841] moved each Query Store bar to the hour the work RAN rather than the hour it was last collected, which was the right axis but left the window still being filtered on `collection_time`. Once those two stopped being the same instant the chart and its own range disagreed at both edges, in opposite directions: an interval that STARTED before the range but whose closing fetch landed inside it passed the filter and then drew a bar dated **before the range began**, and the range's own final interval - still open when the range ended, so collected afterwards - failed the filter and **vanished**, which is the collection lag [#1841] set out to remove reappearing one layer down as a missing bar instead of a late one. Both the hourly slicer and the duration trend now window on the same expression they key on, in Lite and the Darling Viewer alike. A window is therefore answered in terms of when the work ran, end to end; an interval straddling an edge belongs to the range its start falls in, and contributes its whole total there exactly as it already did everywhere in the range's interior. - - **A bare `collection_time` floor stays, and it is not a filter.** `query_store_stats` is a TimescaleDB hypertable partitioned on `collection_time` and Lite's view unions the live table with the parquet archive, so without a predicate on that column Postgres must open and decompress every chunk and DuckDB cannot skip archive row groups. An interval is always collected after it starts, so `COALESCE(interval_start_time_utc, collection_time) >= start` already implies `collection_time >= start` - the floor prunes without ever deciding membership, the same shape and reasoning as the drill-down collector's `last_execution_time` bound. There is deliberately **no upper twin**: `collection_time` can exceed the interval start without bound when the collector was down as the interval closed, so a ceiling would silently drop exactly the rows a restarted collector just caught up on. Verified against a real PostgreSQL 18.4 + TimescaleDB 2.28.1 store, seeding an interval on each side of a window edge. - -- **The Add Server dialogs now size themselves to the monitor they are actually on** ([#1891]) - [#1828]/[#1829] stopped the dialog growing its footer off the bottom of the screen by clamping against `SystemParameters.WorkArea`, which always reports the **primary** monitor whatever monitor the window is on. On a secondary monitor shorter than the primary that reproduced the very bug being fixed: the dialog was allowed to grow to the primary's height and the buttons went off-screen again. The clamp and the height cap now resolve the actual monitor (`MonitorFromWindow` + `GetMonitorInfo`, converted from physical pixels to WPF units through the window's own presentation source, since an unconverted work area on a 150%-scaled screen would be a worse bug than the one being fixed) and re-evaluate when the dialog is dragged between monitors - the old cap was a one-shot XAML binding read once at load, so a dialog moved to a shorter screen kept the taller screen's cap forever. Every path that cannot get a real answer falls back to the old behaviour, so a single-monitor setup is unchanged. Win32 rather than WinForms because neither app references WinForms and one lookup is not worth a second UI framework; the logic is shared between Lite and the Darling Viewer so the twins cannot drift. - -- **Lite's MCP `get_alert_settings` now reports `cpu.mode`, in the same vocabulary Darling uses** ([#1911]) - **a new key for MCP clients reading Lite.** [#1895] aligned the two apps' alert-settings key NAMES and deliberately stopped short of this one, because Lite did not report the CPU mode at all and adding it as the raw enum name (`Total`/`SqlOnly`) would have traded a key-level mismatch for a value-level one - Darling has emitted `"sql"`/`"total"` since its store schema was written, and its `update_alert_settings` validates against exactly those two, so an enum name would be rejected outright. Darling's spelling wins as the older public surface. An agent can now read `cpu.mode` from either app, compare the answers, and write either one back. - -- **A retention policy whose coverage list GROWS is now re-held, instead of purging past the new tier forever** ([#1877]) - the arming gate creates every retention policy paused and arms it only once every rollup beneath it already reaches as far back as the source does. It could only ever ARM. `add_retention_policy(if_not_exists => true)` returns -1 for a policy the store already has, so nothing paused one whose COVERAGE LIST had grown under it - and [#1869] was the first build to hand a new consumer to a gate that stores had already armed. Such a store logged `HELD PAUSED` for the interval layer on every start and kept purging it anyway. Nothing a read could reach was lost, because every window older than the new tier's floor routes to a tier that holds it; what was lost is reachable DEPTH, since the new day-grain daily could then only ever be backfilled as deep as the 7-day interval layer still held rather than as deep as the store's own history. - - **The fix is a three-valued coverage verdict, and the third value is the whole of it.** The obvious form - "unsafe implies disarm" - is materially worse than the gap it closes, because the probe is deliberately fail-closed: any exception, timeout or transient failure answers "not safe". Under an arm-only gate that costs nothing, but a disarm-capable gate reading the same bool would let one bad probe on a busy store stop purging across every tier at once, and disk would grow until somebody noticed - trading a bounded depth cap for an unbounded disk risk. The probe now answers COVERED, SHORT or UNKNOWN, and only a positive SHORT measurement may stop a policy that is already running. UNKNOWN leaves the policy in whatever state it is already in: a newly created one stays paused exactly as before, and one this store already armed keeps purging. A consumer that exists and holds no rows is a SHORT measurement rather than an unknown - which is precisely the state a newly-added consumer is born in, and is what makes the re-hold fire on the upgrade it was written for. - - **Release is the existing arming path, unchanged, so this is a gate and not a latch.** A re-held policy arms itself on the first start after `--backfill-rollups` carries the new consumer past the horizon, with no manual step, exactly as a first-time hold has always released. The catalog sweep's own drop still reads the same verdict through the same map, so the two purge paths cannot judge one drop differently ([#1784]) - and for the three raw tiers this removes a case where they already could, since an armed policy used to keep dropping the very chunks the sweep was refusing to touch. The per-policy warning now names the specific tier that is short instead of the whole coverage list, and the startup summary reports how many policies were left as-is because their coverage could not be read at all. Two narrower things are filed rather than folded in: the ordering this now depends on is still asserted by hand rather than derived from the policy list, so a future policy on a daily tier could reintroduce the risk from the other side ([#1905]), and a re-hold still reads identically to a first-time hold in the log even though only one of them stopped something that was running ([#1906]). - -- **A cross-database lock now names the database the contended object actually lives in, from both blocking collectors** ([#1893]) - [#1876] made the two collectors' `contentious_object` labels agree at the incident identity, and corrected the `blocked_process_report` sentinel to name the lock RESOURCE's database rather than the blocked session's. The DMV snapshot side could not follow: its normalization runs in C# against a stored row whose only database name is `database_name`, which the collector writes as the blocked SESSION's database. A cross-database lock is held in one database by a session running in another, so for exactly that case the two sides still disagreed - the report row said `database: StackOverflow2013` where the snapshot row said `database: master` - and one contended object still raised two alerts. - - **The resource's database id comes from `sys.dm_os_waiting_tasks.resource_description`, not from the wait resource.** MS Learn documents every lock resource type's description as carrying a `dbid=` token - `keylock`, `pagelock`, `ridlock`, `objectlock`, `databaselock`, `filelock`, `extentlock`, `applicationlock`, `metadatalock`, `hobtlock` and `allocunitlock` - so a single parse covers every lock shape, where splitting `wait_resource` positionally would need a branch per resource type and would still miss the ones whose layout differs. `DB_NAME()` then runs in the same query, which is the whole reason this had to move server-side: the id was always sitting in the row, but nothing downstream could turn it into a name. Verified live on SQL Server 2022 against a genuine cross-database KEY lock, whose description reads `keylock hobtid=72057594045726720 dbid=11 id=lock... mode=X associatedObjectId=...`; the same blocking pair produced `Unresolved: key lock, database: pm1893_res` from both collectors, byte for byte, where before the fix the snapshot row carried the raw `KEY: 11:72057594045726720 (8194443284a0)` and could only ever have been named with the session's database. - - **The snapshot sweep deliberately still does NOT resolve the object behind a KEY, PAGE or RID lock.** That is the per-database lookup the report side needs a server-side cursor and [#1865]'s permission screen to do safely, and this collector runs on a far tighter cadence - naming the database is a string parse, resolving the object is a cross-database metadata read per row. Only the naming changed. The parse is restricted to `LCK_%` waits that have a wait resource, so latch and `RESOURCE_SEMAPHORE` rows - whose `resource_description` is a bare `::` or a latch class, with no `dbid=` to read - keep byte-identically what they had; and where no `dbid=` is found the row keeps its old raw value rather than being relabelled with a database nobody verified. - - **Query text only - no schema moved**, so no upgrade-folder work and no store migration in any of the three stores; the collector writes the same 19 columns with a better value in one of them, pinned by a test. The deprecated Dashboard's hand-maintained twin (`install/56_collect_dmv_blocking_snapshot.sql`) takes the identical change, guarded by a test that reads the file, so the SQL Server store cannot drift from Lite's and Darling's. This is the **third and final** blocking-fingerprint transition in this release, after [#1865] and [#1876]: DMV-sourced incidents for cross-database locks re-fire once against their new key. Consolidating all three into one upgrade was deliberate. One narrower difference remains and is filed rather than folded in ([#1898]): [#1865]'s trailing reason (`(no metadata access)`, `(page reallocated)`) exists only on the report side, so when it is present the two labels still differ by that suffix - dropping it from the identity while keeping it on the row is a product decision, not a tweak. - -- **CI's gated-live PostgreSQL now runs the worker sizing the product ships, so TimescaleDB's background jobs can actually launch** ([#1888]) - the throwaway cluster in `build.yml`'s `darling-pg` job and its `nightly.yml` mirror is stood up by shell rather than by the product, and it appended only three settings: the TimescaleDB preload, the port, and the loopback bind. So the entire gated-live suite ran at PostgreSQL's default `max_worker_processes = 8` with TimescaleDB's default `timescaledb.max_background_workers = 16` - **the exact under-provisioned shape the product exists to avoid**. `DarlingManagedPostgres.BuildWorkerSizingConfAppend` writes these on every managed start precisely because 8 slots cannot launch the per-hypertable compression, retention and continuous-aggregate policy jobs; its comment records the live capture that drove it (21 job failures against 7 successes, with "failed to start a background worker" storms in the postmaster log). Both workflows now append the product's own derivation from the live hypertable count (`TimescaleSupport.HypertableCount` = the 38-collector catalog plus `collection_log` = 39): `timescaledb.max_background_workers = HypertableCount + 2` = **41**, and `max_worker_processes = 3 + that + 8` = **52**. - - **The cost of the old value was not just coverage, it was reproducibility.** Whether a policy job launches depended on a free slot at that instant, so a test racing the scheduler passed or failed by luck - which is exactly how [#1862]'s compression flake behaved: it failed twice on CI in two unrecognizably different ways and could not be reproduced locally at all until `max_worker_processes` was raised on the same rig, after which the background run fired every time. Measured on this repo's pg-runtime, same cluster, only that setting changed: `add_compression_policy` over three eligible chunks compressed nothing in six seconds at 8 (the job never launched, `total_runs` NULL) and all three at 64. - - **Two guards, because the numbers are literals in YAML and literals derived from the catalog go stale.** A workflow cannot call into the product, and a hard-coded pin from the 27-hypertable era already went stale that way inside `DarlingManagedPostgresTests`. `CiClusterWorkerSizingTests` parses the REAL workflow files (linked into the test project's `Fixtures\`, so there is no second copy to drift) and requires both numbers to equal what the formula produces today, requires the two workflows to configure the cluster identically, and carries a mutation test that runs the same parse over an altered copy so a regex which silently stopped matching cannot pass as "no drift". Add a collector and it fails on the pull request that adds it, naming the file and the number. `CiClusterWorkerSizingLiveTests` then asks the RUNNING server what it is serving, because a conf line that never took effect is indistinguishable from one that did by inspection, and `max_worker_processes` is restart-only - it is gated on `DARLING_TEST_PGRUNTIME` as well as `DARLING_TEST_PG` (the pair CI and the nightly both set) since the product sizes workers in managed mode only, and it asserts a FLOOR rather than equality because over-provisioning races nothing. - - **Verified as its own pass, which is why this was not a one-line drive-by.** Making the scheduler work reliably for the first time is a behaviour change for the whole suite, so any test quietly relying on background jobs being unable to run would start failing. Three consecutive full gated-live runs on fresh databases against a raised cluster (PostgreSQL 18.4 + TimescaleDB 2.28.1), plus the compression- and aggregate-heavy classes over ten more fresh databases, all came back green - and the store's own accounting is the proof the scheduler was genuinely running rather than merely unblocked: `timescaledb_information.job_stats` reported **41 total runs, 41 successes, 0 failures**. [#1874] had already fixed the three compression tests that carried a dependency on the scheduler NOT running, by adding the policy and parking it before any eligible chunk exists; no further test was found to rely on it. - - **The `+ 2` in the formula turns out to be exact, not slack.** A migrated store carries precisely 41 jobs - 39 `policy_compression` (one per hypertable) plus TimescaleDB's own `policy_telemetry` and `policy_job_stat_history_retention` - against 41 background workers. What that also means is that the limit is CLUSTER-wide while the derivation is per-STORE, and CI's cluster is not the product's shape: the suite mints scratch databases alongside the main store (five databases coexisting at peak, measured), each one holding a scheduler slot. Two `out of background workers` warnings for compression jobs appeared across four full runs, in the databases' contention rather than in the main store, which still ran 41/41. They are benign - TimescaleDB retries the job on its next schedule, and no test failed in any run - but sizing a shared test cluster against a cluster-wide pool is a judgement the product's single-store formula does not answer, so it is filed as [#1899] rather than guessed at here. Deviating from the product's number is exactly the divergence this change closes. - - **The verification pass found one real defect, and it was the whole reason for doing one.** With the scheduler launching reliably, three `TimescaleSupportTests` compression tests failed the full suite every run - and passed alone, and passed as a class. The test helper that is supposed to hand them a policy that cannot fire created and parked it as TWO autocommit statements, so the scheduler could take the job in between; parking a job that has ALREADY launched does not recall the run in flight ([#1874]), and that launched run evaluates its body when it gets a worker - under full-suite load, late enough that the test's rows have landed, so it compresses chunks the test is about to count. Both halves of this change are what made it certain rather than occasional: more slots make the launch itself reliable, more parallel load widens the launch-to-execute gap. It is **not a new break** - the same test fails intermittently at the OLD worker settings (1 of 2 full runs measured, which is exactly how it stayed green on CI). Fixed with the lever the product already pulls for retention policies ([#1705]): create and park in ONE transaction, so the `bgw_job` row is invisible until it already reads `scheduled = false` and the scheduler - a separate backend - can never see it armed. Three consecutive full runs on fresh databases went from 3 failures every time to 0. - - The stale worker numbers in `Darling/README.md` (`28`/`40`, from the 27-hypertable era) are corrected to state the formula rather than a snapshot, along with the adoption line's stale `32/32` hypertable count. - -- **The same blocked object no longer raises two different alerts depending on which collector saw it** ([#1876]) - two collectors write `contentious_object` and they never agreed on how. `blocked_process_report` writes a plain `schema.object`, or an `Unresolved: key lock, database: Foo` sentinel when the lookup failed; `dmv_blocking_snapshots` writes a `QUOTENAME`'d `[schema].[object]` and, for the KEY, PAGE and RID locks it does not resolve at all, the **raw wait resource**. The alert engine merges both into one list and hashes that label into the incident's dedup key, so `dbo.Users` and `[dbo].[Users]` were two incidents for one table - and the DMV fallback, which exists to stand in for the XE session when it captured nothing (AWS RDS, or an unset blocked-process threshold), could never dedup against the reports it is standing in for. **The raw wait resource was the worse half**: `KEY: 6:72057594041991168 (8194443284a0)` carries a hobt id and a per-VALUE lock hash, so it was not merely a second identity for the object, it was a NEW identity on almost every sample - one recurring unresolvable lock rendered as an unbounded stream of one-occurrence incidents, each with its own cooldown. Those rows now collapse to the report side's sentinel and group by (lock type, database) exactly as report rows do. - - **Normalized at the incident identity - not at collection, and deliberately not where the two collectors' rows are merged.** Doing it on the read side at all is what fixes rows **already in the store**; a collector-side fix would only have helped rows gathered after the upgrade, leaving every historical DMV row still fingerprinting by lock hash. Doing it at the grouper specifically buys two things the obvious placement does not. **Coverage:** two independent producers raise blocking incidents - the live alert builders, which go through the merge, and the `top_blocking_chains` drill-down behind the analysis alerts, whose SQL `UNION`s the two collectors before anything downstream can tell them apart - and the grouper is the one point neither can bypass, which also makes a future third producer correct by default. **Not destroying evidence:** `dmv_blocking_snapshots` stores no `wait_resource` column at all, so for a lock it could not resolve, `contentious_object` IS the raw resource, hobt id and lock hash included - and that merge feeds the blocking grids and both `get_blocking` MCP tools as well as the alert path, so rewriting rows there would have fixed the fingerprint by deleting the only copy of the evidence an operator needs to chase the lock. **Grids, MCP payloads and the store keep exactly what the collector wrote; only what the ALERT groups on and names as the contended object is normalized.** - - The transform is total and idempotent rather than a DMV-only branch, since at the grouper the source is no longer knowable. It is safe to apply blind: a label already in report form has no bracket to strip and cannot match the wait-resource shape, because that pattern requires an ALL-CAPS leading token (which `Unresolved: ` is not) and a plain `schema.object` has no `": "` at all. [#1865]'s reason suffix is explicitly preserved - pinned by a test, since undoing it here would have been silent. Unquoting is a real `QUOTENAME` inverse rather than a bracket strip, so an identifier legitimately containing `]` survives instead of quietly becoming a different object, and anything that does not parse cleanly is left as stored rather than mangled. - - **The unresolved label also names the wrong database, and now names the right one.** It read `DB_NAME(database_id)` - the EVENT's database, where the blocked session was *running* - while the lookup that failed concerned the lock resource's database, where the object nobody could name actually lives. For a cross-database lock those differ, so the row named one database while [#1865]'s reason described another; verified live on SQL Server 2022 by running the same seeded row under both expressions, where the old one reported `database: master` for a KEY lock whose resource was in `StackOverflow2013`. It now reads `COALESCE(DB_NAME(resource_database_id), DB_NAME(database_id), N'unknown')`, so a resource shape that carries no database id still falls back to exactly what it printed before. - - **Fingerprint churn is bounded to the fallback path plus cross-database locks.** DMV-sourced blocking incidents inside their cooldown re-fire once against their new key; report-sourced ones are untouched except for genuinely cross-database locks, whose label changes. This release already carries one report-side transition from [#1865], so consolidating both into the same release is deliberate rather than spreading the churn over two upgrades. One narrower case survives and is filed rather than folded in ([#1893]): the DMV sentinel names the blocked session's database because that is the only database name on the row, so a genuine cross-database lock still fingerprints differently from its report-side twin - the resource database id is in the raw wait resource, but naming it needs the collector, not the read. Applies to Lite and Darling, which share the grouper, the transform and the collector. - -- **A blocked-process collector that runs per database no longer discards its own diagnostics** ([#1875]) - [#1851] let a payload collector return an optional trailing failure set after its rows, and both runners read it on the plain single-query path. The per-database path - Azure SQL DB, where the XE collectors run once per monitored database ([#1535]) - never advanced its reader to that set. That was invisible while `database_size_stats` was the only collector declaring the contract, because it never runs per database; [#1865] made `blocked_process_report` the second declarer **and** the first that does, so on Azure its batch built those rows every cycle and threw them away. The path now reads the set once per database and reports it once per cycle. - - **The reporting is what could not be copied from the plain path.** That path reads once, so it assigns the read's own note straight onto the run. This one reads N times, and neither half generalizes: N single-shot note assignments keep only the LAST database's, and N calls to the capped logger give a 200-database server 200 five-line bursts instead of one. Both are the same mistake - a per-READ answer to a per-CYCLE question - so the accumulation lives in the shared collector driver beside the read rather than being re-derived in each runner's loop body, which is the same reason [#1556] moved that loop there to begin with. Failures accumulate across the cycle's databases, compose ONE note through the identical formatter the other two channels use (so an operator still greps one wording), and the 5-line cap applies once to the whole cycle. A cycle where nothing failed composes no note at all, exactly as this path behaved before. The accumulator takes the whole per-read outcome rather than its failure list, so discarding that read's own note is one documented step instead of an omission repeated at each call site. - -- **Alerts that have no number to report show a dash instead of "0.00"** ([#1846]) - the alert-history `current_value` and `threshold_value` columns are NOT NULL doubles, and a whole family of alerts has no measurement to put in them: their value is a role (`PRIMARY`), a connection state (`DISCONNECTED`), a suspend reason, a failure reason, or the literal `resolved`. Producers hand the history store a display STRING, the store's parser yields 0 for any text carrying no digit, and both grids rendered that sentinel as `0.00` - a number the alert never made. The read side now renders an em dash for those metrics, in Lite's Alert History and the Darling Viewer's alike. **The write side is untouched**: the column stays NOT NULL and 0 stays the stored sentinel, so this corrects display for rows already collected with no migration and no re-collection. - - **The list is 19 metrics, not the 9 the issue named**, because it was enumerated from the fire sites rather than from the report. Nine were known: the seven AG metrics plus Server Unreachable and Server Restored. The other ten are every resolution notice - Darling's `BuildResolutionRecord` hardcodes `CurrentValueText: "resolved"` with a null numeric for BOTH its own self-alert recoveries (Collection Resumed, Capture Restored, Agent Restarted, Store Disk Pressure Resolved, Compression Job Recovered) and the shared engine's resolution callback (CPU Resolved, Blocking Cleared, Blocking Wait Cleared, Deadlocks Cleared, Poison Waits Cleared, Long-Running Queries Cleared, tempdb Space Resolved, Volume Free Space Resolved, Long-Running Jobs Cleared). Those reach alert history only in Darling, whose grid therefore carried most of the `0.00` rows. Resolutions are matched through the existing shared `AlertMetricClassifier.IsResolution` rather than listed by name, so a future recovery metric renders correctly without an edit, and the predicate lives in `PerformanceMonitor.Common` beside it rather than being hand-mirrored into two formatters - which is the drift that class was created to stop. - - **The dash is gated on a stored 0, never on the metric name alone**, and that gate is load-bearing rather than defensive bookkeeping. The value parser scans to the first digit ANYWHERE in the text, so several of these metrics do not reliably store 0 at all: `AG Sync Fell Behind` spells its lag seconds into its prose, and any of them can pick a digit out of an object name (`SQL01`, `Sales2024`). Those rows keep rendering whatever was parsed, and a metric that starts supplying a real numeric later is shown rather than hidden. Three Darling self-alerts are deliberately EXCLUDED for the same reason (`Collection Stopped`, `Store Runtime Upgrade`, `Compression Job Stuck`, tracked as [#1881]) - and `Store Disk Pressure` most of all, whose parsed value is percent-free, where a genuine 0 means a full volume. A measurement that happens to be zero is not a missing value: `Blocking Detected` at 0 still reads `0`, and both suites pin that separately from the dash. - -- **Plan-regression analysis stopped silently discarding one replica's Query Store rows on an Availability Group** ([#1850]) - four analysis-layer dedups predate [#1841] tier 1 and kept a key NARROWER than the read side's: `(database_name, query_id, plan_id, runtime_stats_interval_id, first_execution_time)`, with no `replica_role`. `sys.query_store_runtime_stats` is keyed by (plan_id, interval, execution_type, replica_group), and on a SQL Server 2022+ AG with Query Store for secondary replicas enabled the primary holds ONE shared Query Store carrying every replica's rows - so two rows differing only in `replica_role` are distinct legitimate work, and the `rn = 1` filter was not de-duplicating them, it was DROPPING one. That is an under-count, which is worse than the double-count the dedup exists to fix: a double-count is visible in the number, a dropped row is silent. [#1845] used exactly this reasoning to put `replica_role` in the read-side key; these four are the same defect one layer down. - - **Split rather than blended**, so the numbers are both complete and readable: `replica_role` is carried through the downstream grouping and the ranking, a regression is measured WITHIN a replica (this replica's current plan against the best plan this replica has run, never a cross-replica comparison of two different workloads), and it is exposed in both drill-down row shapes so an operator can tell which replica regressed. Adding it to the dedup alone would have made the totals complete while blending primary and secondary workload into one indistinguishable number. The drill-down field flows to the MCP payload and to email/webhook/toast detail automatically, since those renderers enumerate every field. - - **The joins use `IS NOT DISTINCT FROM`, never `=`**, and that is the whole risk of this change rather than a detail: `replica_role` is NULL on every standalone server, every non-AG server and everything below SQL Server 2022, and `NULL = NULL` is UNKNOWN - an equi-join would have matched nothing and silently disabled plan-regression detection for the overwhelming majority of installs. Verified live rather than by string pin, against a real DuckDB and a real PostgreSQL 18.4 + TimescaleDB, with a NULL-replica arm in both suites for exactly that case. The AG arm was watched RED first: two replicas running the same two plans over the same two intervals, the old key reporting **one** offender at **3x** where the truth is **two** offenders and **12x** - dropping the primary did not merely lose a row, it reported the secondary's milder regression as the server's worst. One downstream consequence is fixed mechanically: the force-plan recommendation reads a fixed set of keys, so two rows for one query would have rendered two `sp_query_store_force_plan` calls naming the same query with different plan ids, and it now keeps the worst one per query. Which replica's regression SHOULD drive that recommendation is a product question, tracked as [#1882]. Zero effect on any non-AG server. - -- **A database or server whose name contains an underscore no longer renders with the underscore missing and a stray Alt key attached to it** ([#1857]) - WPF reads a single `_` in a control's Content as an access-key marker: it removes the character from the rendered label and claims an Alt key. That applies to BOUND content exactly as it does to literals, which [#1847] fixed only for the literals. So a database called `Prod_Reporting` rendered in the **Excluded Databases** picker as `ProdReporting` with the R underlined - a name matching nothing the user typed, in the one dialog whose entire job is picking databases by name - and a list of such names registered a pile of duplicate keys, which degrades Alt+key from "press this" to "cycle focus". Server names did the same in the Viewer's **Manage Tags** picker. The bound labels now render through a `TextBlock`, whose `Text` does not parse access keys, in **Lite, the Darling Viewer and the deprecated Dashboard** (a rendering bug, so the deprecated app gets it too). The `Foreground` binding is restated on the TextBlock because the themes carry an implicit `TextBlock` style that would otherwise win and lose the greyed-out styling on stale rows. - - **A repo-wide sweep for the same shape found three more sites the issue had not.** The **Recommendations** section headers are Expanders bound to a data-derived string that falls back to the raw fact key when a finding has no advice prose, so `SOS_SCHEDULER_YIELD - CRITICAL` had been rendering as `SOSSCHEDULER_YIELD - CRITICAL` and binding Alt+S, in all three apps; they now render through a `HeaderTemplate`. The Viewer's **tag-assign submenu** builds a `MenuItem` per tag straight from the tag name, so a tag named `prod_east` read as `prodeast` and claimed Alt+E; it now doubles the underscores, WPF's escape for a literal one, matching the guard the wait-type menu headers already used. Which controls actually parse access keys was settled by **running them rather than reading about them** - a probe that instantiates each control, applies its template and reads the registered key off the resulting `AccessText`: **Button, CheckBox, RadioButton, Label, ToggleButton, GroupBox, Expander, TabItem, MenuItem and DataGridColumnHeader parse; ComboBoxItem, ListBoxItem, TreeViewItem and plain string items in a ListBox or ComboBox do not**, which is what clears the several combo and list bindings that carry server and database names. Only the FIRST single underscore in a string becomes the key; the rest render literally, which is why the mangling was easy to miss on a name with two. -- **Lite's data migration no longer leaves your real store in the folder `Setup.exe` deletes when an empty one beat it to the new location** ([#1842] review) - [#1832]'s migration refuses to overwrite anything already at the destination, which is right, but it decided "already there" from bare existence - and that cannot tell a finished migration from a placeholder created after a failed one. The gap was reachable from a single transient lock. If `monitor.duckdb` failed to move once, the app opened DuckDB at the new root and a fresh EMPTY database appeared there the same session; from the next launch on, the new root looked complete and the real multi-gigabyte store was never looked at again. The likelier variant was `config\`: startup runs `ConfigSeeder` regardless of whether the migration failed, so a failed `config\` move left a seeded stand-in - defaults, no `settings.json` - permanently occupying the target while the user's real alert thresholds and SMTP settings sat in the install directory waiting for the next installer run to delete them. **Both now end with the data safe.** An EMPTY directory or a zero-byte file at the target is recognized as the placeholder it is and the real artifact moves over it. Anything holding actual content stays live and untouched, and the legacy copy is moved out of the install directory into `recovered-from-install-dir\` under the data root, where `Setup.exe` cannot reach it - logged by name, with `DATA-MOVED.txt` pointing at it. The same treatment now applies when the new root is already complete and the old one still holds artifacts: those are quarantined rather than left behind with a "delete this by hand" note, and quarantined rather than MERGED, so a stale `archive\` from an older install cannot quietly become the live one. **Deciding which store you want is still yours** - size is no evidence of intent, so nothing swaps a bigger legacy store in for a live one; it just stops being somewhere deletable while you decide. - -- **Lite's new "Total blocked wait" box now grays out, resets, and shows up in the alert preview like every other threshold** ([#1840] review) - [#1839] wired `AlertBlockingWaitSecondsBox` into load, save and JSON export but not into the three sibling call sites every other alert box gets, so in Lite (not Darling, which got all of them) unchecking **Alerts Enabled** left this one field editable, **Restore Defaults** left a stale value in it, and the "Will alert when:" preview silently omitted the gate even when it was configured and on. Writing the drift guard for it turned up a second live instance of exactly the same miss - `AlertLongRunningQueryMaxResultsBox`, wired into neither in Lite and into both in Darling - fixed alongside. Both apps' settings windows are now pinned by a test that fails if a threshold box is loaded without also being gated and reset, which is the half of "add a threshold box" that fails silently. - -- **Lite and Darling's MCP `get_alert_settings` now use the same key names for the same fields** ([#1840] review) - **breaking for MCP clients reading Lite's payload.** An agent or script written against one app's schema and pointed at the other saw different key names for identical concepts. Lite now matches Darling's existing spelling: `blocking.threshold_count` becomes `blocking.count_threshold`, `deadlocks.threshold` becomes `deadlocks.count_threshold`, and the top-level `notifications_enabled` becomes `alerts_enabled`. That last one was not merely different but actively confusing - Darling spells `notifications_enabled` something else entirely (the analysis section's own toggle), so the same key meant two things depending on which app answered. Darling's names were chosen because they predate Lite's and its `update_alert_settings` already accepts them on the way in, so a read-modify-write round-trips. - -- **A grouped number in an alert's display text is no longer truncated to its first digit group** ([#1834] review) - the history stores' text fallback scanned digits and the culture's decimal separator but not its GROUP separator, so an en-US `"1,234"` recorded 1 - and under de-DE, where the group separator IS `.`, an ordinary `"1.234"` did the same. No alert producer formats with grouping today, but the alert code right beside it does (`AlertContextBuilders` fills its detail fields with `"N0"`), and a plausible-looking wrong number is harder to notice than the silent `0` this fallback was written to eliminate ([#1830]). Whole groups are now understood, strictly: `"1,234"` reads as 1234 while `"1,5"`, `"55,66"` and a comma-joined list like `"1,2,3"` stop at the first run exactly as before. The strictness matters because .NET's own `AllowThousands` does not validate group placement - `double.Parse("1,2,3", en-US)` returns 123 - so trusting the framework here would have fused any comma-separated list into one number. - -- **Installing Darling over a service that was re-homed to a domain account no longer silently re-ACLs the config around the wrong principal** ([#1824] review) - the installer reads the service's real logon account before hardening `darling.json`, precisely so an upgrade cannot strip the grant of a domain account or gMSA an operator moved the service to. But the read's `catch` was silent, so a WMI failure fell back to assuming the default virtual account and reproduced the exact lockout the check exists to prevent, with nothing on screen to explain it. `sc.exe qc` is now tried as a non-WMI second opinion so a broken WMI repository alone cannot derail an upgrade, and if the account still cannot be determined the behavior splits by path: a FRESH install warns and proceeds (the default is provably right - `sc create` just set it), while an UPGRADE stops with an actionable error rather than ACLing a principal it had to guess. - -- **The documentation stops recommending `SQLAgentReaderRole` in the four places [#1823] missed** ([#1826] review) - that release established that the role gates the `sp_help_job*` procedures this product never calls and grants no `SELECT` on the tables every Agent collector actually reads, then corrected the main grant blocks and left the rest contradicting them. The main README's FinOps section no longer lists the role among "the grants above"; the Darling README stops describing `HasMsdbAccess` as tied to it (it is exactly `HAS_DBACCESS('msdb')`, any access at all) and its troubleshooting entry stops telling operators to grant the role for job alerts; Lite's failed-job skip path now names the same remedy Darling and the deprecated Dashboard already do; and the deprecated Dashboard's own least-privilege script - which still handed out the role in copy-pasteable SQL - now grants the msdb job tables directly. - -- **A blocked-process row that says `Unresolved` now says WHY it is unresolved** ([#1865]) - the blocked-process collector resolves the contended object from the lock's wait resource (KEY through `sys.partitions`, PAGE/RID through `sys.dm_db_page_info`), and when that lookup failed the row still landed, labelled `Unresolved: key lock, database: Foo`. So the failure was visible - which is why [#1851] classed it as a gap rather than a swallow - but the reason never was, and the two reasons call for opposite responses: **"the login has no metadata access to that database" is a permission fix, and "the page was reallocated between the event firing and the read" is normal and nobody's to fix.** The label now carries the short cause - `Unresolved: page lock, database: Foo (page reallocated)` - so the grid row answers the question where it is being asked. A row nobody ever tried to resolve, like an `OBJECT` lock, is labelled byte-for-byte as it was before. - - **The screen is the design, and the reason this was not a mechanical adoption of [#1851]'s channel.** These cursors run per contended database on EVERY blocked-process cycle, and the common cause is a least-privilege login that will still be least-privilege tomorrow, so routing it through the probe-failure channel unscreened would have meant a permanent note on the collection_log row plus a warning burst every five minutes for a posture that is not changing - exactly what [#1854] had to add `HAS_DBACCESS` to `query_store`'s enumeration to stop. So the permission cause is **screened before it is attempted and reported through the channel zero times**, not once: `HAS_DBACCESS` is now checked at both resolution sites, and a database that fails it is stamped and skipped rather than tried. The guard that was already there was never a screen - verified on SQL Server 2022 against a login with no user in the target database, `DB_NAME()` still returns the name and `DATABASEPROPERTYEX` still reports `ONLINE`, so the cross-database lookup was attempted and raised error 916 on every cycle, forever. Only the transient cause rides the channel, tagged with its own name (`PAGE/RID lock resolution (page reallocated): …`), because an unpredictable failure is information rather than noise. Two smaller reasons ride along for rows that previously just said `Unresolved` with no explanation available anywhere: a database dropped or offline between the event and the read reads `(database unavailable)`, and on SQL Server 2016 and 2017 - where `sys.dm_db_page_info` does not exist at all - every page and RID row now reads `(page lookup needs sql server 2019)` instead of leaving an operator to discover that the engine simply cannot answer. - - **There is deliberately no server-level permission screen**, though `sys.dm_db_page_info` is gated by one (`VIEW SERVER STATE`; `VIEW SERVER PERFORMANCE STATE` on 2022+) and adding it looks obviously right. It would be a branch that never fires: the `sys.dm_xe_session_targets` read this same batch opens with needs that permission too, so a login without it fails the FIRST statement with error 297 and never reaches the page loop - verified, along with the finding that decided the screen's shape, that a login holding `VIEW SERVER STATE` but having no user in the contended database gets 916 from `sys.dm_db_page_info` rather than a permission-class denial, which is why both sites needed the SAME screen rather than different ones. A test pins the absence so the appealing dead branch does not get added back. Both handlers also moved OUT of their nested dynamic SQL, which is a real robustness gain rather than tidying: a `TRY`/`CATCH` written inside a dynamic batch cannot catch that batch's own compile-time failure, and compile time is exactly when both of these fail - a cross-database reference the login cannot bind, and an `Invalid object name` on `sys.dm_db_page_info` before 2019. The page lookup deliberately STAYS in dynamic SQL even though nothing is spliced into it, for that same 2016/2017 reason. - - **One consequence worth knowing before you take the update:** the label is hashed into the blocking alert's dedup fingerprint (`IncidentGrouping` builds the incident identity from it), so unresolved-blocking incidents currently inside their cooldown will re-fire once against their new key, and historical rows keep the old one. Nothing truncates - every store declares the column at `nvarchar(4000)` or unbounded, and the collector's own staging column is `nvarchar(4000)`. Applies to Lite and Darling, which share the collector definition; live-verified on SQL Server 2022 across four scenarios (healthy, resolvable, screened denial, reallocated page). Two adjacent gaps this surfaced are filed rather than folded in: the trailing failure set goes unread on Azure SQL DB, which runs this collector per database, a path that reads through its own contract ([#1875]), and `blocked_process_report` and `dmv_blocking_snapshots` format `contentious_object` differently even though the alert engine merges them into one fingerprinted list ([#1876]). - -- **A database whose file space `database_size_stats` could not read is now named on the collection-log row instead of vanishing** ([#1851]) - the collector's server-side cursor probes every online database it can enter with a cross-database call, and every failure of that probe went into an empty `CATCH`. A database that was mid-restore, or that the login could not enter, therefore contributed no `used_size_mb` at all: its files still appeared, with a NULL that reads exactly like a file whose space was genuinely unreadable, under a run that reported SUCCESS and said nothing had happened. [#1837] closed the same swallow for the ENUMERATING collectors, and this one could not use that fix - an enumeration's first result set is a bare item list, so the failures can ride behind it, while here the first result set IS the payload the host stores. So the contract is now generalized to the plain path: a definition may declare that its batch returns an optional trailing `(item_name, error_text)` result set AFTER its payload, and both runners read it once the definition's own read returns, through the SAME shared reader, note composer, wording and 5-line log cap the enumeration channel uses - two channels, one thing for an operator to grep. **Collectors that do not declare it are untouched**: their reader is never advanced past the payload, which matters because a payload reader can legitimately hold result sets the definition itself consumes (`tempdb_stats` reads two), and a runner that advanced unconditionally would read one of those as failures. Declaring it does not oblige every run to produce the set either - an absent set reads as zero failures, which is what lets one flag cover `database_size_stats`' on-prem cursor and its cursor-less Azure SQL DB query without a target branch. A trailing set that cannot be read - the wrong shape, or a batch that raised an error after emitting its rows - is reported AS a probe failure rather than thrown, because the payload rows are already collected and killing a good cycle to announce a diagnostics fault trades a quiet problem for a loud unrelated one. The other empty `CATCH`es in the collectors were audited and left alone: `server_properties`, `blocked_process_report` and `query_snapshots` each leave one enrichment value NULL behind a documented fallback rather than dropping rows, and `blocked_process_report`'s already surfaces itself in the payload as an `Unresolved:` label. Applies to Lite and Darling, which share the collector definition. - -- **Collection Health's Note and Last Error now show the NEWEST message, not the alphabetically greatest one** ([#1855]) - both columns were read as `MAX(error_message)` over the trailing window, which returns the greatest STRING, not the most recent one. That was harmless for as long as every message was a stable failure-mode sentence, and the entry below broke the assumption by shipping the first note that carries a NUMBER: text does not sort like a count, so `12 item(s) failed their enumeration probe` sorts BELOW `9 item(s)`, and a collector whose probe-failure count moved cycle to cycle displayed an arbitrary earlier run's number with nothing to mark it stale. Both columns now take the message from the newest run that left one, by ranking each class of message newest-first and reading the top row. **A later clean run does not blank the note**: the column still answers "the last thing this collector reported in the window", which is what makes the `(3 of 96 runs)` qualifier mean anything - ranking on time alone would empty it the moment one cycle came back quiet. **A note still cannot surface as a failure**: when no failing run in the window carried text, the error ranking falls through to the newest row of any class, so the read re-checks the status on that row - without it, a quiet enumeration's note would be reported as the collector's last error, the exact confusion the separate Note column exists to prevent. The `last error` TIME is unchanged and still names the newest failure outright, text or not, because "when did this last fail" is about the run. The fix needed no dialect split despite there being no portable "value at the greatest timestamp" aggregate (DuckDB has `arg_max`, PostgreSQL has no `max_by`): the ranking is one byte-identical query shape on both stores, verified by running the same string against a real DuckDB and a real PostgreSQL 18.4, and it settles an exact-timestamp tie the same way on each - which the old `MAX` did not, since DuckDB compares text by bytes and PostgreSQL by the database collation, so Lite's grid and the Darling Viewer's could disagree on identical data. Applies to Lite's Collection Health grid, the Darling Viewer's, and both `get_collection_health` MCP tools with the web dashboard's table behind them. **The Darling Viewer's FLEET rollup deliberately carries no exemplar text at all**, where it previously carried the wrong-run `MAX`: its only caller counts collectors and failing bands for the status bar, no surface renders a fleet row's message, and ranking one per group turns a parallel hash aggregate into a serial sort of every row in the window - measured at 0.84s to 13.9s over a 200-server, 4-million-row store. The note COUNT, which is order-independent and cheap, stays real on both reads. - -- **The Query Store hourly and daily rollups stopped baking in the pre-dedup inflation - a Custom Views panel reaching past the raw tier no longer steps by two orders of magnitude at the boundary** ([#1849]) - [#1841] tier 1 and tier 2 corrected every RAW-tier read, and said plainly what they could not reach: `query_store_stats_hourly` had materialized `sum(execution_count)` from un-deduped rows and `query_store_stats_daily` inherited it, so a panel whose window stayed inside the raw tier read corrected numbers and a window past it read inflated ones, with a visible step where the router crossed over. No read-side edit could undo it - the duplicates are gone once materialized, and the rollup output carries no interval identity to key on. Three NEW continuous aggregates now carry the corrected numbers: `query_store_stats_interval_hourly` collapses each Query Store interval to its final snapshot with `last(execution_count, collection_time)`, and `query_store_stats_corrected_hourly` / `_corrected_daily` sum those deduped values into the SAME column names the old pair carries, so every composed panel reads them with no change. Measured on a seeded store: one interval re-collected 496 times inside an hour reports **506** where the original rollup reports **123,311**. - - **The old rollups are KEPT, and that is why a boundary still exists.** A continuous aggregate's columns cannot be ALTERed, so reshaping means DROP and recreate - and with retention active the rebuild would re-materialize from 4 days of raw and permanently destroy the 21-day hourly and indefinite daily history ([#1759]/[#1793]: materialized history is never destroyed). So the corrected rollups start EMPTY and deepen from deploy while the old pair keeps everything it already holds, and reads prefer the corrected one wherever it has actually materialized the window. **Past that point a window still reads the old inflated numbers.** The step has not been removed, it has been moved outward - and it shrinks every day as the corrected rollups deepen and the old rows age out. `--backfill-rollups` closes it outright, and it already covers the new rollups with the same disk preflight, in dependency order. - - **The raw purge cannot outrun any of it.** `query_store_stats` now has TWO rollup families reading it, so the [#1680] arming gate became an AND over both: raw's retention policy stays PAUSED until the corrected interval layer AND the original hourly each reach back over everything raw holds, and arms itself on the next start once they do. A gate that checked either one alone would have dropped raw history the corrected rollups had never seen - which is the state EVERY existing store is in the moment it takes this build. The plurality is evaluated in the reader rather than folded into SQL on purpose: `GREATEST` skips NULLs, so an empty new rollup would have vanished from the comparison and the gate would have passed on the old one alone. - - **Sizing was settled before shipping, not after** ([#1849] flags capacity as a real input; [#1581] is why). The interval layer keys on query_id/plan_id/interval, so its cardinality is near-raw: on a 600-query store at the default 5-minute `query_store` cadence, 24 hours of collection costs raw 40 MB, the interval layer 11 MB (28% of raw), and each composer-grain rollup 4.4 MB. It therefore gets its own SHORT 7-day horizon rather than inheriting the 21-day one - 79 MB against 238 MB projected, which would have made the store's intermediate dedup layer larger than its whole raw tier. Seven days rather than four because it has to outlive raw with margin or the arming gate races itself. - - **No schema migration and no viewer version gate**, deliberately: continuous aggregates are runtime TimescaleDB setup, not schema objects, so the existing `to_regclass` availability probe answers "does this store have them" and a viewer pointed at a store whose service predates this build routes to the old pair exactly as before. Bumping the store schema version for objects no migration creates would have locked viewers out of stores that are functionally identical. - - Five CAGG shapes were live-probed against PostgreSQL 18.4 + TimescaleDB 2.28.1 to settle the design, and one result is worth recording because it is in no documentation and it inverts the shape every other daily rollup in this codebase uses: **an identity-width hierarchical continuous aggregate is a LEAF**. A child whose bucket equals its parent's width creates and refreshes normally, but nothing can be built on top of it - a further aggregate fails with `time bucket function must reference the primary hypertable dimension column`. Depth is not the blocker (a plain 1h -> 1d -> 7d chain is accepted), the equal width is. So `_corrected_daily` is a SIBLING of `_corrected_hourly` sourced from the interval layer directly, where every other daily here is built on its hourly. One residual is left and filed rather than hidden: an interval whose snapshots straddle an hour boundary is counted once per collection HOUR (~2) instead of once per COLLECTION (up to 496), which is irreducible at the hourly grain and removable at the daily grain only by adding a fourth near-raw-cardinality aggregate - a costed capacity decision, written up with the live probe results and the measurements at [#1869]. - -- **The Query Store DAILY numbers stopped over-counting by roughly 2x - the tier that is kept indefinitely is now the accurate one** ([#1869]) - the corrected rollups above dedup each Query Store interval within a collection HOUR, so an interval whose snapshots straddle an hour boundary produced two rows, each holding a cumulative value, and the daily sum counted it once per hour it was collected in rather than once. That is irreducible at the hourly grain - an interval genuinely collected across two hours has to appear in both - but not at the daily grain, and the daily tier is the one kept forever, so a permanent 2x there is a permanent lie. Two more continuous aggregates remove it: `query_store_stats_interval_daily` re-dedups the interval across the whole DAY with `last(execution_count, bucket)`, and `query_store_stats_daygrain_daily` collapses that to the same composer columns the other dailies carry, so every composed panel reads it with no change. Measured on a seeded store with the shape [#1869] published: a day whose true total is **515** reads **1,013** from the hour-grain daily - **1.97x** - and the new one reads 515. - - **What it does NOT fix, because a mis-count is worth stating exactly.** An interval whose snapshots straddle MIDNIGHT is still counted once per collection DAY. It is the identical argument one grain up and equally irreducible there, but it is far smaller and the difference is structural rather than lucky: the collector fetches an interval while its `last_execution_time` keeps advancing, so a 60-minute interval is collected across an hour boundary almost always and a day boundary about once per 24 intervals - roughly a 4% over-count against the 97% removed. It is pinned by a test against a real store so the claim cannot quietly drift, and filed with the cost of the fifth aggregate that would shrink it at [#1879], along with the reason no finite dedup grain removes it outright. One upgrade note was filed the same way at [#1877] and is FIXED below in this same release: the purge gate could only arm, so a store whose interval-layer policy was already armed under [#1849] kept purging while the new level caught up. It is now re-held until the new level covers it, and running `--backfill-rollups` promptly after upgrading is what releases it. - - **Three dailies now, best-first, each rung the same rule.** The day-grain daily is a NEW aggregate that starts empty, so it wins only where it has actually materialized the window; older windows fall to the corrected daily, and older ones still to the superseded original - each step comparative, so a store that has been backfilled never silently reads the coarser number. The hourly tier is untouched: there is no better hourly to route to. - - **The purge gates go one level deeper too.** `query_store_stats_interval_hourly` now has THREE consumers, and its retention policy stays PAUSED until all three reach back over everything it holds. The third is the load-bearing one on a store taking this build, because that store has a fully caught-up interval layer and an empty day-grain one - precisely the state where a gate reading only the [#1849] pair would drop the only interval-grain copy of history the new daily has never seen. The new layer takes a 10-day horizon against the interval layer's 7 for the same reason that one takes 7 against raw's 4: a consumer that expired before its source would hold its source's purge forever. - - **Capacity was measured, not projected** ([#1581]): on the same seeded 600-query store at the default 5-minute cadence, and reproducing [#1849]'s own interval-layer figures to the row (28,800 rows / 11 MB), the new interval-grain daily costs **15,000 rows / 4.7 MB per day** - near-raw cardinality like its hourly source, but keyed on interval x DAY where that one keys on interval x HOUR, so the fourth near-raw-cardinality object is the smallest of them at **~47 MB over its 10-day horizon** against 79 MB for the layer below it. The composer-grain view above it measured byte-for-byte identical to the daily it sits beside (448 kB each, ~600 rows a day), which is why it is kept indefinitely like every other daily. No schema migration and no viewer version gate, for the same reason as [#1849]: continuous aggregates are runtime setup, so existence is the probe and a store on an older service keeps reading the corrected daily. - - The three-level chain is legal only because the new dedup level WIDENS (1 hour to 1 day) - the leaf rule above forbids building on an identity-width child, not on depth - and both levels were live-probed accepted on PostgreSQL 18.4 + TimescaleDB 2.28.1 along with their refresh and retention policies. `--backfill-rollups` covers both automatically and now orders its targets by DEPTH from raw rather than by a hierarchical flag: the flag can only sort a chain two levels tall, and refreshing a rollup before its source materializes nothing while reporting success. - -- **Query Store charts now put an interval's work in the hour it RAN, and the duration trend stopped overstating** ([#1841], tier 2) - the collector stores the real interval identity at last: `runtime_stats_interval_id` and `interval_start_time_utc`, read straight from `sys.query_store_runtime_stats_interval` and converted to UTC at collection. Two things follow. **Every dedup now keys on the real interval** instead of the `first_execution_time` proxy tier 1 had to settle for. The proxy stays in the key beside it, because it is the only identity rows collected before this version have - but where the real id exists it also fixes what the proxy could not: a row Query Store never attributed a first execution to had NO identity under tier 1 and collapsed with every other such interval of the same plan, which under-counts. **And the bars stopped lagging.** A Query Store interval was previously drawn in the hour it was last COLLECTED, which on the default 60-minute interval is reliably one bar late, because the closing fetch lands in the cycle after the interval ends. Slicer bars are now placed at the interval's own start. The **duration-trend chart is corrected rather than excluded**, reversing tier 1's deliberate hold: each interval contributes its true final total once, at the hour it ran, so the chart neither triple-counts a growing interval nor collapses to a single zero-valued point. The premise that blocked this turned out to be false - Query Store's interval start and `first_execution_time` are both `datetimeoffset` (verified live on a server reading +00:00 while sitting at UTC-4), and the collector already normalized them to UTC before storing, so the interval clock was never the monitored server's local wall time. **Nothing needs re-collecting, and a store holding older rows does not lie**: reads split on whether a row carries the identity, with no overlap and no gap, so pre-upgrade rows keep exactly the behavior they always had and the mixture resolves itself as they age out of retention. Lite schema v49, Darling store V41, both additive and nullable. Watched failing first against real DuckDB and a real PostgreSQL 18 + TimescaleDB store - including the live proof of the one-bucket lag (a bar at 16:00 for work that ran at 15:00) and of the tier-1 under-count (5 executions reported where 11 ran). Two things remain open and are filed rather than hidden: the hourly/daily rollups still bake in the pre-dedup inflation, so a Custom Views panel reaching past the raw tier still steps at the routing boundary ([#1849], with the corrected-rollup design and the live TimescaleDB probe results that constrain it), and four analysis-layer dedups still drop a replica's rows on an Availability Group primary ([#1850]). - -- **Query Store totals no longer count the same interval once per collection - live stores were overstating hot queries by one to two orders of magnitude** ([#1841]) - Query Store's runtime-stats rows are CUMULATIVE per-interval snapshots, and the collector is incremental on `last_execution_time`, so while a query keeps running its OPEN interval is re-fetched every cycle and stored again with a growing `execution_count`. The schema has no interval-id column, and every aggregate read simply summed the raw rows - so one interval's work was counted once per collection. Measured on a real store: the same (server, database, query_id, plan_id) appeared **496 times inside a single hour bucket** with `execution_count` of 1 each time (one interval, 496 collections, its CPU and duration billed 496 times), alongside cases growing 1 -> 23 cumulatively where every re-collection re-charged the accumulated total as well. Execution-count weighting did not rescue the averages: the repeated snapshots carry DIFFERENT growing weights AND different `avg_*` values, so an open interval was weighted by the triangular sum of its own growth. Every affected read now collapses each interval to its LATEST snapshot before aggregating - keyed on `first_execution_time`, the interval identity the schema does expose - which is the same `ROW_NUMBER() ... WHERE rn = 1` shape the plan-regression analysis collectors have always used. **This corrects the numbers for data already collected**; no re-collection or migration is needed. Fixed in both SKUs: Lite's Query Store time-slice bars, top-queries grid and current-vs-baseline comparison, the Darling Viewer's equivalents plus its regressions grid and slicer overlay-on-select, the service's MCP/REST `get_query_store_top`, and the Custom Views raw route. Three things this does NOT change, each stated at the source so none of them reads as an oversight. **Bar placement still lags**: a bar is keyed on when an interval was last COLLECTED, not when it ran, which on Query Store's default 60-minute interval is reliably one bucket late - window totals are right, placement is not, and correcting it needs `first_execution_time`, which is the monitored server's LOCAL wall clock against a UTC axis. **The duration-trend chart is deliberately left un-deduped and still overstates**: deduping is right for totals and wrong for that chart, because it keeps one row per interval at the collection where the interval closed, and a 60-minute interval against a 5-minute cadence collapses twelve snapshots onto one timestamp - a 1-hour window would render a single point valued zero. Better an honestly-labelled inflated line than an empty chart, until tier 2 can place the work at the time it ran. **The Custom Views rollup is not repaired**: `query_store_stats_hourly` materialized its sums from un-deduped rows, so a panel whose window reaches past the raw tier still reads the old inflated numbers, and no read-side change can undo that - a continuous aggregate cannot contain a window function, so it needs a rebuild rather than an edit. All three are tracked on [#1841]. Two further reads are correct as they stand: the per-collection history drilldown (a raw projection with no aggregate - the "show me every snapshot" surface) and the stored-plan lookups (already latest-row-only). New round-trips execute the corrected reads against real DuckDB and real Postgres and assert the interval is counted once, each watched failing against the old queries first; the composed panel gained the first test that ever RUNS the compiler's output against Postgres. - -- **Alert History rows stored under the old "TempDB Space" metric name show their percentage again** (field find during [#1830]'s triage) - lowercasing the tempdb token across both apps' UI also changed this alert's metric_name KEY, from `TempDB Space` to `tempdb Space`, and that change knowingly left already-stored history rows carrying the old name. The Value/Threshold formatter matches the name ordinally, so those rows missed the percent case and fell through to the bare two-decimal default: a tempdb alert archived before the rename showed `87.30` where the row beside it showed `87.3%`. Both formatters (Lite and the Darling Viewer) now accept the old spelling as well. Nothing writes it any more - it is kept solely so rows already on disk format like the new ones. - -- **An enumerated collector that enumerates NOTHING now says so on its collection-log row** ([#1837], minimal core) - `query_store`, `index_object_stats`, and `database_scoped_config` list their databases first and then collect per database, and when that list comes back empty the cycle returned early and logged a bare `SUCCESS` with 0 rows. That row is byte-identical to a healthy collector whose databases were simply quiet, which is what let [#1833]'s Azure defect sit behind a green Collection Health for as long as it did: Query Store enabled nowhere, a database-scoped collector filtered down to nothing, and an ordinary idle cycle all logged the same thing. The status deliberately stays `SUCCESS` - nothing failed - but the row now carries a fixed, greppable message saying the enumeration yielded 0 items, on the column `SUCCESS` rows otherwise leave null. Both SKUs, from one shared constant so the wording cannot drift. Every health count, band, and self-alert keys on status rather than on that column, so the note is visible in the collection-log detail grid and inert everywhere else. The Collection Health treatment of that message, and the reason an enumeration can come back empty in the first place, are the entry below. - -- **Collection Health now shows what a collector that collected NOTHING reported - and an enumeration can finally say why it found nothing** ([#1837]) - the message the entry above puts on a zero-item cycle was written only to the raw collection-log detail grid, which is the surface nobody opens until they already suspect something. Two changes make "this collector has been coming back empty" a thing you can see, without turning a legitimately empty server into an alarm. - - **The Collection Health grid, the Viewer's grid, the web dashboard's table, and the `get_collection_health` MCP tool gained a Note column.** It reads the message off `SUCCESS` rows specifically - not "anything that is not an error", which would have dragged Darling's `SESSION_MISSING` rows (a real capture fault, with its own self-alert) into a column whose own tooltip says it is not an error. It carries a qualifier built from the same aggregate: `enumeration yielded 0 items ... (all 96 runs)` when every run in the trailing window came back that way, `(3 of 96 runs)` when it happens sometimes. That distinction is the point - a collector whose databases occasionally go quiet is normal, and a collector that has returned nothing for a week is the case worth a look. **The band is deliberately untouched.** A target with no user databases, no Always On groups, or nothing matching a collector's filter is legitimately empty and keeps reading HEALTHY; making "empty" a band would cry wolf on exactly those installs, and nothing in the banding reads the new columns. All three surfaces render the qualifier through one shared helper - the web table binds a `note_summary` the MCP tool composes with that same helper rather than re-deriving it in JavaScript - so the same store row cannot read three ways, and paired tests in each suite assert that two collectors identical except for the note band identically. The note shown was originally an exemplar from the window rather than guaranteed to be the newest, because the read took a `MAX` over the message text; that is corrected in the entry above ([#1855]), which lands in this same release, so the column shows the newest note on both stores. Sharpening it further - "empty AND the target HAS user databases" - needs a per-server database inventory at health-read time, which is a cross-collector join with its own staleness questions, and is tracked as [#1852]. - - **An enumerating collector can now report items it could not PROBE.** This is what makes the note say something useful rather than just "0". The on-prem `query_store` enumeration walks every online database and asks each one whether Query Store is usable through `[db].sys.sp_executesql`; every failure of that question - a database mid-restore, an Always On failover mid-cursor, a login that cannot enter it - went into an empty `CATCH`. A login that could enter no database at all therefore enumerated zero items and logged one indistinguishable `SUCCESS`, which is the same shape [#1836] found on Azure. The failures could not simply be added to the enumeration's result set, because that result set IS the item list both runners consume as database names - anything added to it would be collected FROM. So the shared enumeration contract gained an **optional second result set** of `(item_name, error_text)` rows: the driver reads it after the item list, writes the first five per-item errors to the app log with a count for any remainder, and appends a fixed summary to the collection-log note. `query_store`'s cursor now records `(database, ERROR_MESSAGE())` there instead of discarding it, so "0 items" and "0 items, and here is the access error on all fourteen of them" are finally different rows. **Enumerations that return one result set are unaffected** - no second set means no failures, no note, nothing logged - which is every other enumerating collector today. Both hosts read the enumeration through one shared method, so the item read, the failure read, and the note wording cannot drift between Lite and Darling; a malformed second result set is reported through the contract as a probe failure rather than thrown, because failing a whole collection cycle to announce a diagnostics defect would trade one invisible problem for a loud unrelated one. The same empty-`CATCH` shape in `database_size_stats` is NOT covered - it is not an enumerating collector, and giving the plain single-query path a failure channel is a separate design decision - tracked as [#1851]. - - Two corrections found while building this ride along. **`query_store`'s enumeration now skips databases the login cannot enter** - it was the one enumeration without the `HAS_DBACCESS` screen its three siblings got in [#1823], so a least-privilege install took a 916 per inaccessible database per cycle. That was invisible while the `CATCH` discarded it and would have become a permanent probe-failure note plus a warning burst every five minutes the moment these failures started being recorded, for a permission posture that is not changing; the screen keeps the new channel for the failures worth reading. And **Lite's per-run collection-log fields are now keyed by server.** A collection cycle runs the monitored servers in PARALLEL on one service instance while the collectors within a server run sequentially, but the `sql_duration_ms` / `duckdb_duration_ms` values (and now the note) were plain instance fields shared across those parallel tasks: server B's reset at the top of its run could blank server A's timings between A's write and A's collection_log read. Harmless-looking as a timing wobble, but with a note attached it meant server A's "enumeration yielded 0 items" could land on server B's row for a collector that does not even enumerate. They now live in one per-server slot, which is sufficient exactly because two collectors on one server never overlap. Darling was never exposed - its note is a return value. - -- **A Lite collector that fails before it runs no longer logs the previous collector's timings as its own** (adjacent find while fixing the above) - the per-call `sql_duration_ms` / `duckdb_duration_ms` fields were cleared inside the definition runner, but a collector can throw before the runner is ever entered: the Extended Events session ensure for `deadlocks` and `blocked_process_report` runs first, and when it fails the resulting `ERROR` row was written with whatever the previously-run collector had left in those fields. They are now cleared once per run, at the top, where every path that reaches the collection-log write passes through. - -- **Opening the Settings window after a data loss no longer deletes the webhook URLs that survived it** ([#1832]) - `App.LoadAlertSettings` read the Teams, Slack, and generic webhook secrets out of Credential Manager at the TAIL of its `settings.json` parse, behind an `if (!File.Exists(path)) return;` early exit. With `settings.json` gone - exactly the state the install-directory wipe above produced - the load returned before those four reads ever happened, so the Settings window's webhook boxes rendered EMPTY while the credentials were still sitting in the store. Clicking Save then wrote those blanks back through `SaveWebhookUrl`, which DELETES the credential when handed a blank: the secrets that had survived the data loss were destroyed by looking at the settings window afterwards. The credential reads now run FIRST and unconditionally, before the early exit, so the boxes populate from Credential Manager whether or not `settings.json` exists and Save round-trips what it displayed. The Save path is otherwise unchanged - blank still means delete, which is how you clear a webhook on purpose. A settings-file-absent test pins that all four keys are read. - -- **A tray notification when an update is available, not just a title-bar suffix** ([#1832]) - the startup update check appended "Update vX available (Help > About)" to the window title and did nothing else, and a title bar nobody reads is how installs both stayed on old builds and got "upgraded" by re-downloading `Setup.exe` - the one path that deleted their data. The check now also raises one tray balloon per launch pointing at Help > About and saying that route updates in place and keeps your data. Same check, same cadence, one more surface; the update UX is otherwise untouched. - -- **Query Store collects on Azure SQL Database** ([#1836]) - the second half of [#1833]'s finding, and the worse one: `query_store` returned zero rows on Azure SQL Database, on every cycle, since the collector existed. It is built as an enumeration - list the databases whose Query Store is usable, then collect each one - and its Azure enumeration probed every candidate through `QUOTENAME(@db) + N'.sys.sp_executesql'`, a cross-database three-part reference that Azure SQL Database rejects for **every** database, from `master` and from a user database alike. Each rejection landed in an empty `CATCH`, so the database list came back empty and the cycle logged `SUCCESS` with 0 rows, forever, with nothing anywhere saying why. Setting the server entry's Database field could not save it either: the collector would still have been asking one database for the whole instance, the wrong way. Azure SQL Database now runs `query_store` the way it runs Top Queries and its five database-scoped siblings - one connection per database (`RunsPerDatabase`) - so a logical server with ten databases collects ten databases' Query Store instead of none. Each per-database run starts with the SAME eligibility gate the on-prem enumeration probes with (`actual_state IN (1, 2, 4)` and `readonly_reason & 8 = 0`, the [#1546] and [#1558] semantics), only evaluated locally against the connected database: a database with Query Store off, errored, or read-only because it is a readable secondary costs one catalog lookup and returns no rows without ever touching the Query Store views. Both SKUs get it, from the shared definition. **The two execution shapes are built from one payload body** - the on-prem form is that same string quote-doubled to nest inside `[db].sys.sp_executesql`, and a test asserts the containment - because a 54-column payload kept in two hand-maintained copies is exactly how the two paths would stop agreeing about what the reader's column ordinals mean. Two column decisions ride along, and they go opposite ways on purpose: `plan_type` is now populated on Azure (the probe reports PRODUCTVERSION 12 there, but the engine is evergreen and the column's own documentation lists Azure SQL Database), while `replica_role` is deliberately left NULL on Azure rather than joined - `sys.query_store_runtime_stats.replica_group_id` is documented for SQL Server 2022+ and is silent on Azure SQL Database where its sibling columns are explicit, and Query Store for secondary replicas is documented as unavailable on Hyperscale, so referencing it could fail the entire payload in every database to gain an attribution that Azure may not even offer. The column set is unchanged either way - Azure emits the same typed NULL placeholder at the same position a pre-2022 box does. Finally, the 24-hour catch-up clamp ([#1556]'s answer to a stopped service coming back and asking for a 30-day backlog in one query) moved INTO the collector's cutoff computation: the host's per-database branch is shared with the deadlock and blocked-process collectors, where clamping would wrongly truncate legitimate catch-up, so the bound now travels with the collector that needs it rather than with the code path. The per-database truncation warning fires on this path too, in both SKUs. - -- **Top Procedures collects on Azure SQL Database** ([#1833]) - `procedure_stats` was one of two database-scoped collectors that never opted into the Azure per-database connection path: it ran once on the server entry's own connection - whose catalog defaults to `master` when the entry's Database field is blank - and its Azure variant's `WHERE s.database_id = DB_ID()` then matched only master's procedures. Zero user rows, logged `SUCCESS`, empty Top Procedures grid that looked healthy, while Top Queries (which does run per database) kept working right next to it - the asymmetry that fingerprinted the bug. The collector now declares `RunsPerDatabase` on Azure like `query_stats` and its five database-scoped siblings, and the per-database connection makes `DB_ID()` mean each user database in turn - exactly what that predicate was written for. Both Lite and Darling get it (shared definition). The other collector with the same gap is Query Store, which needs a per-database rework on Azure rather than a one-line override - tracked as [#1836], with the invisible-zero-rows Collection Health gap that hid both defects tracked as [#1837]. - -- **Chart x-axis timestamps now honor the Local/Server/UTC time toggle** ([#1831]) - Lite plots chart X values in server time and converts for display at render, but the ONE render surface doing the axis labels - the single shared `LabelFormatter` all three apps reach - never consulted the display mode, so every chart's bottom axis showed server time no matter what the toggle said, while grids, slicers, tooltips, and the crosshair all converted around it. Refresh and reconnect could not help: they re-plotted the same unconverted labels. The formatter now converts each label through the `UiTimeContext` hook Lite wired at startup for exactly this purpose and never used here; the Darling Viewer pre-converts its plotted X and deliberately leaves the hook at identity, so the change is a no-op there by construction - the double-conversion trap a Viewer-style port into Lite would have hit is documented at the site and pinned by test. Three companions ride along, all the same defect in different clothes: the display-mode toggle now re-plots the charts immediately (it refreshed nine grids and six slicers and zero charts, so even the fixed formatter only showed on the next data cycle); the Queries heatmap's hand-built axis labels convert like that chart's own tooltip always did; and the three history windows switch to the shared formatter, gaining both the conversion and the date-change labels. The deprecated Dashboard's startup hack that force-pinned the dropdown to ServerTime ("charts always render in server time") is retired - and its saved preference, which was written on every change and read never, is finally restored at startup. New tests pin the formatter's conversion, the Viewer's identity no-op, and the display-date boundary behavior - nothing anywhere tested the formatter before. - -- **High CPU alerts no longer record 0 as their value in Alert History** ([#1830]) - the alert fired with no numeric value, and the history stores' fallback tried to parse the display text `"87% (Total CPU)"` - which ends with a parenthesis, so the trailing-`%` trim did nothing, the parse failed, and the `: 0` arm silently stored zero for every High CPU row ever written, in Lite and Darling alike (the deprecated Dashboard stores the text and was immune). Detection, thresholds, and every notification surface were always correct - the toast, email, and webhook all carry the text - so only the audit trail and the MCP alert tool were corrupted, which is exactly what made it invisible. The engine now passes real numerics for High CPU, Blocking Detected, and Deadlocks Detected; per-event delivery carries each incident's occurrence count (and the overflow trailer's COUNT - its `"+N more incident(s)"` text is unparseable by design and was the second live instance of the same coercion); and both stores' fallback goes through a shared leading-numeric parser so a future decorated text cannot re-coin a silent 0. Parses with CurrentCulture on purpose - the producers format with it, and the reporter's own locale writes `92,5%`. Two tests that PINNED the null-numerics behavior are flipped to pin the fix, and a store round-trip now asserts the exact field case - `"87% (Total CPU)"`, no numerics - lands as 87, not 0. - -- **The Add Server dialog's buttons can no longer land off-screen when a taller auth mode is selected** ([#1828]) - the dialog combined `SizeToContent` growth with `ResizeMode="NoResize"` and no scroll, so on a 1080p work area selecting SQL auth, Service Principal, or Managed Identity grew the window past the screen bottom, taking Test Connection / Save / Cancel with it - reachable only by blind tabbing, with resize disabled as the escape hatch. Both copies (Lite and the Darling Viewer; the deprecated Dashboard's twin already had the correct shape and supplied it) now scroll the form in a star row with the status line and buttons pinned OUTSIDE the scroll, clamp the window to the work area, and allow resizing with a grip. Because `SizeToContent` growth is top-anchored, `MaxHeight` alone still let mid-session growth slide the footer past the screen edge without ever hitting the clamp - a `SizeChanged` handler pulls the window up when its bottom crosses the work area, the SizeToContent-friendly form of the Dashboard's top-pinning. The buttons also gain Alt mnemonics (the reporter looked for them) and Cancel gains `IsCancel`, so Esc closes the dialog like its sibling dialogs already did. - -- **The CDC capture-job probe no longer fires an "Invalid object name 'msdb.dbo.cdc_jobs'" error event on servers that never configured CDC** (field report; the probe shipped in [#1096]) - `msdb.dbo.cdc_jobs` is created lazily on first CDC configuration, so on a no-CDC server the query-snapshots probe raised error 208 every collection cycle. The `sp_executesql`-in-TRY/CATCH shape absorbed the *failure* correctly (collection proceeded on the text fallback, which is why nothing ever looked broken from the product's side) - but TRY/CATCH suppresses the failure, NOT the server-side `error_reported` event, so any fleet error monitoring watching those events saw a recurring "Invalid object name" attributed to the monitoring login. Proven with an Extended Events session on a cdc-less server: the old shape survives yet fires exactly one 208 event per run; an `OBJECT_ID` existence pre-guard fires none. The guard lands in the shared collector (Lite + Darling) and the deprecated Dashboard's NOC-health twin; the TRY/CATCH stays as the belt for denied SELECTs and create races, and a login without metadata visibility on the table degrades to the same text fallback it always had. - -- **The documented least-privilege monitoring grants now actually run every collector, verified live** ([#1823] field report) - a user provisioned a login exactly as the README said and got five distinct failures, all reproduced with a scratch login on SQL Server 2025 carrying only the documented grants. The msdb guidance was simply wrong: `SQLAgentReaderRole` gates the `sp_help_job*` procedures - which this product never calls - and grants NO SELECT on the base tables every Agent collector reads directly, so `running_jobs`, `job_history`, `agent_status`, and the failed-job alerts all failed with error 229 on a login holding the documented role. Both READMEs now prescribe the verified set: direct SELECT on the six msdb job tables plus EXECUTE on `agent_datetime` (a scalar function no read role covers), `CONNECT ANY DATABASE` for the per-database collectors (current and future databases, no per-database users), `VIEW ANY DEFINITION` promoted from AG-only to core - without it per-database catalog views return ZERO ROWS with no error anywhere, so a minimal login collected nothing from the index/object collectors while looking healthy - and optional `ALTER TRACE` for the default trace, documented with its real cost (not read-only; implies SHOWPLAN). Three code paths hardened to match: `database_scoped_config` now self-skips databases the login cannot enter (`HAS_DBACCESS`, the same filter `index_object_stats` and `database_size_stats` already had - on-prem only, since the probe answers falsely cross-database on Azure), error 8189 from `sys.traces` now classifies as `PERMISSIONS` instead of `ERROR` in both Lite and Darling so withholding `ALTER TRACE` is a clean supported choice, and the failed-job skip message in Darling and the deprecated Dashboard stops recommending the role that provably does not work. - -- **The managed-store credential refusal now recognizes a re-homed service account and prints the ownership fix** ([#1823] field report) - the superuser credential `pg-credential.dpapi` is refused unless OWNED by SYSTEM, Administrators, or the service account (an anti-pre-plant guard), and an account switch trips it legitimately: the file is still owned by the previous service account, because the runbook's `icacls` grants change permissions, never ownership. The error told that operator "tampered with or pre-planted ... re-initialize the data directory (destroys collected history)" - alarming, wrong for the common cause, and pointing at destroying a healthy store. It now leads with the account-switch case and its runnable two-line fix (`takeown /a` to hand ownership to Administrators, which stays trusted across any future account change, plus a direct grant on the protected file), names the actual current owner, and keeps the tamper framing for operators who did NOT change the account. The README runbook gains the same missing step, including why the sibling role credentials self-heal (one-time `discarding and regenerating` warnings, not faults) while the superuser's cannot. - -- **Darling installs re-homed to a domain account or gMSA survive upgrades and get correct remediation text** ([#1823], also [#1802]) - with `"auth": "integrated"` the service connects to monitored servers as its Log On account, and operators change that account for exactly that reason - but two paths still assumed the default virtual account. `install-darling.ps1`'s upgrade path preserves a custom Log On account, then rebuilt `darling.json`'s DACL around `NT SERVICE\PerformanceMonitor Darling` anyway, stripping the operator's grant and locking the re-homed service out of its own config on the next start; the hardening (and its printed `icacls` hint) now targets the account the service actually runs as. The service's own ACL-failure log lines had the same defect baked into their remediation - an `icacls /grant` naming the virtual account, which on a re-homed install is a fix that cannot work - and now name the running identity. The Darling README also gains the full account-switch runbook that previously lived only in an issue answer: Services.msc/`sc config` routes (gMSA included), the Windows-login grants, the one-time file re-grant that bites everyone who skips it, and the caveat that `--test-connection` runs as the console user, not the service account. - -- **The gated-live Darling test suite no longer depends on which class the runner happens to schedule first** ([#1862]) - against a fresh PostgreSQL store, `PayloadDimensionLiveTests.DimensionGc_DefersWhenAFactFloorIsUnmeasurable_ThenPrunesOnceItIs` died 3-5ms into the run with `42P01: relation "timescaledb_information.continuous_aggregates" does not exist`, and passed whenever a sibling test in its own class ran ahead of it. Nothing was wrong with the test's subject. Migration does not create the TimescaleDB extension - `PgMigrations.MigrateAsync` is deliberately engine-plain and `TimescaleSupport.TryEnableAsync` installs it at runtime - so "the store is established" was an EMERGENT property of test order: `CREATE EXTENSION` is persistent and database-wide, so the first live class to run it silently established the store for all sixty-odd that followed, and any class that read a Timescale catalog without enabling it first worked only when it was not first. **The failure therefore MOVED between runs**, landing on whichever class drew the short straw - it was also seen as a `DarlingAnomalyBaselineTests` count mismatch - which is the expensive shape: it reads as if the change under test broke something it never touched, and gets re-run rather than diagnosed. CI could not catch it either, because it builds a throwaway cluster per run and so sampled the scheduling lottery once. The `live-postgres` collection now has a collection fixture that migrates the store and enables TimescaleDB ONCE, before the first class in it runs; xUnit awaits that ahead of every member, so the guarantee holds whatever order the runner picks. Migration runs BEFORE the extension deliberately, matching what the service does on every start and preserving the fresh-store path that `V23`'s `pg_extension` guard and `TimescaleSupportTests` both depend on. Two masked ordering bugs behind it were fixed as well: a continuous aggregate cannot be built over a heap, so the payload-dimension helper that creates the rollups now converts the hypertables first the way every other caller already did (its absence surfaced three steps downstream as a missing `query_stats_hourly`), and the anomaly-baseline test now asserts that all nine baseline relations EXIST rather than that the call CREATED nine, which was a count of shared mutable store state and legitimately answered eight whenever a sibling's aggregate was still standing. Verified by running the full gated-live suite three consecutive times on fresh databases and by running each of the 80 live classes alone against its own brand-new store. Test-suite only - no shipped code changed. - -- **The compression tests no longer race the background job that `add_compression_policy` starts on its own** ([#1889]) - pre-existing on `dev`, found while verifying the entry above. `add_compression_policy` creates its job SCHEDULED with no `initial_start`, and TimescaleDB launches it within a second or two (the [#1788] behaviour), so adding the policy AFTER inserting eligible chunks hands a background run three chunks to compress while the test is trying to observe its own deterministic `run_job`. That background session carries no `lock_timeout` - the default is wait-forever - so it queued behind the `ACCESS EXCLUSIVE` lock the isolation test takes on the middle chunk and compressed it the instant the test rolled the blocker back, landing directly on the assertions. It surfaced as two failures that look nothing like each other: `Expected: 2 / Actual: 3` when the background run beat the first assertion, and `Expected: 1 / Actual: 0` when the chunk flipped BETWEEN the two reads so neither count saw it. The three tests that insert chunks now add the policy FIRST, while there is nothing to compress, and park its job before any row exists - the ordering is the load-bearing half, because parking a job that has already launched does not recall the run in flight. Same idiom the file already used for the [#1760] sentinel probe, and the same lever the payload-dimension tests pull against this behaviour on the continuous-aggregate side; `run_job` still executes a parked job, so the deterministic path the tests are built on is unchanged. **It never reproduced locally** because the test cluster runs `max_worker_processes = 8` against TimescaleDB's default `max_background_workers = 16`, so job launches routinely fail outright and the race cannot happen - twenty consecutive local runs passed for that reason alone, and raising the limit on the same rig made the background run fire every time. That provisioning gap is [#1888]. Test-suite only - no shipped code changed. - -- **A live test that fails to clean up after itself now fails the run instead of reporting success** ([#1873]) - the live classes' teardowns ran each statement through a best-effort helper whose `catch` was empty in one class and wrote a single `Console.WriteLine` in another, and xUnit attaches console output to the test that produced it, so on a PASSING test nobody reads it. A `DROP MATERIALIZED VIEW` that lost a race therefore reported success, the continuous aggregate survived into the shared `DARLING_TEST_PG` store, and on a reused database every later run inherited it - which is how `query_stats_db_hourly` and `query_stats_db_daily` came to persist, changing compose's tier routing, feeding the [#1784] coverage gate, and making `EnsureBaselineFallbackViewsAsync` a no-op for whatever they shadowed. [#1862] removed the one assertion that made the residue visible, which made it quieter rather than rarer. **The collision is now reproduced rather than inferred**: on PostgreSQL 18.4 + TimescaleDB 2.28.1, a `DROP MATERIALIZED VIEW` concurrent with a `refresh_continuous_aggregate` of the same aggregate fails and leaves the aggregate STANDING, every time it collides - as `40P01` when the deadlock detector picks the drop as its victim while it deletes from `continuous_aggs_materialization_invalidation_log`, and as `XX000 tuple concurrently deleted` when the refresh's own catalog maintenance beats it to a row, from the identical setup. - - **That second SQLSTATE is why the fix does not classify errors.** A retry allow-list is the natural shape and it is the wrong one here: `XX000` is `internal_error`, the class no one would put on such a list, yet it is what an ordinary refresh collision produces. Removals now run through one helper that retries ANY failure and asks the catalog whether the object is actually gone - the postcondition is the only thing that has to be true, so nothing has to be right about which errors count. It also makes the reverse case free: `DROP OWNED BY` has no `IF EXISTS` form and raises `42704` on every pre-test call in the security-split class, and a probe that finds no such role reports success rather than needing an exemption. Each aggregate's refresh policy is removed BEFORE its drop, which bounds the collision to the one run already executing instead of one the scheduler keeps relaunching ([#1788]); the retry then outlasts that. All three swallow sites found by an audit of the project were converted - the payload-dimension and Timescale-support cleanups and the security-split class's CLUSTER-wide roles, which outlive even a `DROP DATABASE`. - - **The alarm is rung where it cannot lie about a test's result.** These cleanups run in `finally` blocks, and a throw from a finally REPLACES the body's in-flight exception, which is the masking [#1794] exists to prevent - so the helper never throws. It records what it could not remove, with the name of the test that owned it, and the `live-postgres` collection fixture compares the store's relations against the set captured when the run began and fails the RUN at teardown, after the last test has reported. Measured on xUnit v3 3.2.2 rather than assumed: that surfaces as `[Test Collection Cleanup Failure (live-postgres)]` and exits non-zero while the tests themselves still pass, which is the correct attribution - the run is what is broken, not any one of them. Both halves accuse independently, because an armed retention policy that could not be removed leaves no relation behind and will still drop another test's chunks on its own schedule. One catalog reading was corrected on the way: `timescaledb_information.jobs` reports a refresh policy against the user-facing VIEW, not the materialization hypertable its column names invite, so the obvious join matches nothing and yields a verification that always passes. The opposite shape - sixteen `finally` blocks that clean up LOUDLY but on the body's connection, so a throw from the finally replaces the failure it was cleaning up after - is [#1896], filed rather than folded in: the residue check above already catches what they abandon, so what is left is a diagnosis cost rather than a correctness one, and sixteen tests' failure semantics wants its own change. Test-suite only - no shipped code changed. - -- **A live test that fails mid-body now reports its OWN error, not whatever its teardown said afterwards** ([#1896]) - the sixteen `finally` blocks the entry above deferred are converted. They cleaned up on the BODY's connection and threw straight out of the `finally`, which is the shape [#1794] was filed for: a throw from a `finally` REPLACES the exception already in flight, so a test that failed for a real reason reported the teardown's error instead, and every cleanup statement after the throwing one was abandoned - leaving debris in the shared store that the next run inherits as an unrelated flake. The connection is the reason the two halves compound: it is the body's failure that closes it, so a teardown running there fails BECAUSE of the thing it is about to hide. **Demonstrated rather than argued**: with a body failure and a cleanup failure both forced into one converted test, the converted shape reports the body's exception and the pre-change shape reports the cleanup's `42883`, losing the body's entirely. All sixteen now run through `LiveStoreCleanup`, which opens its own connection, uses `CancellationToken.None` so a cancelled run still restores, and stays silent only while the body's exception is in flight. - - **Two of the sixteen deliberately do NOT move to a fresh connection, and that is the interesting half.** Resetting `lock_timeout` is a SESSION setting and rolling back a blocking transaction belongs to the connection holding it, so handing either a new connection would leave the real session altered and the real lock held while reporting success - the failure mode is quieter than the one being fixed, not louder. They take a new `RunOwnedAsync` overload that carries the same masking rule over the caller's own resources. The rule itself is now pinned by `LiveStoreCleanupTests`, which is new: roughly twenty call sites already depended on `LiveStoreCleanup`, and nothing asserted the property they depend on. It covers both directions, plus a source-level check that both overloads keep the filter - because the connection-opening one cannot be reached without a store, and a suite that silently lost the rule would keep failing and simply say the wrong reason. Three adjacent corrections ride along: a teardown helper that fetched `TestContext.Current.CancellationToken` internally (which would have re-signalled itself on the very cancelled-run path `LiveStoreCleanup` passes `None` to avoid), a duplicated and character-corrupted copy of a comment, and a stale `` left by [#1873]'s rename that the compiler does not check. The ~124 remaining `finally` blocks in other live classes are the same shape at a much smaller blast radius - they delete their own sentinel rows rather than dropping aggregates, tables or cluster-wide roles - and are [#1902]. Test-suite only - no shipped code changed. - -- **The live teardowns that could corrupt OTHER tests' results are converted first, and the backlog can now only shrink** ([#1902], batch one of three) - the same defect as the entry above, applied to the 126-teardown backlog it deferred: cleanup running on the BODY's connection and thrown straight out of the `finally`, so a test that failed for a real reason reports its teardown's error instead. Batch one takes **39 teardowns across 24 classes**, chosen on blast radius rather than convenience: every class whose leaked rows change what OTHER tests SEE rather than merely what they count. That is the `config.*` writers - a stranded `config_mute_rules` row mutes another test's alerts, a stranded `config_alert_settings` value hands every later test a CPU threshold this one invented, `config_edge_trigger_watermarks` moves an edge trigger, `config_monitored_servers` and `custom_views` change what the fleet and view readers enumerate - and the `collect.servers` registrations behind the MCP tool tests, where a leaked row is a phantom server in every fleet-wide read. The remaining 87 are the ones whose leaked rows only inflate their own sentinel's counts, and are batches two and three. - - **The two gotchas [#1896] predicted both recurred, at scale.** Ten teardown helpers across six classes fetched `TestContext.Current.CancellationToken` internally, which under `LiveStoreCleanup` - deliberately `CancellationToken.None`, so a cancelled run still restores the store - would have re-signalled itself on exactly the path the helper exists to survive; they take a token now. Two classes already opened their own cleanup connection and so were half-right, but still threw from the `finally` and still used the test's own token; `LiveStoreCleanup` supplies both halves and the hand-rolled connection goes with it. One teardown deletes through the mute-rule STORE, which is the API under test and owns its own data source, so it takes `RunOwnedAsync` - it was never exposed to the closed-connection half and only needs the masking rule. **A one-number ratchet lands with the batch**: `LiveCleanupConversionRatchetTests` counts what remains and fails both if the count RISES (someone adds a new one behind the batches) and if it FALLS without the ceiling coming down (a batch leaving slack that new ones could be spent on). A named allowlist of eighty entries would be a merge-conflict generator that every batch edits by hand; one number cannot be partially wrong. When it reaches zero the assertion becomes the real invariant and [#1902] closes. Test-suite only - no shipped code changed. - -- **The whole Viewer read-path test family now reports its own failures too, and the ratchet drops to 19** ([#1902], batch two of three) - 68 teardowns across the 21 `Viewer*` live classes, taken as one tranche because they are one shape: plant rows for a viewer query, read them back, delete them in a `finally` that ran on the body's connection. **Every one of the 21 had the token gotcha** - 25 delete helpers reaching for `TestContext.Current.CancellationToken` internally, which under `LiveStoreCleanup` (deliberately `CancellationToken.None`, so a cancelled run still restores the store) re-signals itself on precisely the path the helper exists to survive. None of those files declares a local `ct` at all, so their body call sites now pass the ambient token explicitly rather than introducing a mixed idiom inside one method. What is left for batch three is the 19 Darling engine sites, including the one teardown that cleans up through an `NpgsqlDataSource` rather than a connection - deliberately isolated so it gets read properly instead of being buried in a sixty-eight-site diff. - -- **A compression test that asserts a job has NEVER RUN stopped losing to the job actually running** ([#1888]'s predicted follow-through, found by a batch-two run) - `StuckCompressionJobsSql_NeverRunJob_ReadsNullLastRunStartedAt` pins the [#1760] sentinel: `timescaledb_information.job_stats.last_run_started_at` reads `-infinity`, not NULL, for a job that has never executed, and the production query has to neutralise that or the stuck-Running arm fires on a job that never ran. The test created its compression policy UNPARKED, and `add_compression_policy` creates the job SCHEDULED with no `initial_start` so TimescaleDB launches it within a second or two ([#1788]) - one background launch destroys the sentinel outright, and unlike a chunk count no amount of re-reading recovers it. [#1889] fixed exactly this for three sibling tests by creating and parking in ONE transaction, so the scheduler (a separate backend) cannot see the job until the row already reads `scheduled = false`; this was the fourth site, and it now uses the same helper. [#1888] called it: raising CI's worker slots "will make background jobs launch reliably for the first time, so any other test that has been quietly relying on the scheduler being unable to run is going to start failing ... whether others exist is unknown, and finding out is the actual work here." **Proven both ways rather than by re-running until green**: with the old unparked creation and a deliberate four-second scheduler window the assertion fails as `Expected: True / Actual: False` - byte-identical to the intermittent full-suite failure - and with the parked creation and the same window it passes every time. No permanent guard is added, for cause: three other live tests in the same file create unparked policies BY DESIGN (an already-armed legacy policy is what the converge tests converge), so any source-level rule would either fire on those or be narrowed until it pinned nothing. - -- **No live test cleans up on the connection its own body used any more** ([#1902], batch three of three - closes it) - the last 19 teardowns, all in the Darling engine classes: `DarlingCollectorRunner` (4), `DarlingAnalysisStore` and `DarlingDeltaSeeder` (2 each), and one each in `DarlingAlertReadAdapter`, `DarlingAnalysisPipeline`, `DarlingAnomalyBaseline`, `DarlingCompose`, `DarlingFleetReader`, `DarlingModuleMap`, `DarlingWatermarkSeedLive`, `ExcludedDatabases`, `PgFactCollector`, plus both of `LiveCleanupBatchTests`' own. Ten more helpers across eight classes had the cancellation-token gotcha and were threaded, finishing the sweep that began in batch one. - - **The `NpgsqlDataSource` teardowns were saved for last, and the read they were saved for changes what the fix is FOR.** `DarlingCollectorRunnerTests` cleans up through a data source rather than a connection, and a data source hands out a fresh connection per call - so those four sites were never exposed to the half of the defect everything else was, the one where the body's failure closes the very session the teardown then uses. What they lacked is the other two: the masking rule, and a token that survives cancellation (they passed the body's `ct`, which on a cancelled run is already signalled, so the delete was skipped exactly when it mattered most). They now go through `LiveStoreCleanup.RunAsync` off the connection string, which additionally gives them the explicit `SET search_path` the data source's own connections never issued - so the move is a strict improvement rather than a lateral one. `CleanServerRowsAsync` keeps its data-source overload for the PRE-clean calls, which legitimately draw from the source the test already holds. - - **The ratchet becomes the invariant, and rewriting it found a real hole.** Its detector had scanned a fixed thirteen lines after each `finally`, and that window overruns the block: in `LiveCleanupBatchTests` it ran past the closing brace into the NEXT method's doc comment, which mentions `` in prose - so an unconverted teardown was marked compliant by a comment that merely NAMED the helper it was supposed to call. That is the same shape [#1874] found in the [#1776] rule, where a class quoting `[Collection("live-postgres")]` while explaining the rule exempted itself from it. The block is now brace-matched, which reads what the compiler reads. The old detector also had a matching false POSITIVE (a converted teardown whose twelve-line comment pushed its own `LiveStoreCleanup` call out of the window), and the two cancelled numerically - so the 126/87/19 counts reported by batches one and two were right in total and wrong in composition. With the detector fixed and the last sites converted the count is zero, the ceiling constant is deleted, and the assertion now states the invariant directly: no live test cleans up on its own body's connection. Opening a fresh connection BY HAND is deliberately not accepted as compliant - it is half the fix, it still throws from the `finally`, and an exemption shaped "this one is correct by hand" is one a later incorrect site inherits. Test-suite only - no shipped code changed. - -- **The live compression tests had two ~200-second windows around midnight where they failed for the clock rather than for the code** ([#1972], caught on a dev-to-PR pair the same night) - three `*_AgainstDevPostgres` tests seeded their chunks with `now()`-relative timestamps spanning 200 seconds into day-aligned chunks, so what time of day the suite ran decided how many chunks a seed produced. Run between 00:00:00 and 00:03:20 UTC the young seed straddled midnight and landed TWO chunks instead of one, and the uncompressed count asserted 1 against an actual 2; run between 23:56:40 and 23:59:59 an old seed crossed the boundary N days back into two closed, both-eligible chunks, and the compressed count failed the same way in the other direction. This is not the reused-database flake that moves between unrelated classes - it is deterministic inside the window and green outside it, which is what the evidence showed: a `dev` run started 23:50 UTC went green, a PR run carrying a byte-identical `TimescaleSupportTests.cs` failed at 00:01:38 UTC on exactly the line above, and a rerun outside the window passed with no code change. All four seeds now anchor to midday (`date_trunc('day', now()::timestamp)` plus twelve hours), which puts 12 hours of margin either side of a 200-second span, so chunk placement no longer depends on when the suite runs. Confirmed against a live TimescaleDB rig by entering both windows on demand with literal anchors rather than waiting for them: the current forms reproduce the expected-1-actual-2 shape in both directions, the midday form gives 1 and 1, and a sweep across all 1440 minutes of the day splits on 3 minutes for each `now()`-relative form and none for either midday one. The young chunk keeps the property its test pins - midday today sits in today's chunk, whose `range_end` is tomorrow's midnight, so the "still young" assert holds at every hour. Test-suite only - no shipped code changed. - -- **The gated-live plan-capture E2E no longer lets the target's plan cache decide its verdict** ([#1988]) - `EndToEnd_QueryStats_PlanCaptureFlag_TogglesStoredPlanXml` failed deterministically on the standard local rig with "flag-on capture should store at least one non-empty plan", and the diagnosis found TWO stacked causes, only one of them environmental. The deterministic one: the test asserts on the inline `query_plan_xml` column, which [#1767]'s payload-dimension diversion writes NULL by design - the XML goes to the digest-keyed `query_plan_dim` and every product reader coalesces the inline column with the dim join - so the test failed on ANY rig where a plan was captured, and "environmental" was only half the story. The environmental one: under `optimize for ad hoc workloads`, a query executed once caches a Compiled Plan STUB - `dm_exec_query_stats` carries the row and its text renders, but `dm_exec_text_query_plan` renders NULL - so a box whose recent user-database activity was all once-executed ad-hoc text (a test box between suite runs, exactly) had no capturable plan for the collector to find even before the diversion mattered. The test now seeds its own scratch database on the SQL target, runs a marker query TWICE with byte-identical text (the second execution replaces the stub with a full Compiled Plan), scopes the plan assertions to that marker row, and reads plans the way the product does - coalesced through the dim join. The old skip-when-idle goes with it: the seeded marker guarantees collectable rows, so the test runs meaningfully on a completely cold box too. Verified against a Docker SQL2022 + TimescaleDB pair through the real `DarlingCollectorRunner` path in both directions: with adhoc-optimize ON and a stub-heavy cache (90 ambient rows, only the marker carrying a plan - the old test's exact failure state), and with it OFF on a cold cache (1 row, where the old test skipped) - all assertions green in both. The reproduction first confirmed the old test's failure shape on the same rig: rows landed, every reachable plan NULL. Test-suite only - no shipped code changed. - -- **Dependabot's PRs now target `dev`, where they can actually merge** ([#1822]) - the config never set `target-branch`, so Dependabot opened its weekly grouped PRs against the repository default branch, `main` - and the check-branches guard rejects any PR to main that is not dev itself, so every one of them was born unmergeable. The first one demonstrated a second problem on its way down: the NuGet group half-applied the ModelContextProtocol 1.4.1-to-2.0.0 major (three projects bumped, Lite and deprecated/Dashboard left at 1.4.1), which locked-mode restore rejected outright (NU1605 downgrade conflicts, NU1004 lockfile inconsistencies) - and even a mechanically clean bump of that major changes runtime behavior on both MCP hosts (stateless-by-default HTTP, discovery-first negotiation, deprecated Roots/Sampling/Logging APIs). Both update blocks now carry `target-branch: "dev"`, and `ModelContextProtocol*` semver-majors are ignored - same shape as the existing upload-artifact ignore - until the deliberate migration ([#1990]) lands, at which point the ignore rule comes out and minors/patches keep flowing through the weekly group throughout. One honest limit: Dependabot reads this file from the DEFAULT branch, so the retarget takes effect at the next dev-to-main release promotion, not on merge. Config only - no shipped code changed. - -## [3.3.0] - 2026-07-29 - -### Important - -- **Darling stores still on PostgreSQL 17 are upgraded IN PLACE to PostgreSQL 18.4 + TimescaleDB 2.28.1 on the first service start** ([#1718], [#1752]). The store is offline while it runs (budget roughly 2 minutes per 68 MB of store); copy mode needs about 2x the data directory free on the same volume, and the old data directory is retained as an automatic rollback for two clean starts. A package carrying an OLDER PostgreSQL major than the running store refuses instead of downgrading. -- **On Darling stores with pre-existing history, the raw retention purges stay safely HELD until rollup coverage reaches the oldest raw row** ([#1788], [#1799]). Run the new `--backfill-rollups` operator verb (disk-preflighted; it refuses with numbers rather than filling the volume) to materialize coverage; the arming gate then releases the purges by itself on the next start, followed by one LARGE one-time space reclaim. Until then the per-sweep `Retention purge SKIPPED` warnings are expected, not faults. -- **The first start on this release also retunes every compression policy to an hourly tick and raises `maintenance_work_mem` to the measured compression floor** ([#1779], [#1780]); the managed PostgreSQL restarts once to apply the configuration. The store migrates to **schema v39** (partial indexes supporting the dimension GC's measured floor, [#1813]) — automatic, no operator action. -- **The Full Dashboard and the CLI Installer are retired**: moved under `deprecated/`, no longer built into release artifacts. Existing installs keep working and remain on bug-fix support; the release artifacts are Lite, the Darling service, and the Darling viewer. - -### Added - -- **The gated live-Postgres leg now asserts the cluster it connected to serves the pinned runtime versions** ([#1806], closes [#1787]) - `DarlingPgRuntimeVersionPinTests` proves the ASSEMBLED bundle matches `fetch-pg-runtime.ps1`'s pins, but nothing tied the cluster `DARLING_TEST_PG` points at to that bundle: when a fixture DDL failure looked like an older CI TimescaleDB version, disproving the claim took a night of derivation (it was a `search_path` effect) when one catalog row would have settled it instantly. A new `[Collection("live-postgres")]` test - active only when `DARLING_TEST_PGRUNTIME` is also set, which is how CI's darling-pg job and the nightly both run - reads `SHOW server_version` and `pg_available_extensions.default_version` / `installed_version` for `timescaledb` and requires them to equal the script's pins, sourced from the same parser the bundle guard uses so the two can never disagree about what "pinned" means. A stale fixture cache or a wrong cluster now names itself in the failure instead of masquerading as a product bug. - -- **The backfill's slice refreshes now pass `refresh_newest_first` explicitly, and a degrade is impossible to ignore** ([#1797]) - the newest-first slicing that makes `--backfill-rollups` resumable relied on TimescaleDB's `refresh_newest_first` DEFAULT being true, which left the verb's completeness-by-construction claim one flipped engine default away from silently inverting (an oldest-first run's measured floor reaches bottom on the first slice, so an interruption looks COMPLETE). The option is now passed explicitly through the 2.21+ `options` argument. On a bring-your-own store older than 2.21 the call raises `42883`, and the run latches ONCE to the no-options form for the remainder - per-run, not per-slice - while DISCLOSING the degrade through a sink the compiler REQUIRES at every call site. The requirement is the point: an earlier cut carried the disclosure as a trailing OPTIONAL parameter, and it shipped as a dead no-op through a green suite precisely because nothing had to acknowledge it existed - where a parameter carries a safety obligation, `CS7036` is the enforcement. Measured on a live store: one `42883` in a 152-slice run, the backfill completing correctly through the fallback and saying so. - -- **Two guards that turn structural properties into checked ones** ([#1796]) - review closers on [#1789] / [#1793], tests only. **The raw-tier retention policies are now pinned as DERIVED from `TimescaleSupport.RawTierCoverage`**, not merely equal to it. That map is what makes the tiered retention POLICY and the catalog SWEEP unable to judge the same drop differently, and until now the derivation was structural only: re-hardcoding the three rows beside the map produces an identical policy set and passes every behavioural test, right up until the map gains a fourth tier or one of the three changes its covering aggregate — at which point the two paths silently disagree about the exact thing the map exists to keep aligned. No behavioural test can see that, because a duplicated list behaves identically; only reading the construction can. Watched red by re-hardcoding the three rows exactly as a future edit would. - - **And the coverage-skip log line is pinned in FULL rather than by prefix**, matching its dimension-GC sibling. Its TAIL is the actionable half — *resumes by itself once a backfill extends coverage* is what tells an operator the state is self-correcting rather than a fault to chase — and a prefix pin covers only the part naming the table. That is not hypothetical: the prefix-pin class already let a wrong line ship earlier in this work, where every assertion passed while the message named a cause the guard never detected. Watched red by deleting just the tail, and the failure output shows the prefix still intact — which is precisely the mutation the old pin would have called green. - -- **The catalog retention sweep no longer deletes history the rollups never captured** ([#1793], closes [#1784]) - two purges drop the same raw chunks, and only one of them honored the #1680 coverage gate. The tiered 4-day policy is created paused and armed only once the hourly aggregate covers everything the table holds; `DarlingRetention.PurgeAsync` then dropped those very chunks at the per-collector 30-day horizon with **no coverage check at all**. On a store where the gate is deliberately holding the policy - because the rollup does not reach back over raw's history - the sweep destroyed exactly the uncovered history the gate exists to protect. The gate was not bypassed by a bug in the gate; it was bypassed by an older path that never learned the invariant. Field fuse on a production instance: raw from ~Jul 9, coverage from ~Jul 24, so the uncovered slice began dying ~**Aug 8** unless an operator ran a backfill in time - and the product must not depend on that. Both purges now consult the SAME predicate, reached through one shared map of raw tier to covering aggregate, so they cannot judge the same drop differently. - - **It is a binary judgement, not a clamped cutoff, and that is forced by the mechanism.** `drop_chunks` can only remove the OLDEST chunks, and when coverage lags it is exactly the oldest chunks that are uncovered - so no cutoff value drops the covered tail while sparing the uncovered head. Working the intuitive `min(catalog horizon, coverage floor)` clamp against the field numbers shows it: at Aug 20 it yields Jul 21, which deletes Jul 9-21, every row of it uncovered. In the healthy case it fails the other way, clamping to the coverage floor and never dropping anything at all. Tiered tables are deliberately NOT excluded outright either - the moment coverage reaches back the normal horizon applies again, so a never-armed policy cannot mean unbounded growth. - - This also **inverts #1759's framing**: held tiers were recorded there as a growth problem, when the sweep was in fact bounding these tables at 30 days by spending rollup-uncovered history to do it. A comment on that issue now says so. - - Both halves are pinned, because a guard that never drops anything would satisfy the first alone: with coverage lagging a 45-day-old row SURVIVES the sweep, and once the aggregate is refreshed back over it the same row is dropped. Watched red both ways against a live PostgreSQL 18.4 + TimescaleDB store - guard removed, the uncovered row is deleted (`Expected: 1, Actual: 0`); guard forced always-skip, the backstop stops backstopping (`Expected: 0, Actual: 1`). - -- **The payload-dimension GC now stands down when the purge it depends on failed** ([#1789], closes [#1782]) - the dimension GC prunes `query_text_dim` / `query_plan_dim` rows whose `last_seen` has aged past the fact tables' own retention plus a margin, on the reasoning that any fact which could still reference that content is gone. The fact purge immediately above it in the same sweep is **failure-isolated**: a table whose `drop_chunks` fails, and whose DELETE fallback then also fails, is warned and skipped while the sweep continues straight into the GC. Those rows are still there, still carrying digests, and their dimension rows get pruned out from under them - the resolving view then hands back NULL payload for real collected data, with **no error raised anywhere**, which reads as a collection outage rather than a retention bug. A dim-feeding table that failed its purge now defers the GC for that cycle with one fixed log line (`dimension GC deferred: raw purges held, facts may outlive their policy window`), self-ending on the next sweep that purges cleanly and needing no operator. - - **The issue's original premise was disproved before this was built, and the fix is keyed to a different trigger than the one reported.** #1782 proposed that a HELD tiered retention policy could let facts outlive the cutoff. It cannot: there are TWO purges on these tables, and only the tiered 4-day policy is coverage-gated. Darling's own catalog sweep drops the same tables at their per-collector horizon - 30 days for `query_stats` and `procedure_stats` - with no coverage check and no held check at all. So a held policy changes whether facts live 4 days or 30, both inside the 32-day dimension cutoff, and the ordering survives any retention override because both sides resolve through the same `retentionDaysFor`. Guarding on the held state would have deferred the GC indefinitely on a store whose policies are held for unrelated reasons - unbounded dimension growth, which is the exact failure #1768 exists to prevent - to prevent something that cannot happen by that route. The guard is keyed to the reachable trigger instead. - - Verified at the seam rather than at the sides, because the defect is wiring: the live test injects a **genuine** total purge failure (the fact table is renamed for the duration, so both statements fail against a relation that does not exist) and asserts the referenced dimension row survives, then restores it and asserts the same row is pruned once the purge is healthy - so the test cannot pass merely because the GC never ran. Watched red against the unguarded form. - - Establishing all of the above surfaced a larger, opposite-signed defect in the same file, filed separately as [#1784]: that ungated catalog sweep **bypasses the #1680 coverage gate entirely**, dropping rollup-uncovered raw history at 30 days on a store where the gate is deliberately holding the tiered policy to protect it. - -- **The query text and plan XML now live once each, not once per collected row** ([#1768], closes [#1767]) - on a production field instance `query_stats` (176 GB) and `procedure_stats` (58 GB) were **94% of a 250 GB store**, and the cause was schema-level rather than anything retention or compression could reach. Chunk-level measurement: a representative `query_stats` chunk was 88.97 GB raw (12.41 GB table + 74.15 GB TOAST) compressing to 17.98 GB - of which **17.5 MB was the table and 16.73 GB was TOAST**. Columnar compression crushes the monitoring numerics 665x; TOAST compression manages 4.1x because it is byte-level and has no cross-row awareness, so re-collecting the same cached plan every minute stores that plan again every minute. **99.9% of the compressed chunk was payload, and the monitoring numbers were nearly free.** Both payloads move into hash-keyed dimension tables (`collect.query_text_dim`, `collect.query_plan_dim`) and the fact rows carry only a digest. A dedup PoC over a real 1-hour window - 514,896 rows, 3,166 MB of inline payload - collapsed to ~23 MB of distinct content, and the ratio *improves* as the window widens because distinct plans grow far slower than rows. `procedure_stats` has no `query_text` column at all, so its entire share is plan XML; both fact tables share ONE plan dimension, which also means the same vendor application's plan collected from fifty servers is stored once fleet-wide. - - **The key is a SHA-256 CONTENT digest, not one of SQL Server's own hash columns**, and that is a deliberate departure from the issue's sketch because each alternative is wrong in a way that fails silently. `query_plan_hash` is a plan-SHAPE hash: the same shape recompiled with different sniffed parameter values carries the same hash and DIFFERENT XML, so a shape-keyed dimension with `ON CONFLICT DO NOTHING` would pin the first XML forever and serve stale `ParameterCompiledValue`s - the single most diagnostic element of a plan for this product's audience. `query_hash` is a query-SHAPE hash and would pin one arbitrary literal variant where every reader today shows the LATEST text. `plan_handle` does not determine the XML at all: `query_stats` captures the STATEMENT plan via `dm_exec_text_query_plan(plan_handle, statement_start_offset, statement_end_offset)` and the offsets are not stored - **this codebase already recorded that exact failure**, in `QueryStatsCollector`'s delta key ("keying on plan_handle alone cross-contaminated multi-statement plans (one plan_handle, many statements at different offsets)"). A content digest rests on no cache-internals assumption. It costs two `bytea` columns per row, roughly 1% of the payload it removes, and it does **not** cost dedup ratio, because the repetition being collapsed is the same cached plan re-read each cycle: measured on a live SQL Server, **69 of 89 cached plans hashed byte-identical across a 20-second gap**, which is the property the whole design turns on. - - **Migration is zero-rewrite**, the peak-disk-before-relief trap recorded and rejected in [#1759]: V38 adds the dimensions and three nullable digest columns and touches not one existing row. The inline columns REMAIN, new rows leave them NULL, and raw retention ages the old inline copies out on its own - the hourly and daily aggregates never carried the payload, so nothing above raw is affected. Dropping the columns is a later, separate decision, deliberately not taken here. - - **The failure mode this change carries is silence**, and most of the work went there: a reader that keeps selecting the raw column does not error, it returns NULL for every row written since the migration, which looks exactly like a collection outage. So `v_query_stats` is rebuilt as a payload-RESOLVING view (`COALESCE(inline, dimension)`) whose column list is GENERATED from the collector definition and therefore cannot go stale the way a hand-written list would - that is the same defect V14 exists to fix, not re-introduced. Resolving it there fixes every reader that goes through the view and makes future readers correct by default. It is also excluded from `AllPassthroughViews`, because "ADD COLUMN, then `CREATE OR REPLACE VIEW v_x AS SELECT * FROM x`" is the established idiom here (V14, V15, V27, V28) and applying it to this view would overwrite the COALESCE with no error anywhere. A per-citation inventory covered every consumer: nine `query_stats` readers moved to the resolving view, the two `procedure_stats` plan readers resolve the dimension in their own SQL **with the `IS NOT NULL` guard moved onto the COALESCED expression** (leaving it on the bare column filters out every new row before the join can help, which is the single most likely regression and the one that most looks like a data problem), and both presence flags test `OR ..._digest IS NOT NULL` rather than paying for a join. The Top Queries aggregate deliberately still reads the base table: it needs only a presence flag over the whole window, and the resolving view would make Postgres join the plan dimension per row to evaluate it. - - **Write path.** Payloads divert at the binary-COPY writer, keyed by payload column ORDINAL derived from the collector definition by name - a rename now throws rather than silently reverting to inline storage. The dimension upsert shares ONE transaction with the fact COPY, so no session ever observes a fact row whose digest resolves to nothing. It could not be literal dims-first ordering: the digests are only known once the rows stream, and a second `WritePayload` pass is not available because `WritePayload` CONSUMES the delta state (`context.Deltas.CalculateDelta`), so a second pass would report every delta as zero. The transaction gives the stronger guarantee anyway. A new `EndPayload` check pins the positional COPY contract for every collector, not just the diverted ones. Dimension GC rides the existing retention sweep on a `last_seen` watermark rather than an orphan anti-join against two hypertables, at the widest effective FACT retention plus a margin covering `drop_chunks` granularity and the watermark's own hourly refresh guard - so a raised per-collector retention override can never outlive the dimensions and orphan a reader. - - **Verified against a real PostgreSQL 18.4 + TimescaleDB store**, not a mock, because a green local run is no evidence at all for which relations a query reads - those failures are swallowed, not thrown. Every guard was verified RED by mutation: removing `ON CONFLICT` reproduced `23505 duplicate key value violates unique constraint "query_text_dim_pkey"`; dropping the view's COALESCE reproduced the exact silent-NULL symptom (`Column 'query_text' is null`); dropping one payload-index increment reproduced `Collector wrote 10 payload values but declares 50 payload columns`; re-adding the view to the passthrough set went red on the exclusion guard; and committing the COPY before the dimension flush went red on the ordering pin. That last one earned its own test for a reason worth recording: the live atomic-visibility test MIRRORS the write path rather than calling it (the writer is private), so a production reorder left it green - the seam between them needed its own pin. - - **Three findings that contradict the issue's premises, recorded rather than quietly worked around.** (1) The compression-delay lever is **already at its floor**: `CompressAfterDays` is 1, not 24h-and-tunable, and the binding constraint is `ChunkIntervalDays = 1` - a chunk cannot compress before it closes, so lowering the delay alone does nothing. (2) Raw retention **cannot** simply go below 4 days: 4 is 3 (the hourly aggregate's own refresh window) + 1 safety day, so the raw drop never outruns the aggregate preserving it; shortening it means shortening the refresh window first. (3) The `TOP (200)` row cap is deliberate in direction but inherited in magnitude - raised from 50 in Feb 2026 to widen coverage after a recency filter freed up slots, with no rationale for 200 specifically and no storage analysis, four and a half months before plan capture existed. It is left alone: row caps do not bound bytes, which is the whole lesson of this issue, and after dedup the marginal cost of a row is numerics only. **Lite is deliberately untouched**: it has the same inline schema but `CapturePlanXml` defaults false and Lite never sets it, so it has never stored a single plan; its `query_text` measured 1.72 MB compressed per server-month in the parquet archive (345x on that column) against a hot store hard-capped at 512 MB. There is no problem there to port a mechanism to. - -- **Eight members had someone else's documentation, and seven had lost their own** ([#1751]) - a repo-wide pin plus the eight fixes it found. When an edit inserts a new member and its `` ABOVE an existing summary instead of below it, the existing doc block is stranded on the wrong member: the new member reads with a description of something else entirely, and the member the block actually described is left with **no documentation at all**. XML docs take the LAST summary, so IntelliSense, generated docs and every analyzer render correctly; the build is clean, nothing warns, and no behavioral test can see it. The only reader misled is a human reading top-down, who gets the wrong description first. In [#1739] the same region was read at least three times by two people over several hours, one of whom had quoted a paragraph out of it, and neither saw the duplicate - a regex found it in one pass, which is why this is a test and not a review note. **The class is live, not historical**: two of the eight were introduced by [#1739]/[#1744] itself, the very PRs that fixed two other instances of it, and shipped to dev - `RunCollectionLoopAsync` and `EvaluateCompressionJobsAsync` both lost their documentation to an inserted `#1706` member. The other six were pre-existing across four projects: `CreateQueryStatsDailySql`, `RefreshVisibleAsync`, `ServerInventorySql`, `FindRepoRoot` and `class RemoteCollectorService` had each been left undocumented the same way, and `GetQueryStoreHistoryAsync` carried a superseded description of itself alongside the current one. **Seven of the eight were fixes by MOVING the orphaned block back to the member it describes, not by deleting it** - the obvious "remove the duplicate summary" sweep would have destroyed real documentation at seven sites, which is why the guard's failure message says so explicitly rather than leaving the next person to work it out. The pin walks every `.cs` outside `bin`/`obj` from the repo root and fails on `` immediately followed by ``; it FAILS rather than skips when it cannot locate the tree, because a guard that silently skips is a guard that silently stops guarding. Verified in both directions: green on the fixed tree, and red naming `Lite\Services\RemoteCollectorService.cs:29` when a stacked block is re-introduced - a Lite mutation caught by a test living in `Darling.Tests`, which is the cross-project reach the rule needs. Documentation and one test only; no behavior changes - -- **Darling store runtime upgrades: in-place 17 to 18, validated end to end** ([#1718]) - a field store has always run whatever PostgreSQL and TimescaleDB it was first created with, forever, because the runtime probe returns early the moment `pg-runtime\pgsql\bin\pg_ctl.exe` exists. Shipping a newer bundle changed nothing on any host that had already extracted one, which is exactly the drift [#1705] caught. The detector is now the zip's SHA256, recorded in `pg-runtime.stamp` at extraction: a package whose zip hash differs from the stamp carries a new runtime, and that one signal covers both a TimescaleDB-only bump and a PostgreSQL major jump. On a change the current runtime is RESCUED to `pg-runtime-prev` (whole tree, not just `bin` - the old postmaster resolves `$libdir` relative to itself) before the new one is extracted, because `pg_upgrade` needs both runtimes present at once and rescue-before-extract is the difference between an upgradeable store and an unupgradeable one. The orchestration then measures disk headroom and picks copy mode (default; the old cluster survives intact) or hard-link mode (loudly, since it trades the rollback away) or aborts safe; bridges TimescaleDB to the bundled version on the OLD cluster first (mandatory, not hygiene - pg_upgrade recreates the extension pinned at whatever version is installed, and the new runtime ships exactly one version-suffixed library); initdbs the new cluster to the old one's MEASURED locale/encoding/checksum identity rather than the new major's defaults (PostgreSQL 18 flipped initdb to checksums-on, and pg_upgrade hard-refuses a mismatch); writes the managed conf blocks before pg_upgrade rather than after, because `shared_preload_libraries = 'timescaledb'` has to be live for the extension to restore; runs `--check` as a dry run before the real pass; swaps the data directories; and verifies the result with a server-version check, an extension-version check and a real read of `collect.collection_log`. Any failure at any step reverts the runtime, leaves the untouched old data directory in place, and records the failing zip's hash in `pg-runtime.blocked` so a known-bad package is not retried on every restart. Same-major runs still get the TimescaleDB extension update on their own, which is [#1705]'s case by itself. Both terminal states raise a real fleet-level self-alert (`Store Runtime Upgrade`) through the normal deliverer, carried out of the bootstrap and fired once the alert engine exists - the store is DOWN while an upgrade runs, so its START can only ever be a log line, but a success on the new major and a failure back on the old one both happen with a live store; the failure alert names the step and says plainly that the store is still collecting and nothing was lost, and a hard-link success says outright that there is NO rollback copy. - - **Validated end to end on a quiet box.** A real PostgreSQL 17.10 store - 20,000 collector rows, 500 plan-XML rows totalling ~41 MB of TOAST, a continuous aggregate and four compressed chunks - upgraded to 18.4 through the production bootstrap, with row counts, an ORDERED row checksum, continuous-aggregate rows, compressed-chunk count and the extension version all compared before and after. Getting there flushed out four bugs, three of them ours and all fixed here. **Port collision**: pg_upgrade defaults BOTH throwaway postmasters to the fixed well-known `50432`, so any leftover postmaster - or a second upgrade anywhere on the host - collides and pg_upgrade silently talks to a stranger's cluster, caught as `FATAL: role "darling" does not exist` answered by a postmaster this service never started; now `--old-port 55432 --new-port 55433`. **IPv4-only loopback**: pg_upgrade has no Unix sockets on Windows so it dials its clusters BY NAME, and Windows resolves `localhost` to `::1` first, while the v1 conf block pins `listen_addresses = '127.0.0.1'` - leaving nothing on `::1` to answer (`connection to server at "localhost" (::1), port 50432 failed`, verbatim); `-c listen_addresses=localhost` now rides `-o`/`-O` for the upgrade window only, never editing the store's conf. That is the same trap `BuildConnectionString` already dodges by using the literal address - never applied to a tool that dials on our behalf. **A revert that could brick the runtime**: it deleted the live runtime before moving the rescued one back, and a partial delete (a postmaster still holding its binaries) left `pg-runtime/pgsql` with no `pg_ctl.exe` - an unbootable install produced by the very path that exists to make failure safe; both failure paths now move-aside, never delete-first. **A first-connection race**: `pg_ctl -w` returns when the postmaster accepts connections, which on Windows is not the same as the next backend spawning and surviving - every backend re-reserves the shared-memory region at the postmaster's base address (the error-487 surface behind the 1 GB `shared_buffers` cap), and losing that race authenticates fine then dies on the first query, so the open-and-first-query unit now retries with a fresh connection per attempt while a `PostgresException` still fails fast. Also corrected: the retention sweep ran AFTER the upgrade and aged the brand-new rollback copy on the very start that created it. `Darling/tools/new-upgraded-store-fixture.ps1` builds both runtimes for the gated fixture, which uses a capturing logger so a red run names the failed step, and `DARLING_TEST_KEEP` leaves the tree on disk for diagnosis. That fixture is the [#1705] CI gap: darling-pg only ever creates fresh stores. - -- **The AG replica chips now mark which node is the local one** ([#1735]) - V37 added `is_local` to the replica grain (#1696) and every AG surface now selects and renders it: the web dashboard, the Darling viewer tab and Lite's tab. It is a small label that carries the whole per-perspective model: without it a reader cannot tell a row where the server is describing ITSELF from a row where it is describing a node it only sees across the wire, and those carry very different weight, because the DMV populates operational state and recovery health for the local replica ONLY. That is the same asymmetry that makes one card per (reporting server, AG) the right shape, now visible instead of implied. Shown only when explicitly true: a NULL means UNKNOWN, not remote - rows collected before the column existed genuinely do not know, and labelling them either way would assert something the data cannot support. Inserting the column shifted the replica read's ordinals in all three readers, so both test matrices gained SQL pins alongside the display pins. -- **Darling: an AG failover is reported once, not once per monitored node, and a standing replica disconnect can re-alert** ([#1734]) - completes #1696. **Cross-node de-duplication:** every replica in an Availability Group is visible from every node, so a fully-monitored 3-node AG collected the same replica three times and reported one failover THREE times. `ag_replica_states` gains `is_local` (read from the DMV rather than inferred - matching a monitored server against `replica_server_name` is unreliable across instance names, aliases and listeners) and one server now judges each AG. Which one is not arbitrary: a SECONDARY's `sys.dm_hadr_*` carries only its OWN row, measured on the live fixture, so only the primary sees the whole group - vantage is ranked None < Remote < Local < LocalPrimary, the best available wins, a secondary yields to the primary as soon as the primary is monitored, and ties keep the incumbent so authority cannot oscillate. The AG edge state drops the server id from its key as part of this, because an AG is ONE object however many of its nodes are watched; that is what lets authority move without re-baselining and losing an alert, or double-firing one. Removing a server now releases its CLAIM rather than dropping the group's state, since another node may still be watching it. A NULL `is_local` reads as "unknown" and never as "not local" - rows predating the migration genuinely do not know, and de-duplicating on a false negative would drop a real alert. **Disconnect re-fire:** "AG Replica Disconnected" was a pure edge, so a replica disconnected for a week announced it once; `ag_disconnect_refire_minutes` (default 0 = off, clamped 0-1440) re-announces on the [#1674] pattern - same metric name so webhook automation keyed on it re-triggers, stamped on DELIVERY only so a suppressed alert cannot consume the window, and cleared on reconnect. Both columns ride store migration V37. - -- **Availability Groups tab in Lite, and one shared AG projection behind every surface** ([#1731]) - Lite gets the AG topology tab Darling's viewer got in #1722, reading its own local DuckDB, and the banding and card-shaping rules move into `PerformanceMonitor.Common.AgTopology` so the web dashboard, the Darling viewer and Lite all draw the same AG from one set of rules. There were two copies after #1722; there is now one, and twin test matrices in both test projects fail together if the shared rules change - which is what makes "these surfaces cannot disagree" a checked claim instead of an intention. The brushes stay per app (Common cannot reference WPF): the model carries the verdict, a `SeverityBrushConverter` carries the colour, which also removed the brush properties from the models entirely. **Lite needed its own read, and that is the part worth knowing.** The AG alert read from #1696 looks reusable and is not: it selects five columns where a topology view needs twelve, and it DROPS rows whose `ag_name` or `replica_server_name` is NULL - correctly, because those are the alert's state-key identity and a row that cannot be keyed cannot have an edge tracked for it. A topology view must do the opposite, since those NULLs appear under WSFC quorum loss, which is exactly when an operator opens the page. Lite's read also correlates its latest-snapshot MAX per server rather than globally, so a lagging instance is not erased by a livelier one's newer timestamp. Same hidden-until-AG-rows reveal as Darling's, same honest empty state, same worst-first ordering. Pointed at a secondary, Lite shows one replica and no primary - that is what that server can actually see, and saying so beats implying it knows the whole group. -- **Lite gets the Availability Group alerts, off a shared policy** ([#1726]) - Lite has collected both AG grains since [#1688] but could not tell you a replica had failed over, disconnected, fallen behind, or had data movement suspended; only Darling could. Lite now raises the same four conditions under the SAME metric names, so a webhook keyed on `AG Failover` matches whichever app sent it. The rules are shared rather than reimplemented: a new `PerformanceMonitor.Common.AgAlertPolicy` holds the readings, the metric-name consts and every pure decision, on the `ConnectionAlertPolicy` pattern - each app owns only its own edge STATE and its own delivery, which is the part that is genuinely app-shaped. Darling was refactored onto it with no behavior change (its whole suite passes untouched, which is why the lift was done as its own step). Lite reads the latest snapshot of each grain from DuckDB, each freshness-gated on its OWN collection time because the two AG collectors are scheduled independently and a stale database-grain snapshot must not be vouched for by a healthy replica-grain one; the evaluator is WPF-free so it pins directly, and delivery goes through the same mute-check and send path every other Lite alert uses, inheriting muting, silencing, the combined history row and the email/webhook fan-out. Three settings mirror Darling's V35 knobs (`notify_ag_health` on, `ag_lag_alert_seconds` 300, `ag_redo_queue_alert_kb` 0 = off), clamped to the same ranges on load AND on save so the stored value and the effective value cannot disagree. Server removal drops the AG state, or a remove-then-re-add would compare the new first sighting against the old role and page a phantom failover. Every rule the earlier AG work paid for carries over: a first sighting is a silent baseline, NULL is never a transition, and a suspended row may raise an alarm but may never clear one - including that a suspended secondary drifting past the threshold still fires. Lite's collector-coverage ratchet went red on this exactly as designed - both AG tables were allow-listed as collect-only, and adding a reader forced the entries out - so that allow-list is now empty. - -- **Availability Groups tab in the WPF viewer** ([#1722]) - the AG topology on the client's primary surface, completing #991's reader half. The desktop twin of the web dashboard's Availability Groups page: one card per AG with per-replica chips (role, connected / operational state, synchronization health, availability + failover mode, endpoint on hover) over a database grid carrying synchronization state, log-send and redo queue sizes, send / redo rates, secondary lag, derived drain estimates, and the suspend reason when data movement has stopped. Every severity is computed in the reader and the XAML only binds the brush it produced, so the tab cannot drift from the web page's verdict. **One card per (reporting server, AG), deliberately not merged** - the same rule the web page follows, and the live AG fixture shows exactly why: the primary reports both replicas and names the primary, while the secondary reports ONE replica and no primary at all, because `sys.dm_hadr_availability_replica_states` returns only local information when queried on a secondary. Collapsing those two views would let the blind one overwrite the complete one, so the header states groups, reporting servers and views as three separate counts rather than leaving a reader to wonder why one AG name appears twice. The tab ships hidden and reveals itself once a sweep finds AG rows (Always On is opt-in; most fleets would otherwise carry a permanently empty tab), converging without a restart and never vanishing again mid-look if a later sweep reads zero. The reader is a COPY of the service's `DarlingAgReader` rather than a reference - the viewer has no ProjectReference to the headless service - so the banding rules are duplicated deliberately and pinned by tests on both sides to keep them honest; the viewer's copy reads bare table names (search_path resolves `collect`) and joins the enabled registry so a disabled server's AGs leave with it. This also retires the last two entries in the viewer-coverage ratchet: `ag_replica_states` and `ag_database_replica_states` were carried as tracked debt when #991 shipped collection-only, and the pin that tracked them is the same pin that now certifies the surface exists. Lite gets no AG tab in this pass - it is a single-server app and the fleet view is Darling's job. -- **The bundled PostgreSQL runtime now proves its own version instead of being taken at its word** ([#1713]) - `fetch-pg-runtime.ps1` has pinned PostgreSQL 18.4 + TimescaleDB 2.28.1-for-PG18 since #1690, but the `pg-runtime.zip` sitting beside it was assembled from the 17.10 download **before** those pins were bumped, and nothing anywhere compared the two - so a bundle everyone described as PG 18 booted `starting PostgreSQL 17.10` in the field, and inspection could never have caught it (same filename, same layout, same size to the nearest tens of MB). Two new guards close that gap from both sides: an **ungated** test parses the real fetch script (delivered beside the test binary by Link+copy, so it can never read a stale second copy) and pins that `$pgVersion`'s major is the intended 18, that `$pgUrl` actually names `$pgVersion` (they drifted apart once already - a "18.4" script fed a 17.10 bundle), and that `$tsUrl` targets the matching PostgreSQL major and TimescaleDB version, catching a half-finished pin bump on the pull request that introduces it; and a **DARLING_TEST_PGRUNTIME-gated** test asks `pg_ctl.exe --version` what the *assembled* runtime really is and requires it to equal both the script's pin and the intended major, plus requires `timescaledb--.sql` and `timescaledb-.dll` to be present so a stale TimescaleDB payload (the #1705 class of drift) fails the same way. The gated half runs wherever the runtime is already extracted - CI's darling-pg job and nightly both set the variable - and it is `--version`, not a boot, so it costs milliseconds. Verified in both directions: green against a freshly assembled 18.4 tree, and red (`Expected: 18, Actual: 17`) against the exact stale artifact that shipped. **No runtime, packaging, or configuration behavior changes** - this is the pin that makes the version claim self-verifying, not the version move itself. Also recorded for the record, because the issue's premise did not survive contact: **zstd is not a TOAST compression method in PostgreSQL 18 or 19** (`default_toast_compression` accepts `pglz` and `lz4` only; the server rejects `zstd` outright), so the "better blob compression" driver behind the 18 move does not exist - and `default_toast_compression = lz4`, the strongest codec PostgreSQL does offer, has been in the v1 conf block all along, measuring 10.84% smaller than pglz on 85 real plan-XML documents - -- **AG fixture evidence: state the commit-time conclusion the guidance rests on** ([#1709]) - [#1708] corrected the AG lag guidance to "do not derive lag from commit times at all", and cites `tools/ag-fixture/VALIDATION.md`. The file carried both halves of the argument but in separate sections, never drawn together - so the reader who found one half could reasonably conclude the other case was safe to guard against. It now states them side by side: on a **suspended** replica `last_commit_time` freezes, so `now - last_commit_time` stops growing exactly when replication stops (**silent**, understating at the moment it is worst); on a **healthy but quiet** database nothing commits, so the same delta grows without bound (**loud**, measured 1757 seconds at zero real lag). Guarding only the suspended case does not make a commit-delta trigger safe, it converts a silent failure into a noisy one - and the loud half is the likelier to ship, because it appears the first time anyone points it at a database nobody is writing to, whereas the silent half needs someone to suspend something. Evidence only; no behavior change. -- **AG fixture evidence: the frozen suspended-row surface, and a lag correction** ([#1707]) - [#1702] and [#1703] both cite `tools/ag-fixture/VALIDATION.md` for how a suspended row behaves, and the file did not yet contain the measurements behind several of those claims. This lands them, with one **correction to my own earlier entry**: [#1704] said `secondary_lag_seconds` latched late (`0` at +15s, then climbing), which implied a short bounded window. On a group with **no write load** it can never latch at all - reproduced independently here, `0` at every sample across a full 60-second suspension while already `NOT SYNCHRONIZING` with the last hardened log **~29 minutes** stale. One earlier idle run did latch at +30s and these did not, so the timing is not dependable in either direction and nothing should be built on it. Stated plainly in the file: **a lag threshold alone cannot detect suspended data movement on a quiet group**; the dedicated suspended-state alert owns that case. Also landed, with sample tables: all four `*_time` columns freeze at their last pre-suspension instant (so a cross-replica commit-time delta stops growing exactly when replication stops); `redo_queue_size` freezes rather than growing; `est_redo_completion_time_min` holds a small static reassuring value throughout a suspension (0.0144 min under load, 0 when idle) because both its queue and rate are frozen, making a suspended replica look *healthier* than a working one; `last_received_time` read NULL in every sample on both runs; and `last_commit_time` is not a heartbeat - measured **1757 seconds behind wall clock on a `SYNCHRONIZED`, non-suspended, zero-lag replica** simply because the database was quiet, so `now - last_commit_time` is not a lag measure and grows without bound on a healthy idle replica. Evidence only; no behavior change. -- **AG fixture evidence: what `secondary_lag_seconds` actually measures on a suspended row** ([#1704]) - the collector doc block and the suspended-row alert rule both now cite `tools/ag-fixture/VALIDATION.md` for the claim that the column reports staleness rather than time since suspension; this lands the measurements behind it. Two parties reproduced the accrual on the fixture but from different starting points - 0 under write load, ~3993 on an idle group - and re-testing the idle case explicitly reconciles them: while movement is ACTIVE it reads 0 no matter how long the group has been idle (measured 0 with 373 seconds since the last hardening), and once SUSPENDED it latches onto roughly `now - last_hardened_time` (measured ~450 against 424) and climbs from there. Same behavior, different bases. Also recorded: it does **not** latch the moment movement stops - at +15s suspended it still read `0` while already `NOT SYNCHRONIZING` - which is what makes "a suspended row may raise an alarm but may never clear one" the only safe rule, and the reason the magnitude is staleness rather than volume at risk (`log_send_queue_size` would be the volume measure, and it is NULL while suspended). That late-latch turned out to understate the problem; see [#1707] for the quiet-group case where it never latches at all. Evidence only; no behavior change. -- **Darling Web: an "AG Health" seed notebook** ([#1699]) - the fifth Custom Views v2 seed template, built on the Availability Group measures from [#1688] and [#1695]. Five panels in diagnostic order: secondary lag over time grouped by replica; a **dual-axis** line putting log send rate against redo rate on their own axes (the one panel that answers "which side is the bottleneck" - send outpacing redo means the secondary is receiving faster than it can replay, and failover time is growing); a **stacked** redo-queue series by database; a **stat tile** for the worst estimated redo drain; and a top-10 bar of send-queue backlog by database. The prose between panels carries the two things that make the numbers readable rather than merely present: point the view at the PRIMARY (a secondary only ever sees its own replica in `sys.dm_hadr_*`, so a secondary-scoped view is a one-row self-view), and a blank drain estimate means there is no drain rate - idle, caught up, or suspended - not that it drains instantly. Also adds the two template helpers the existing seeds never needed, `overlay` support on the time-series builder and a scalar `statPanel`, so this is the first seed exercising dual-axis, stacked and stat modes end to end. **The templates drift-guard got a real hole closed along the way**: it hand-mirrors each template's panels for validation, but nothing checked the mirror COVERED every template, so a sixth template added without a mirror entry would have gone silently unvalidated until it 400'd in someone's browser. It now reads the template keys out of `notebook.js` and requires the mirror to match exactly, verified by renaming a key and watching it go red. -- **AG latency: commit-time columns, drain-time estimates, and the primary-side perfmon counters** ([#1695]) - completes the Availability Group latency picture [#1688] started. `ag_database_replica_states` gains **four DMV timestamps the reference query skips** (`last_commit_time`, `last_hardened_time`, `last_redone_time`, `last_received_time`) - unlike `secondary_lag_seconds` these are directly comparable across replicas, which is what the canonical primary-vs-secondary commit-time lag math needs - plus **two server-computed drain-time estimates**, `est_redo_completion_time_min` and `est_send_drain_time_min` (queue / rate / 60). Both estimates carry the two guards the raw expression needs: `* 1.0` stops BIGINT-over-BIGINT integer division flooring a sub-minute drain to zero, and `NULLIF(rate, 0)` stops the divide-by-zero an idle or suspended replica would otherwise raise - which would fail the whole cycle, not one column. A NULL estimate honestly means "no drain rate" and is never coerced to 0, which would read as "drains instantly". They are computed per row at the sample's own instant rather than composed later as a ratio of two window averages, because avg(queue)/avg(rate) is not the average of the per-sample ratios and the two diverge worst exactly when rates swing. Both are Gauge compose measures (duration family, native minutes). **Two perfmon counters** join the existing whitelist - `Transaction Delay` and `Mirrored Write Transactions/sec` - carrying the PRIMARY side of commit latency (their ratio is the average delay per mirrored transaction); zero new schema, they ride `perfmon_stats`. Verified on SQL2022 that both exist even with no AGs configured, live on `SQLServer:Database Replica`, and occur exactly once server-wide, so the collector's counter_name-only filter cannot collide. Together with `HADR_SYNC_COMMIT`, which already flows through `wait_stats`, sync-commit pressure is now composable end to end: primary-side delay, secondary-side queues and rates, drain estimates, and the wait itself. **A shipped doc claim is corrected against a live AG.** [#1688] restated MS Learn's assertion that `secondary_lag_seconds` reads 0 while data movement is suspended; a Docker AG fixture measured the inverse on SQL Server 2022 in a `CLUSTER_TYPE = NONE` group - it reads 0 while movement is ACTIVE and caught up, and accrues monotonically once suspended (0 to 62 s across a 60 s `SUSPEND_FROM_USER`, back to 0 on resume). So a suspended replica does not hide as zero lag and a lag threshold fires on its own; reading `is_suspended` alongside explains WHY lag is climbing rather than catching lag that is masked. Two further measured quirks are now documented: `log_send_queue_size` goes NULL while suspended while `redo_queue_size` FREEZES at its last value (so a redo-queue reading on a suspended replica is stale, not current), and collecting from a SECONDARY yields a one-row self-view because `sys.dm_hadr_*` carries only the local replica there - a complete AG picture requires collecting from the primary. Store migration V36 appends the six columns additively rather than widening V34 in place, because V34's `CREATE TABLE IF NOT EXISTS` is a no-op on an already-migrated store and editing it would silently leave every existing store short the new columns while fresh installs got them; the schema pin now reconstructs the current shape from V34 + V36 and compares it to the generator, with both failure modes confirmed by planted defects. Hannah Vernon's SqlServerAgMonitor is now credited in `THIRD_PARTY_NOTICES.md` alongside the existing collector-header attribution. -- **AG collection: document the second grant it needs, and make the lag trap filterable** ([#1691]) - review follow-ups to [#1688]. **The grant is the one that matters in the field.** Both AG collectors join the `sys.availability_groups` / `sys.availability_replicas` CATALOG VIEWS to the `sys.dm_hadr_*` DMVs, and while the DMVs are covered by the `VIEW SERVER STATE` the product asks for, [the catalog views require `VIEW ANY DEFINITION`](https://learn.microsoft.com/en-us/sql/database-engine/availability-groups/windows/monitor-availability-groups-transact-sql) - which catalog views enforce by HIDING ROWS, not by raising an error. So on a fully configured AG cluster a monitoring login with only the documented grant returned zero rows, which is exactly what an AG-less server returns: the collectors would have looked healthy forever while collecting nothing, with no error anywhere to notice. Now called out in both READMEs (the Lite/Darling grant script, the Darling permission table's If-missing column) and in both collector headers, along with the fingerprint that identifies it if it is ever worth detecting automatically - the DMV returning rows while the catalog view returns none is unambiguous, and `SERVERPROPERTY('IsHadrEnabled')` is readable by every login. **The documented lag trap is now actionable instead of just documented**: `secondary_lag_seconds` reads 0 rather than NULL while data movement is suspended, so a suspended replica charts as perfectly healthy - but nothing exposed the suspension state to a panel, so the misread the code comment warned about was unavoidable. `synchronization_state_desc` and `suspend_reason_desc` are now compose dimensions, so a lag panel can filter suspended replicas out or group by suspend reason. (`is_suspended` itself cannot be a dimension: the compiler binds filter values as text, which would not match a boolean column.) **And the V34 migration is now genuinely pinned.** Its test asserted only that each column NAME appeared somewhere in the DDL - a V34 with the columns reordered, `is_local` typed `text`, or a spurious `NOT NULL` passed every test, while a comment claimed the shape was pinned elsewhere. It now compares the whole generated `CREATE TABLE` via the existing `Migrations_JobHistoryAndAgentStatus_MatchGeneratedFreshShape` idiom; detection power confirmed by planting `is_local text` and watching it go red. That test's long-standing rationale was corrected too - it blamed a "positional binary COPY", but `PgCollectorRowWriter.CopyCommandFor` emits a named column list so Postgres binds by name; names and types are the real hazard on that side, and the positional appender is Lite's. -- **Darling: Availability Group alerts - failover, replica disconnected, sync fell behind, database suspended** ([#1692]) - the alert half of #991, over the collectors [#1688] landed. Four conditions on the headless service's existing per-server self-alert sweep: **`AG Failover`** (Warning - a replica's `role_desc` changed since the previous sweep), **`AG Replica Disconnected`** (Critical - `connected_state_desc` crossed into `DISCONNECTED`, with an informational **`AG Replica Reconnected`** on recovery), **`AG Sync Fell Behind`** (Warning - a secondary past `ag_lag_alert_seconds` or `ag_redo_queue_alert_kb`), and **`AG Database Suspended`** (Warning - `is_suspended` false to true, carrying `suspend_reason_desc`). The metric names are webhook automation keys and are consts rather than inline literals, and all five are registered in the shared `AlertSeverity` map so an alert-history replay that reaches the map without an explicit severity override does not render INFO-blue. State is keyed per AG grain (ag+replica, ag+database+replica) rather than per server, so two lagging databases on one host track and recover independently, but every alert still FIRES under the real `server_id`, keeping the per-server delivery-mode override, mute rules and history correlation intact - the grain lives in the alert text. **First sighting of any replica or database is a silent baseline** and **a NULL state string is never a transition**: under WSFC quorum loss the AG catalog views serve only locally cached metadata, so any column can read NULL, and treating that as an edge would spray alerts across the whole fleet at the moment the cluster is already in trouble. The sync decision returns THREE states rather than a bool, which is the load-bearing detail: MS Learn documents `secondary_lag_seconds` reading `0` (not NULL) while data movement is SUSPENDED, so the seconds trigger has to abstain on a suspended row - and if abstaining were a plain "not behind", the caller would read it as recovery and announce *"AG Sync Recovered - has caught up with the primary"* in the same sweep that reported the database suspended. Only a database a sweep actually MEASURED as caught up resolves a standing alert, which also makes cross-server resolution structurally impossible instead of something a scoping check has to catch. The redo-queue trigger keeps judging while suspended, because that backlog is real and still growing. Store-backed settings on `config_alert_settings` (Darling store migration **V35**, every column `NOT NULL DEFAULT`): `notify_ag_health` (default on - a fleet with no AGs collects no AG rows and is silent anyway, and an operator who does run AGs should not have to find a switch to be told about a failover), `ag_lag_alert_seconds` (default 300, clamped 0-86400) and `ag_redo_queue_alert_kb` (default 0 = off, clamped 0-1073741824 - a healthy redo queue size is entirely workload-specific, so a shipped guess would page half the fleet on day one). All three are read live through the same by-reference settings seam a store reload hot-swaps, editable from the viewer's Settings window, and the whole AG read is skipped when the master switch is off, so an AG-free fleet pays nothing on the sweep. Deliberate limits, stated rather than hidden: Lite collects both grains but has no AG alerts yet; every replica is visible from every node, so a 3-node AG with all 3 nodes monitored reports a role change once per monitored server; and `AG Replica Disconnected` is a pure edge with no [#1674]-style re-fire. -- **Availability Group health collection in Lite + Darling** ([#1688]) - closes #991 and the AG item of #1606, the one real coverage gap the Datadog DBM comparison turned up. Two new shared collectors, both server-scope and both zero-cost on a server without Always On: **`ag_replica_states`** (replica grain - role, operational/connected state, recovery and synchronization health, availability and failover mode, endpoint URL) from `sys.availability_replicas` joined to `sys.availability_groups` and `sys.dm_hadr_availability_replica_states`, and **`ag_database_replica_states`** (database grain - synchronization state, log send and redo queue sizes, send and redo rates, secondary lag, suspension state and reason, and both LSNs) from `sys.dm_hadr_database_replica_states`. The metric surface follows Hannah Vernon's [SqlServerAgMonitor](https://github.com/HannahVernon/SqlServerAgMonitor) (MIT), attributed in both collector headers. `WHERE COALESCE(is_distributed, 0) = 0` keeps distributed-AG container rows out while member AGs still appear; drilling into a DAG's remote members needs a connection per member and is out of scope. Runs everywhere except Azure SQL DB, which has no AG surface - an AG-less on-prem server is deliberately NOT gated off, it collects and stores zero rows, so turning Always On on later starts producing data with no configuration change. Three details that are easy to get wrong and are pinned by test: the queues, rates and lag are **instantaneous gauges, not counters**, so neither collector touches the delta framework and the compose measures are Gauge (avg/min/max, never SUM - summing a backlog over a window is a category error); `last_hardened_lsn` / `last_commit_lsn` are `numeric(25, 0)`, **wider than BIGINT**, so they are converted server-side and stored as text rather than silently overflowing; and every column is read null-tolerantly, because under WSFC quorum loss `sys.availability_replicas` serves only locally cached metadata and `endpoint_url` is documented NULL. Both apps schedule it per minute with 30-day retention (the grain at which a lag or queue spike is still visible), and it is in all three cadence presets (per-minute on Aggressive and Balanced, 5 min on Low-Impact). **Custom Views v2** gets five Gauge measures on the database grain - send queue, redo queue, send rate, redo rate and secondary lag - under an Availability Groups category, sliceable and groupable by AG, database and replica plus the synchronization/suspend state (and the universal server dimension), which lights up send-rate-vs-redo-rate on a dual axis and worst-lag-per-replica out of the box. Only the database-grain table carries measures - the replica-grain table is all state strings with nothing numeric to aggregate, so it is stored for the coming viewer tab rather than queryable from Custom Views today. Darling store migration V34; Lite's storage registers itself off the collector catalog. Collection only in this first cut: no viewer tab, and no alerts (the alert family landed straight after in [#1692]), and both apps' viewer-coverage ratchets carry the two tables as explicitly tracked debt so the tab cannot be forgotten. That ratchet also got a real fix along the way - it text-scans the viewer's reader layer for a table name, and naming a collector table as a schema-version probe sentinel (as V34 does) made the table read as "already covered", silently exempting it; the scan now strips the probe's `information_schema` lines, which retroactively hardens the same pin for `long_query_completions`. -- **A Docker Availability Group fixture, and what it found about the AG DMVs** ([#1689]) - the AG DMVs return nothing at all on a standalone instance, so AG collection has never had anything to be validated against. `tools/ag-fixture` stands up two SQL Server 2022 containers as a clusterless (`CLUSTER_TYPE = NONE`) availability group - shared endpoint certificate, mirroring endpoints on 5022, automatically-seeded `AgFixtureDb` - in one idempotent `setup.ps1`, with a write-load script to make the queues and rates move and a suspend/resume recipe to fault it. Primary on `localhost,14331`, secondary on `localhost,14332`. Both AG collector queries (#991) were run against it verbatim and validated, including a suspend fault; evidence is recorded in `tools/ag-fixture/VALIDATION.md`. Three behaviors it surfaced that the DMV documentation does not prepare you for: **a secondary reports only itself** - `sys.availability_replicas` holds every replica on both nodes, but `sys.dm_hadr_availability_replica_states` and `sys.dm_hadr_database_replica_states` hold only the local one on a secondary, so a complete AG picture requires collecting from the primary and monitoring only a secondary yields a one-row self-view; **`log_send_queue_size` reads NULL while data movement is suspended** rather than growing, with `redo_queue_size` frozen at its last value, so a send-queue threshold is blind to a suspended secondary and `is_suspended`/`suspend_reason_desc` are the signal for it; and **`secondary_lag_seconds` accrues while suspended** (measured 0 -> 15 -> 31 -> 46 -> 62 across a 60-second suspend, back to 0 on resume), which is the inverse of MS Learn's documented "this value shows as 0 if the data movement is suspended" - it reads 0 when movement is active and caught up, not when it is suspended. Measured on `16.0.4265.3`, clusterless AG only. The fixture is deliberately sized to share a machine with a VM fleet rather than to perform: 1 CPU and 2 GB per container with the engine held to 1536 MB, no SQL Agent, 64 MB database files, and a periodic overwriting log backup in the write-load script - without which a single 90-second run took the log to 1.6 GB, since an AG database must stay in FULL recovery and cannot truncate its own log. Measured steady-state usage is recorded alongside the rest of the evidence. -- **Availability Group topology, in the browser and over MCP** ([#1690]) - closes the reader half of #991, on top of the two AG collectors: a new **Availability Groups** page in Darling Web and a new **`get_ag_health`** MCP tool, both served by one shared reader (`DarlingAgReader`) off `GET /api/ag`, so an agent and a browser see identically banded numbers. Each card is ONE monitored server's VIEW of an AG - replicas as chips (role with the PRIMARY highlighted, connected/operational state, synchronization health), then a database grid carrying synchronization state, log-send and redo queue sizes, send/redo rates, secondary lag, and the suspend reason when data movement has stopped. An AG whose replicas are all monitored appears once per monitored replica, each naming its reporting server: the perspectives genuinely differ (the DMV populates `operational_state` and `recovery_health` for the LOCAL replica only, `connected_state` only from the primary, and a quorum-lost instance answers from cached metadata), so they are deliberately not merged into a consensus the DMVs never agreed on. Two judgement calls are worth knowing about. `SYNCHRONIZING` is banded against the replica's availability mode, not flat: it is the correct steady state for an ASYNCHRONOUS_COMMIT secondary (async never reaches SYNCHRONIZED) and a warning on a SYNCHRONOUS_COMMIT one, so a healthy async fleet is not painted amber. And suspended data movement outranks every state string, because `secondary_lag_seconds` reads 0 - not null - while suspended, which makes a stopped replica look perfectly caught up on lag alone. The drain estimates (`est_send_drain_minutes` / `est_redo_completion_minutes`) are derived queue/rate, null when a non-empty queue is moving at zero rather than a misleading 0 or an infinity. Severities are computed server-side and the page only maps a severity name to a CSS class (R1), all text goes through `textContent` (R4), and the nav entry stays hidden until the store actually has AG rows - Always On is opt-in, so most fleets would otherwise carry a permanent dead link; the `#/ag` route works either way and shows an honest empty state. -- **Darling Web: the reporting layer grows scatter, dual-axis, and brush-zoom** ([#1683]) - the three charting primitives the composer lacked (#1606; "investment in reporting layer is crucial"). **Second measure (`overlay`)**: a panel may carry one more same-source measure - it compiles as a second select expression over the same fact rows (the ratio two-operand precedent generalized: never a join, never a parameter, one query), validated by the exact same aggregate/unit rulebook as the primary and gated to the shapes it belongs to (rejected anywhere else, never silently ignored). **Scatter**: a ranked panel plots one point per group - x ranks by the primary, y is the overlay - with per-point hover titles and drill-to-filter clicks; the workhorse for spotting outliers (executions vs. duration per query hash). **Dual-axis line/area**: an ungrouped time series draws the overlay against its own right-hand axis (own nice scale, own unit caption, dashed line in a reserved color) so CPU%% and batch/sec read together without either flattening the other. **Brush-zoom**: dragging across any time-series chart re-RUNS the panel on the brushed window as an absolute range - the auto bucket re-resolves, the retention tiers re-route, and the partial-window notice stays honest - with a reset chip that survives empty and error results (a historical zoom can legitimately land on one coarse bucket). The run endpoint gains `windowStart`/`windowEnd` (ISO-8601, precedence over `hours`, 90-day span ceiling, reject-not-clamp), which also makes HISTORICAL windows first-class for the MCP `run_custom_view_panel` tool. Under the hood the run context's `NowUtc` decouples from the window end - routing and the retention notice measure age from actual now, pinned by a 30-to-25-days-ago window routing to the daily rollup. CAGG routing takes both measures through one gate: if either can't remap to the rollup columns, the whole panel reads raw. Composer UI: a Second-measure picker on scatter/line/area, scatter in the chart list with the same inline coherence hints the server enforces, and stored definitions round-trip the overlay through edit-and-save. Pinned by a 9-case overlay validation matrix, compile pins (AS value2, zero extra parameters, ranked scatter shape, both-remap-or-raw), a 4-case absolute-window rejection matrix, and the NowUtc routing pin. -- **CI: every `report.*` view now EXECUTES against seeded rows, not just exists** ([#1677]) - the SQL-validation workflow checked the report views by `OBJECT_ID` only, which let two bugs of the same class ship: #1635's per-row bit->nvarchar conversion error and #1666's binary(8) boxed raw into sql_variant - both compile fine and only misbehave when rows flow through the projection. A generated seed script now puts one representative row in every collect/config table after the fresh CI install (NOT NULL columns get type-appropriate values, nullable columns stay NULL so those branches run, compressed-LOB columns get real COMPRESS() payloads, and the change-history tables get correlated toggle PAIRS so the LAG-based views emit actual rows), and a sweep then materializes every `report.*` view with `SELECT * INTO` - COUNT(*) would let the optimizer prune the projection and skip exactly the per-row conversions the sweep exists to catch. Views are enumerated from `sys.views` at run time, so new views are covered the day they ship and the dynamically-built `report.query_snapshots` pair is simply absent on a fresh install rather than a hardcoded failure. Failures list every broken view with its error and fail the job. Detection power proven with a planted per-row Msg 245 view: green sweep without it, red with it. Runs across the whole 2017/2019/2022/2025 matrix; a regeneration dev-tool script rides along for future schema changes. -- **Connection alerts for servers that are already down, and re-alerts during a standing outage** ([#1674]) - closes #1659, the gap split out of #1535: connection alerts were pure edge detection, so an app or service that started while a server was already unreachable never announced the outage (no edge existed), and a standing outage produced exactly one alert however long it lasted - which silently broke the reporter's webhook-driven auto-heal loop the day the app restarted mid-outage. Two OPT-INS, both default-off so the classic one-alert-per-outage behavior is untouched: **alert at first sight** (announce a server already down on the first-ever observation) and **re-alert every N minutes while still down** (0 = off). Re-fires deliver under the SAME `Server Unreachable` metric name deliberately - webhook automation keyed on the metric re-triggers, which is the whole point - with the detail text marking the flavor (`Already unreachable when monitoring started` / `Still unreachable (re-alerting every N min)`). The decision is ONE shared definition (`ConnectionAlertPolicy` in PerformanceMonitor.Common, replacing Lite's `ConnectionEdgeDetector` and the inline machine in Darling's `DarlingSelfAlertEvaluator` - the `SqlErrorClassification` discipline, pinned from both test suites), and the two opt-ins interlock: even with the startup announcement off, re-fire alone re-announces after a mid-outage restart, because the re-baselined outage has no recorded down alert and is due immediately. The re-fire clock stamps on DELIVERY only, so an alert suppressed by the notify toggles never consumes the window. Lite: two settings beside the existing connection toggle (settings.json + Settings window). Darling: V33 store columns on `config_alert_settings` (read live like the V20 toggle; refire clamped 0-1440), editable from the viewer's Settings window; no ACL/provisioning change (the table carries table-level grants, no column carve). -- **Darling: `--configure-network` can now expose the WEB DASHBOARD** ([#1617]) - the wizard offered Store and MCP but not the web dashboard, even though `--enable-web`'s own output told operators to run `--configure-network` to expose it on the LAN - a dead end that forced hand-editing `web.network` into darling.json. Web is now a first-class third surface, fully symmetric with Store/MCP: its own menu choice (plus comma combinations like `1,3`, and `4` = all three), a keep-or-generate DPAPI access token, listen/CIDR inputs validated by the SAME bind resolver the web host fail-closes on (extracted as `ResolveWebBind`, the web twin of `ResolveMcpBind` - never a reimplementation), the comment-preserving `web.network` write, a one-time token print, and next-steps text including the browser login URL (`http://:/?token=...`, exchanged for a session cookie). Disable now removes all three network blocks. After the wizard, `--enable-web` opens the scoped firewall rule on the first try - no hand-editing required. -- **Darling Web: adaptive `auto` time bucket for composed time-series panels** ([#1619]) - a compose custom-view time-series panel can set `timeBucket: "auto"`, and the compiler resolves it to a concrete grain from the panel's window (minute up to 2 days, hour up to 60 days, day beyond) so any range from 1h to 90d renders a readable line and never trips the 5,000-bucket ceiling. Previously a fixed `hour` bucket collapsed a sub-hour workload (e.g. a 30-minute HammerDB run) into a single invisible point, while a fixed `minute` bucket errored past ~3.5 days. The composer now defaults new time-series panels to `auto` and the MCP `describe_custom_view_catalog` recommends it; non-auto buckets compile byte-for-byte as before. Pinned by `DarlingComposeTests` (auto boundary resolution + compile-by-window + non-auto passthrough). -- **Darling Web: custom time span on the view range picker** ([#1619]) - the rendered custom-view range picker gains a "Custom..." option (a number + a hours/days unit, up to the 90-day window ceiling) so a view is no longer limited to the six presets; a non-preset window stays selectable so it survives the 60s refresh. -- **Darling: hourly continuous aggregates for query_stats + procedure_stats (composer query acceleration)** ([#1621]) - the two collector tables that dominate the store (~90%: query_stats ~145 GB, procedure_stats ~49 GB) now have hourly TimescaleDB continuous aggregates (`query_stats_hourly` / `procedure_stats_hourly`) that pre-materialize the exact shape every Custom Views composer panel over them does (`date_trunc('hour', collection_time)` + `SUM(delta_*)` GROUP BY a dimension), grouped by the same `MeasureCatalog` dimensions with SUM/MIN/MAX per delta column + a sample_count (so avg composes correctly at query time). Created idempotently as runtime setup in the worker's TimescaleDB block (not a versioned migration), `WITH NO DATA` with real-time aggregation on and the conservative hourly refresh policy (start_offset 3d, end_offset 1h). Measured ~49x / ~32x row reduction on the field fleet. Query acceleration only - raw rows still exist for the ~2-day hot window (analyze_*_plan needs the plan XML), and this makes no retention decision. -- **Darling: three-tier retention - downsample past the raw hot window, keep the rollups** ([#1623]) - builds the retention layer on the hourly CAGGs (#1621): instead of a flat "drop raw after N days," three tiers so nothing is fully lost, only coarsened. **Raw 4 days** (`add_retention_policy` on query_stats / procedure_stats / query_store_stats - one day past the hourly CAGG's own 3-day refresh window). **Hourly CAGGs 21 days** - adds `query_store_stats_hourly` (built now, ahead of a writable-Query-Store primary; empty on a read-only replica until one exists) plus 21-day retention on all three hourly CAGGs. **Daily CAGGs kept indefinitely** - new HIERARCHICAL `query_stats_daily` / `procedure_stats_daily` sourced from the hourly CAGGs (not raw; 2.28.1 supports CAGG-on-CAGG), 1-day cadence, no retention policy. Each tier's horizon stays past the next tier's 3-day refresh start so a drop never outruns the aggregate preserving it; `add_retention_policy` is a chunk-level drop, so no off-hours window needed. All idempotent runtime setup in the worker's TimescaleDB block. `query_store_stats_daily` deferred until QS-hourly has real data to source from. Cold-start caveat on existing stores: backfill the hourly CAGGs past the raw horizon before the 4-day raw drop's first run (fresh installs are safe automatically). -- **Darling Web: composed panels read the CAGG rollups for old windows instead of truncating at the raw horizon** ([#1625]) - the read side of the retention tiers (#1623): once the 4-day raw drop runs, a Custom Views panel looking back further than 4 days would show only 4 days (the older history lives in the hourly/daily CAGGs the composer never read). The compiler now routes by the window's oldest-point AGE - recent -> raw, hour-age -> the hourly CAGG (21d), day-age -> the daily CAGG (indefinite) - and rewrites the measure's aggregate to the CAGG's pre-aggregated columns (`SUM(delta_x)` -> `SUM(x_sum)`, `AVG` -> `SUM(x_sum)/SUM(sample_count)` exactly, Sum-ratios mapping both operands). Age-based (not display grain, not window span) so Ranked/Scalar panels and historical windows route correctly; the display grain clamps up to the tier's grain since a rollup can't render finer than it was materialized. Covers query_stats, procedure_stats AND Query Store (whose execution-weighted mean reconstructs exactly from #1624's reshaped weighted sums, validated on 159M live executions); only query_stats `object_name` (a #1568 module join) falls back to raw. Every existing panel is byte-for-byte unchanged inside the raw horizon. This unblocks activating retention at the field box - the raw drop coarsens old windows instead of truncating them. -- **Darling Web: Query Store panels route past the 21-day hourly horizon (query_store_stats_daily)** ([#1626]) - completes the CAGG read-routing (#1625): adds the hierarchical `query_store_stats_daily` (sourced from the reshaped `query_store_stats_hourly`, same composer dims + weighted sums, kept indefinitely), so a QS Custom Views panel looking back past the hourly CAGG's 21-day retention routes to the daily rollup instead of capping. The mapper reads the daily's identical columns unchanged. Removes the last read-routing TIER gap; the one remaining dimensional exception, query_stats `object_name`, is closed by [#1627]. -- **Darling Web: composed `object_name` panels on query_stats route to the CAGG rollups too — the last read-routing exception, closed** ([#1627]) - #1625/#1626 left exactly one dimension on raw: a Custom Views panel grouping query_stats by `object_name`. object_name is not a query_stats column - it is stitched read-time from procedure_stats via `sql_handle` (the #1568 module join) - so those panels fell back to raw and truncated at the 4-day horizon while every other dimension reached the 21-day/indefinite rollups. Closed by carrying `sql_handle` into the query_stats hourly + daily CAGGs and adding a retained `collect.module_map` (a compact `sql_handle -> object_name/database/schema` map refreshed at startup and daily from procedure_stats, so the mapping survives the raw 4-day drop). On a CAGG route the compiler now LEFT JOINs `module_map` for the attribution (the RAW path still uses the live window-bounded #1568 procedure_stats CTE, since raw procedure_stats is retained), so `object_name` resolves as `m.object_name` either way and object_name panels reach the hourly (21d) and daily (indefinite) tiers exactly like query_hash. The CAGG reshape self-heals in place - an existing `query_stats_hourly` without `sql_handle` is dropped (CASCADE takes `query_stats_daily`) and rebuilt by the ensure sweep, so a field box upgrading over #1621/#1623 CAGGs reshapes with no manual steps. Nothing composable now falls back to raw for lack of CAGG coverage. -- **Darling viewer: organize the fleet into nested tags** ([#1640]) - the sidebar server list can be grouped into user-authored tags (folders in the sidebar): a server carries any number of them and tags nest up to four levels. Fully opt-in - the list stays a plain flat list until the first tag is created, so an untagged viewer sees no change. Favourites float to a pinned group at the top (collapsing a tag can never hide a starred server) and an Untagged group always accounts for the rest of the fleet; a server carrying multiple tags appears under each. Assign inline from a server row's right-click "Assign Tags" submenu (checkable, indented by nesting), do tag CRUD from a tag header's right-click (New / New Child / Rename / Delete, disabled on the Favorites/Untagged pseudo-groups), or open "Manage Tags..." for a bulk editor - a tag tree with create/rename/delete/nest on the left and a searchable per-tag server checklist with Add-all / Remove-all on the right for first-time tagging of a large fleet. Expand/collapse state persists across restarts (viewer preferences). The projection is a WPF-free `FleetView` rendering headers + servers as one mixed list through a template selector, pinned by `FleetViewTests` (opt-in flat default, favourites/nesting/untagged, collapse hiding descendants, multi-tag duplication, selection skipping headers, and cycle / dangling-parent safety). Built on the V32 `server_tags` / `server_tag_map` config-plane tables. - -### Changed - -- **Darling store: `maintenance_work_mem` raised to the measured compression floor, and existing stores actually get it** ([#1780], closes [#1777]) - TimescaleDB's compression sort runs on `maintenance_work_mem`, not `work_mem`, so that one setting gates how fast the background compression job moves. Measured on a production field instance (16 GB RAM class), three real points during a one-time catch-up of large backlog chunks: at the old formula's landing point (~800 MB) compression moved **9.1 MB/s** of uncompressed input; at 1536 MB it moved **15.5 MB/s**, with a pure-linear null hypothesis predicting 2833s against an actual 1657s - a real effect, not noise; at 4096 MB it moved 16.1 MB/s, so going 2.7x further past 1536 bought nothing measurable. The formula becomes `min(max(5% RAM, 1536MB), 25% RAM, 2048MB)`. The **1536 MB floor is the fix itself**: the old `min(5% RAM, 1 GB)` landed *under* its own cap on a 16 GB host (819 MB), so raising the cap alone would have changed nothing there. The **25%-of-RAM** term is the small-host guard and the **2 GB cap** bounds the big-RAM case at the point the data stopped improving. Landing values are 1024MB / 1536MB / 1536MB / 1638MB / 2048MB on 4 / 8 / 16 / 32 / 64 GB hosts, each row pinned by a test because each one is a different term winning. - - **A formula change alone would have reached only a fresh `initdb`, and every store that needs this is already collecting.** So the change ships with its propagation half: a **v7 marker block** on the same versioned-append pattern v2 through v6 already use, re-stating `maintenance_work_mem` alone the way v5 re-states `shared_buffers`. `postgresql.conf` takes the LAST assignment of a setting, so an existing store adopts the raised value with the v3 block never rewritten - and because the append runs before `pg_ctl start`, it applies on that very start rather than one restart later. Proven end to end against a real PostgreSQL 18.4 + TimescaleDB 2.28.1 store, not asserted: a store was provisioned normally, rewound to its pre-change shape (v7 block removed, v3's value set back to `819MB`), read live at `819MB`, then restarted through the production bootstrap and read live at the new value, with the healed conf showing the legacy line still present and simply outvoted. Every guard was verified RED by mutation: reverting the formula took 10 tests red including every landing row; disabling the heal branch took the propagation E2E red with the existing store keeping 819MB; and dropping the 25% guard took **only** the small-host rows red, leaving 8/16/32/64 GB green, which is the guard proving its own scope. **Honest limits, both carried from the issue:** the three field points are different tables at different sizes rather than a controlled experiment, so the exact threshold between ~800 MB and 1536 MB is unknown and the controlled repro stays open; and the floor raises small hosts proportionally more than the 16 GB class it was measured on (4 GB goes 204 -> 1024 MB). That second one is bounded rather than hand-waved - `maintenance_work_mem` is a per-operation ceiling and not a reservation, PostgreSQL grows the allocation to fit the work, a small host's chunks are small, and PG 17+ (the bundle pins 18.4) made vacuum's dead-TID store grow incrementally instead of allocating the limit up front. One thing operators should not misread: PostgreSQL normalizes units on the way out, so `SHOW maintenance_work_mem` reports `1GB` on a 4 GB host and `2GB` on a 64 GB host. Same setting, different string. That is also why the tests compare bytes via `pg_size_bytes` - the first version compared strings and went red with `Expected: "2048MB", Actual: "2GB"` against a real server - -- **CI: a dev/main push now cancels that branch's superseded in-flight build — newest SHA wins** ([#1729]) - [#1715] scoped cancel-in-progress to pull requests and deliberately left every push run uncancellable; 2026-07-26's ~20-merge evening measured what that costs at train speed: of the day's 30 dev-push `build.yml` runs, **13 were superseded mid-flight** (a newer merge landed before the run finished) and **~60 runner-minutes went to SHAs that were already stale** — capacity the shared Windows runner pool bills against every queued PR (#1697's original complaint). Push events in `build.yml` and `sql-validation.yml` now share a per-branch concurrency group with cancel-in-progress, so only the newest head keeps building. "Safe" was verified from the workflow graph, not asserted: a push run produces nothing any other run consumes — every `upload-artifact` in `build.yml` is gated to the release event (the SignPath path) or `failure()` (darling-pg diagnostics), `sql-validation.yml` uploads nothing at all, no `download-artifact` / `gh run download` / `workflow_run` consumer exists anywhere in the repo, nightly builds its own tree from its own checkout, and a release compiles fresh on the `release` event. Release and merge-queue runs keep their unique per-run groups and remain uncancellable: a release waits on SignPath's manual approval gate, and a queue validation is the last check before its result lands on dev. The accepted trade, stated rather than hidden: push builds are diff-scoped, so a cancelled run's areas are not re-verified until the next change touches them — the nightly and the all-areas dev→main release PR are the backstops. The merge queue that would eliminate the train itself stays unavailable on this personal-account repo ([#1716]: organization-owned repositories only, `422 Invalid rule 'merge_queue'`, re-verified against GitHub's GA announcement); this is the slice of that win that IS available here. -- **CI: build.yml handles the merge_group event** ([#1716]) - lands what [#1715] named as the merge-queue prerequisite: without a `merge_group` trigger, the required `build` and `Darling PostgreSQL tests` checks would never report inside a queue and every queued PR would stall. Queue runs take the same always-restore path as dev/main pushes - a queue run is the last validation before its result lands on dev - and the per-run concurrency group, so they are never cancelled or replaced. Path classification works unchanged in a queue: dorny/paths-filter v4.0.1+ resolves merge_group diffs from the payload's base_sha/head_sha whenever the `base` input is empty, which is exactly what the filter steps pass for non-push events. **Post-merge correction to this entry's original "safe one-click" claim: the click does not exist here.** Merge queues are available only on ORGANIZATION-owned repositories (public on any plan, private on Enterprise Cloud - per the GA announcement, and confirmed empirically: creating the ruleset on this personal-account repo returns `422 Invalid rule 'merge_queue'`). The `.gitattributes merge=union` alternative for the CHANGELOG conflict trains is also out: GitHub's server-side PR merging ignores merge attributes (community discussion #9288), so it would only automate local resolution, not the DIRTY state or the re-push. The wiring stays - inert, zero cost, live the day the repo ever moves to an organization - and until then the train-tax relief is [#1715]'s classification fix: single-area re-pushes re-run ~3m40s instead of ~6m30s, docs-only re-pushes 13s. -- **CI: the change classification the v4 pin bump silently broke is restored - and this time it is probe-validated** ([#1715]) - a timing baseline over the 30 most recent build.yml runs plus a throwaway probe PR (#1714) turned the planned build-time audit into a regression find. The 2026-07-26 08:02 pin bump moved dorny/paths-filter v3 -> v4, and v4 evaluates every filter pattern as an INDEPENDENT predicate under its default predicate-quantifier 'some' (a filter is true when any changed file matches at least one rule), so a bare `!**/*.md` exclusion stopped being a subtraction and became its own rule: "any file that is not markdown". Every area filter carrying that line went true for ANY non-markdown change anywhere in the repo - a single root .gitignore edit built and tested all four products (probe run 30219202642), darling-pg ran the full TimescaleDB suite on every PR since the bump including md-only ones (run 30218459544 matched CHANGELOG.md against the darling filter), and the [#1712] docs fast path shipped unable to engage, because every file matches '**' so its code: gate was never false - its measured md-only 1m43s runs were real, but they were the area filters at work (markdown matches no include), not the fast path. The regression was invisible by construction: the bump's own PR touched build.yml, so root=true forced a full build that looks identical to a correct run, and so does every over-built run after it. Fixed keeping v4 (v3 is on the deprecated-runtime track): area filters carry the markdown carve-out INSIDE each include as an extglob (`Darling/**/!(*.md)`) where quantifier semantics cannot detach it; the uninvertible code: filter is replaced by an `all:` counter with docs-only decided by all_count == docs_count; the classify step additionally refuses to engage while any area filter is lit, so the two classifications can never disagree into a `dotnet build --no-restore` with no restore behind it; and check-version-bump.yml, which had the identical '**'-plus-exclusions shape and an equally dead md-only skip, takes the same counter fix. Validated with a per-file truth table on the probe PR (run 30219765613: a Darling .txt probe, a Darling .md probe, .gitignore and the workflow file in one diff - each filter's matched-file list recorded in the PR body). **Also in this pass:** a PR re-push now cancels that PR's superseded in-flight build.yml/sql-validation.yml runs, while push and release runs keep unique per-run concurrency groups - never queued behind or cancelled by anything, every dev/main commit keeps its own check result; the [#1712] allowlist's flagged judgment calls are ratified (CITATION.cff, Screenshots/) but its directory-wide grants become extension-explicit (`docs/**/*.{md,svg,png,jpg,jpeg,gif}`) so a .sql dropped into docs/ tomorrow defaults to code; the scheduled nightly re-dispatches itself onto the dev ref instead of doing real work from main's copy of the workflow file - scheduled workflows execute the DEFAULT branch's copy against dev's checked-out tree, which is exactly how the 2026-07-26 06:00 nightly failed (run 30194606068: main's stale copy read Dashboard/Dashboard.csproj, moved to deprecated/ by #1612 - the #1550 trap again) - so after a ONE-TIME sync of nightly.yml to main (command in the PR body; the schedule stays red each morning until it happens) nightly logic changes take effect the night they merge to dev, with manual dispatches still always building and the artifact job still pinned to dev; and every job that never carries the release/signing path gets a timeout-minutes ceiling at ~3x its worst cold path (darling-pg 30, nightly build 90 / pg 60 / check 10, sql-validation 30 per leg, claude-review 30) so hung-not-slow failures stop holding a shared-pool runner for the 6h default - build.yml's build job stays unbounded on purpose, because the release path waits on SignPath's manual approval gate. **Measured and deliberately not done** (numbers in the PR body): per-area restore splitting (warm restore is 19-33s; four condition-mirrored restore steps buy seconds at the price of the drift risk [#1701] just retired) and cross-job test splitting (Run Lite tests ~2m25s dominates the full build, but a second Windows job costs ~2m45s of checkout/setup/restore/build before its first test - a net loss on a shared serialized pool). Merge queue remains a recommendation with exact settings in the PR body: it is a repo setting, and build.yml needs a merge_group trigger first or queued PRs stall on never-reporting required checks. -- **CI: a documentation change stops paying for a .NET restore** ([#1712]) - non-executable changes now skip .NET setup, restore and versioning in the required `build` job, while the job still RUNS so the check reports and cannot block a merge. The gate is an explicit **allowlist** of non-executable content (`**/*.md`, `LICENSE`, `CITATION.cff`, `.gitignore`, `.gitattributes`, `docs/**`, `Screenshots/**`) rather than a subtraction, so an unfamiliar new file type defaults to being treated as code - the safe direction to be wrong in. `*.sql`, `*.yml`, `*.csproj`/`*.props`/`packages.lock.json` are deliberately excluded from the allowlist because some job compiles or consumes each of them, and `*.cs` is excluded however comment-only a change looks, since the compiler is what proves it still builds. **Two guards against under-building**: the fast path never engages on a `release` event or on a push to `dev`/`main` (those restore unconditionally - it only forces the restore back on, per-product build/test steps stay path-gated exactly as before), and both jobs now emit a `::notice::` naming WHY they took the path they took plus the file list classified as documentation, because a job that reports success having quietly run nothing is indistinguishable from one that tested everything. Measured against the real gap: markdown-only changes were ALREADY skipping every heavy step (PR #1707, pure `.md`, ran `build` in 1m43s), so what this actually fixes is non-markdown documentation - a `LICENSE`, `.gitignore` or screenshot edit previously matched the catch-all and paid a six-project locked-mode restore for nothing. The remaining ~1m45s floor is `actions/checkout` on a Windows runner, not work this gate can remove. -- **CI: the Lite fast / analysis-heavy test split collapses back into one step** ([#1701]) - the split existed because the seven analysis classes rebuilt the full DuckDB schema inside every test and their subset alone cost ~9 CI minutes, so a narrower lite_analysis path gate let non-analysis Lite changes skip it. After [#1693]/[#1694]/[#1698] that subset runs in ~66s on the same runner, so the split stopped earning its second test-host spin-up - and its hand-maintained class-name filters were a standing drift risk (a renamed analysis class would silently fall out of the heavy filter into the fast bucket). One Run Lite tests step now runs the whole suite behind the lite path gate; the unconsumed lite_analysis filter block is gone. The lite gate is a strict superset of the old lite_analysis gate, so nothing that ran before is skipped now. -- **Lite.Tests: the shared DuckDB fixture extends to 20 fast-bucket classes** ([#1698]) - rounds out [#1693]/[#1694]: a full-suite trx profile showed 400s+ of serial weight remaining in fast-bucket classes with the same per-test disease (a fresh DuckDB + full schema build inside every test; per-row single-connection seeding in several). All 20 eligible classes now take the class fixture with a data-only reset per test - including the four appender round-trip classes that previously built one database PER TABLE PER TEST - with the special cases handled rather than papered over: FindingStoreTests' v3->v4 analysis-schema migration test hand-creates the legacy table shape and therefore keeps its own private database file while its 15 siblings share; the two MCP classes keep a test-local temp dir for the ServerManager config directory; the server-time-helper collection trio keeps its collection attribute. Six classes are deliberately NOT converted because each needs a private database by design - ArchiveViewDedupTests (parquet + archive-view rebuilds), MuteRulesSurviveResetTests (ArchiveAllAndResetAsync deletes the database file), DuckDbSchemaTests (tests schema creation itself), and the three TestAlertDataHelper consumers (the helper writes parquet into the archive dir and rebuilds views) - that list is the output of reading every candidate class end to end, not a guess. Measured: total serial work across the suite 925s -> 760s (IndexObjectStats 60.6s -> 20.8s, SystemEventsReader 36.2s -> 13.8s, PerformanceCalendarData off the top-15); the local wall is parallelism-bound on a many-core box, but CI's 2-core runner is serial-bound and gets the elimination nearly 1:1. Full suite run twice consecutively: identical 1494/1494 results, zero warnings. -- **Lite.Tests: seeding is batched - one connection and one transaction per seed helper instead of a connection and an auto-commit per ROW** ([#1694]) - the profiling follow-up [#1693] demanded. With the schema builds gone, a per-test trx profile showed the analysis-heavy filter was bounded by seeding I/O: every seeded row opened a fresh DuckDB connection and auto-committed a single INSERT, measured at ~90ms/row, and AnomalyDetectorTests (whose tests seed 100-200 baseline rows apiece through its private helpers) was the ENTIRE critical path - 154.5s serial against the filter's ~154s wall. Three mechanical changes, no test-body semantics touched: AnomalyDetectorTests and BaselineProviderTests reuse one per-test-instance seed connection for every row, with AnomalyDetectorTests' seven SeedBaseline* loops each inside one BEGIN/COMMIT; TestDataSeeder holds one lazily-opened connection per instance and every seed helper wraps its inserts in a SeedBatch (BEGIN on construct, COMMIT on dispose) so a helper's rows share one WAL flush - except ClearTestDataAsync, deliberately un-batched because its per-table try/catch DELETEs would abort an explicit transaction on the first missing table and silently skip the rest of the clear; and TestDataSeeder is now IDisposable with all 48 construction sites taking using var seeder, which also hands the speedup to the four seeder-using classes outside the heavy filter (DatabaseFilter, FactScorer, FinOps, FindingStore) for free. Measured locally on an idle box: the analysis-heavy filter 164s -> 19s, the FULL suite 233s -> 78s, per-class serial ceilings from 154.5s down to 20s. For the record, #1693's own CI run showed the fixture change alone was net neutral on the runner (612s step vs the 556s reference, with the fast step drifting +10% identically) - the per-row seeding was the bottleneck all along, and this is the change expected to collapse that CI step. Isolation re-proven under the new transaction semantics: batches commit before any read, a mid-batch failure cannot mask the test's real exception, and the full suite ran twice consecutively with identical 1494/1494 results. -- **Lite.Tests: the analysis-heavy classes share one DuckDB per test class instead of rebuilding the schema per test** ([#1693]) - the seven classes behind CI's "Run Lite analysis-heavy tests" step (AnomalyDetector, FactCollector, FactCollectorMisery, BaselineProvider, InferenceEngine, Scenario, AnalysisService) each created a fresh DuckDB database inside every test and laid the entire Lite schema down first - 36+ collector tables plus indexes, the v_ archive views, and the analysis schema, ~80 DDL statements - so 153 tests (10% of the suite) carried a redundant schema build apiece, at ~1.4s each on a contended box and ~3.6s each on CI's small runner (the 556s step this targets). A new `SharedDuckDbFixture` (xUnit `IClassFixture`) builds the database once per CLASS, and each test starts by resetting DATA only: every base table is emptied except the `schema_version`/`analysis_schema_version` stamps, so the database keeps reading as current rather than as a blank file needing migration, and a stray re-init stays a fast no-op. The reset list is a runtime `information_schema` snapshot, not a hand-maintained list - the AG collector tables that landed on dev mid-branch were covered with zero fixture changes. Deliberately one fixture per class rather than a collection fixture: a single shared collection would SERIALIZE the seven classes against each other and hand back most of the win, while per-class databases keep xUnit's cross-class parallelism intact. Isolation was proven rather than assumed: the reset covers ALL base tables including `analysis_findings`/`analysis_muted`, which the seeder's server-scoped clear never touched (persisted findings and mute rules can no longer leak between tests); none of the seven classes mutates schema, touches parquet, or resets the database file (verified by reading all seven end to end); and the full suite ran twice consecutively with identical results (1479/1479 both runs; 1494/1494 after merging dev's AG work). `TestDataSeeder` is untouched - its per-scenario `ClearTestDataAsync` is load-bearing for tests that re-seed mid-test. Measured locally: analysis-heavy filter 233s -> 175s under a contended box, 164s -> 159s on an idle one - the win scales with contention, which is exactly what CI is. Also removes the ~60 per-test `InitializeAnalysisSchemaAsync` calls that were duplicate work even before the change (`InitializeAsync` already ends by calling it). Net -187 lines. -- **Quarterly maintenance pass: dependency bumps, a zero-warning build restored, repo hygiene** ([#1643]) - the first formal maintenance sweep. **Dependencies** (all patch/minor; `--vulnerable --include-transitive` and `--deprecated` both report zero across all 20 projects): Microsoft.Data.SqlClient 7.0.1 -> 7.0.2, Microsoft.Data.SqlClient.Extensions.Azure 1.0.0 -> 7.0.2 (Microsoft re-versioned the companion package onto the SqlClient 7.x line), Microsoft.Extensions.* 10.0.9 -> 10.0.10, System.Security.Cryptography.ProtectedData 10.0.0 -> 10.0.10, ModelContextProtocol + AspNetCore 1.4.0 -> 1.4.1, ScottPlot.WPF 5.1.58 -> 5.1.59, Microsoft.NET.Test.Sdk 18.6.0 -> 18.8.1, and DuckDB.NET 1.5.2 -> 1.5.3 (validated by running `tools/CompactionRepro --synthetic` - the real production parquet-compaction merge - against the new engine); lock files regenerated with `--force-evaluate`. **Zero-warning build restored**: ~280 accumulated warnings fixed across every project - the big one being CS8629 at ~100 MCP tool call sites in all three apps, where `ServerResolver.ResolveOrError`'s nullable-tuple shape hid the "error null implies resolved non-null" invariant from the compiler; all three resolvers (Dashboard, Lite, Darling) now return a non-nullable `resolved` that is `default` on error, and every call site drops `.Value`. The rest: single-char `StringBuilder.Append` via char alias consts (CA1834), concrete return types on private helpers (CA1859, including the four parallel `BrushFromHex`/`MakeFrozen` copies), cached `CompositeFormat`s in the two template-formatting collectors (CA1863), platform-guard restructuring so the analyzer can prove the DPAPI and Event Log calls Windows-only (CA1416), ordinal string comparisons (CA1310/CA2249/CA1865), an `IViewerServerSecretStore.Get` -> `Find` rename (CA1716), a redundant `HasReportXml` re-declaration removed (CS0108), a latent null-deref guard in the Dashboard actual-plan path (CS8602), and xUnit assertion modernizations (Assert.True/Single/DoesNotContain forms). Two documented suppressions, not silences: CA1720 on the collector column/parameter type enums (naming SQL types is their job) and xUnit1051 in the test projects (the ambient-CancellationToken style nag, suppressed like CS4014). **Hygiene**: 27 dead `worktree-agent-*` branches, a closed-PR checkout, and two merged feature branches deleted; fork remote pruned. The `vpk` CLI pin was verified to match the Velopack `PackageReference`. The pinned GitHub Actions are NOT current - checkout v5 (latest v7), setup-dotnet v5 (v6), cache v4 (v6), upload-artifact v6 (v7), paths-filter v3 (v4); only the SignPath action is current at v2. Those are major bumps with a release-critical blast radius (upload-artifact feeds artifact ids straight into the SignPath signing steps, and v7 introduces non-zipped artifacts), so they are deliberately NOT bundled into a dependency PR - tracked separately. - -- **CI actually runs the deprecated Dashboard's build and tests again** ([#1643]) - #1612 defined a `dashboard` path filter when the Full Dashboard moved under `deprecated/`, but no workflow step ever consumed it, so Dashboard.Tests stopped building and running in CI entirely. Deprecation was supposed to mean bug-fix-only, not unverified - and the gap was self-concealing: it is exactly why ~126 compiler warnings accumulated there unseen, and why all three `ThemeParityTests` had been hard-FAILING since the move (`FindRepoRoot` still looked for `Dashboard\Themes`, which is now `deprecated\Dashboard\Themes`, so every run died on the repo-root assert rather than comparing anything). Both are fixed: the test resolves the deprecated path from a single constant and genuinely compares the palettes again (768/768 pass, no drift found), and the workflow gains the missing restore/build/test steps gated on the existing filter. Dashboard.Tests also gets a `packages.lock.json` so it restores `--locked-mode` like every other CI root. This guard is not Dashboard-only in effect: the palettes it checks are **Lite's too**, so it was Lite's cross-app theme-drift protection that had silently stopped running. - -- **Full Dashboard and the CLI Installer are being retired.** They moved to a `deprecated/` folder and are no longer built into release artifacts; releases now ship Lite + Darling only, with Darling the flagship replacement for the Full Dashboard. The deprecated code still compiles and its tests run in CI. ([#1612]) -- **Darling Web: the custom-view panel grid is capped at two columns on wide screens** ([#1619]) - a dense view (e.g. four panels) no longer crams four-across on a wide monitor; the min column track is >=45% of the row so at most two columns ever fit, and a `span-2` (or lone) panel still takes the full width. -- **Darling: the query_store_stats + procedure_stats continuous aggregates are reshaped to the composer's dimensions** ([#1624]) - the #1621/#1623 CAGGs grouped query_store_stats by Query Store's native query_id/plan_id (which the composer never uses - it reads QS by module_name/query_hash) and dropped procedure_stats' schema_name, so a composed panel over those dimensions could not route to the rollup, and the QS execution-weighted mean was unreconstructable (it stored avg-of-avgs). Regrouped query_store_stats_hourly to server/database_name/module_name/query_hash carrying execution-weighted sums (so the weighted mean composes exactly), and added schema_name to the procedure_stats hourly + daily CAGGs. Reshaped now while the CAGGs are empty (QS is read-only on the current replica) or a day or two old, via a guarded drop+recreate that detects the old shape structurally. Matters ahead of a writable-Query-Store primary; query_stats CAGGs are unchanged. - -### Fixed - -- **The service now hardens EXISTING config backups, not just the live config** ([#1818], closes [#1816]) - verified in the field: `darling.json.bak-*` files created before the backup-creation path learned to lock its output (#1786) kept whatever ACL the folder handed them - on the reporting box, inherited `BUILTIN\Users` read. Against machine-scoped DPAPI blobs that is full exposure: any local account could recover every monitored server's SQL credential plus the MCP and web tokens. The installer's security check flagged the files but only PRINTS the fix. The config-hardening sweep now also hardens every `darling.json.bak-*` sibling at service start (`allowInteractiveRead: false`, matching the creation path - backups are rollback artifacts nothing interactive reads), with the live file's same best-effort-with-runnable-remediation contract and the same still-readable CRITICAL backstop - so one service start closes the exposure on every affected install with no human in the loop. Pinned by ACL tests against real files: both backups stripped of the inherited read, the live file left to its own hardening, a lookalike backup of some other file untouched, and a no-backup start a clean no-op - watched red by disabling the sweep's hardening call. -- **The dimension GC's floor measurement now survives compressed history** ([#1817], closes [#1815]) - the first field run of #1813 timed out on its very first sweep: TimescaleDB compressed chunks carry no btree indexes, so the V39 partial indexes only serve uncompressed chunks, and against months of compressed history the exact `min(collection_time) WHERE digest IS NOT NULL` probe was a full decompress-scan that the driver's 30-second default cancelled (the probe had also shipped without the explicit command timeout every sibling statement carries). The unmeasurable-floor fail-safe did exactly its job - deferred instead of guessing - but that meant the headline #1795 behavior never engaged on exactly the store class it was built for. On a hypertable the floor now comes from chunk catalog metadata: the oldest surviving chunk's `range_start`, one instant read, compression-immune, and conservative in the SAFE direction - every surviving fact row sits at or above its chunk's range_start, so the cutoff can only land older than strictly necessary (prunes less, never dangles), and precision improves automatically as purges and backfill advance the chunk floor. The exact V39-indexed probe remains for plain-PostgreSQL stores and unconverted tables, now under the sweep's own generous timeout. Pinned by a live test that compresses the digest-carrying chunk with the product's own compression settings and then requires the sweep to MEASURE (no deferral, no per-table warning), reclaim the ancient orphan, keep the referenced content - and keep a dim row placed in the band between the chunk floor and the exact digest floor, which only the catalog bound retains: its survival is the proof of WHICH path measured, watched red by disabling the catalog branch. - -- **A stale running_jobs snapshot no longer re-fires the same Long-Running Job alert forever** ([#1814], closes [#1812]) - all three editions read the LATEST running_jobs snapshot with no freshness guard, exactly as reported: a stopped collector, missed cycles, lost msdb access, or the 512 MB reset (with Parquet still serving the newest rows through the archive view) left a stale "latest" that read as NOW. The repetition at cooldown intervals had a second cause the reporter could not see: the per-run cooldown key ({server}:{job}:{start}) is deliberately dropped once it ages past the cooldown, so the stale rows re-armed and re-fired the same historical run every cooldown, forever. - - The fix treats a stale snapshot as NO evidence, in both directions. The shared adapter contract now carries evidence quality (`AnomalousJobsResult`): Lite and Darling probe MAX(collection_time) first and return the stale result - rows skipped entirely - when the newest snapshot is older than the freshness bound, and the shared engine then skips BOTH branches: no fire (the reported defect) and no fabricated "Long-Running Jobs Cleared" off a collector that merely stopped reporting, with the active state left alone so a returning fresh snapshot resumes real evaluation and fires or resolves from actual evidence. The bound derives from each server's EFFECTIVE running_jobs cadence (three missed cycles, floored at 10 minutes), supplied by the two construction sites that know it - Lite's ScheduleManager, Darling's resolved schedule reading the live overrides so a control-plane reload reaches the very next check - because the shipped profiles span 2 to 30 minutes and a fixed bound is either blind for the fast profile or a gag for the slow one (pinned: a 2-hour-old snapshot is stale at the default cadence and CURRENT at a 60-minute one). The Dashboard's own SQL-side read gets the same protection in its house idiom (a fixed 10-minute `DATEADD` bound matching its Agent-driven cadence); its legacy loop keeps its simpler shape, so the one behavioral delta is documented rather than hidden: on a freshly-dead collector it may emit one final "cleared" where Lite and Darling now stay silent. Engine behavior is pinned by a harness test that walks the exact reported sequence - live fire, collector dies, two cooldowns pass with no re-fire and no fabricated resolution, fresh-and-empty evidence returns and the REAL resolution fires - and the adapters are proven against real stores (DuckDB fixture; the live Postgres leg ages the seeded snapshot in place), with the bound watched red by mutation. - -- **The dimension GC prunes against the oldest SURVIVING digest-carrying fact row, not an assumed horizon** ([#1813], closes [#1795]) - the proper closure of the #1782 x #1784 interaction those issues shipped with: query_stats and procedure_stats are both coverage-gated tiers AND dim-feeding tables, so on a coverage-lagging store the #1784 clamp held their purges every sweep, the #1782 guard read that as "purge did not complete" and deferred the whole dimension GC - every sweep, until a backfill landed. Proven by execution: a 400-day-old orphan dimension row survived with nothing failed anywhere. The deferral was logically correct (pruning content the held facts reference would dangle digests and the resolving view would serve NULL payload silently) but blunt: it stopped the GC entirely rather than narrowing it. - - The true safety boundary is measured, not assumed: content older than the oldest surviving digest-carrying fact row cannot be referenced by anything, whatever the reason those facts are still there. The GC now probes each dim-feeding table's floor - `min(collection_time)` under exactly the predicate of a new V39 partial index per table, an index-edge read once per sweep rather than the oldest-chunk walk the pre-#1767 NULL-digest rows would force, the bounded shape the last_seen watermark design (#1768) demands - takes the minimum across tables (the same reasoning as the widest-retention horizon), and clamps the assumed cutoff to one day before it, the same margin the assumed side carries for the hourly last_seen refresh guard. On a healthy store the measured floor sits inside retention and the assumed horizon rules unchanged; on a held store the GC stays bounded instead of stopped, reclaiming content for queries that stopped running long before while everything the held facts reference survives. The blunt guard is thereby unnecessary rather than dormant - it survives in exactly one honest form: a floor that cannot be MEASURED (the table missing or unreachable) still defers the cycle, because pruning on an unknown boundary is the one way to dangle digests, and bounded growth beats silent corruption. The probe predicate, the V39 index predicate, and the dimension map are pinned to each other three ways, so a new dimension column lands in all of them or none. The viewer's schema ladder gains the matching V39 arm (index-existence sentinel, the V22 idiom) so a fully-migrated store maps to exactly the required version instead of tripping the connect gate. - - Chasing the live proof surfaced a REAL test-fixture defect worth its own record: `EnsureContinuousAggregatesAsync` attaches refresh policies whose jobs fire immediately (the #1788 finding), and the class's deep force-refresh collided with them (55P03) while a restore's DROP could collide with a running job's lock and silently strand `query_stats_db_hourly`/`db_daily` in the shared fixture - stranded aggregates whose deep coverage then flipped the #1784 gate for every LATER test, the same manufactured-flake mechanism #1794 documented for connection debris. The tests never needed the scheduler (they refresh manually): the ensure wrapper now removes every rollup's refresh policy immediately, and the one force-refresh that can still catch an already-executing job retries bounded on 55P03 only - the product's own #1788 idiom. Three consecutive full-class live runs leave zero stranded aggregates. - -- **Recommendations warm-up now survives the 512 MB archive/reset, and analysis reads the archive tier everywhere** ([#1811], closes [#1809]) - the 24-hour sufficiency check measured MIN..MAX(collection_time) on the RAW hot wait_stats table, while `ArchiveAllAndResetAsync` copies everything to Parquet and empties the hot store. On a multi-server install that trips the size threshold more often than daily, the measured span restarted with every reset and warm-up never completed - even though the archived history was sitting in Parquet and every other tab could read it through the `v_` union views. The check now reads `v_wait_stats`, so archived plus hot history counts, and a genuinely young install still gates (both pinned, watched red by reverting the query to the raw table). - - The subtlety that made this a 24-file-line sweep rather than a one-word fix: the broken gate was accidentally SHIELDING a second instance of the same defect. The analysis fact collector read raw tables in 22 more places - the window reads (query stats, snapshots, memory, perfmon, file IO, blocking, deadlocks, storage) and the on-load config snapshots (server/database config, trace flags, server properties), which after a reset are EMPTY until the next app start. Fixing only the gate would have run analysis over a thin post-reset hot window, reintroducing the exact fraction-of-period distortion the 24-hour floor exists to prevent ("5 seconds of THREADPOOL looks alarming in a 16-minute window"). All 23 sites now read their `v_` archive views - column-identical by construction (only `config_alert_log` carries a view-only column, and analysis does not read it), dedup-safe where re-collection could duplicate rows (the QUALIFY views), and a catalog-driven sweep guard holds the whole pipeline there: for every table in `ArchiveService.ArchivableTables`, a raw `FROM` anywhere in Lite/Analysis fails the build's tests naming the file - so a new collector's table is guarded the day it exists. -- **Live-Postgres test cleanup no longer depends on the connection the failure just destroyed** ([#1810], closes [#1794]) - three serialized live-store classes (`PayloadDimensionLiveTests`, `DarlingSelfAlertTests`, `DarlingMcpJobToolsLivePostgresTests`) intermittently failed with `Connection is not open` on reused databases, and the issue's diagnosis held: not contention, not the #1776 parallelism class - a lifecycle defect, in every case the same shape. Each test's `finally` ran its restore/delete statements on the BODY's connection, which is the one object the failure being reported may have destroyed (a broken socket, a taint from a failed COPY, a cancellation mid-command all leave it closed). That did three bad things at once: the cleanup threw `InvalidOperationException` from a helper frame - the exact reported signature, with the nightly's occurrence landing in `DeleteDimRowAsync` - the throw REPLACED the body's in-flight exception (a finally-throw wins), so the real failure was never seen, and every cleanup statement after the first was abandoned, leaving renamed tables, aggregate snapshots, and dim rows behind in the reused store to manufacture the next run's "unrelated" flakes. One test's finally restores `collect.query_stats` from a mid-test rename; abandoning THAT leaves the store without its central table for everything after. - - All twelve cleanup sites now run through a shared `LiveStoreCleanup` helper: a freshly-opened connection (with `search_path` SET explicitly, because a pooled physical session opened before the store's first migration keeps the pre-`ALTER DATABASE` default for its whole life - the 42P01 trap the atomic-visibility test already documents), `CancellationToken.None` (a cancelled run must still restore the store; the body's token is by definition already signalled on that path), and body-aware masking: when the body already failed, a cleanup exception is swallowed so the REAL failure surfaces; when the body succeeded, cleanup failures propagate loudly. Deliberately no retry anywhere - the standing house rule for this flake class, and the issue's own remedy note. Verified against a live store: seven consecutive green runs of all three classes, including two immediately after a cluster restart, and the gated tests still skip cleanly without `DARLING_TEST_PG`. - -- **A benign 1-second lock-timeout yield no longer paints a Critical health day** ([#1808], closes [#1805]) - exactly one collector carries `SET LOCK_TIMEOUT 1000` (query_snapshots, both variants), and yielding after 1 second instead of joining a blocking chain is the monitor keeping its never-be-a-blocker promise; the data cost is one skipped point-in-time sweep. But nothing anywhere handled SQL error 1222 specially, so the yield surfaced as a generic collection error - and the daily-health banding treats ONE collection error, alone, as a CRITICAL day, so each yield painted the calendar red over a non-event. The fix is classification only; the collection behavior was already correct. A new `YieldsOnLockTimeout` flag on the shared collector definition (true only for query_snapshots, so a 1222 from any other collector - which never sets the guard - stays an ERROR) routes both runners' catch sites to a new `YIELDED` collection_log status. The exclusion needed no reader changes at all: every error count that feeds collector health, the fleet roll-ups, and the daily band's collErrors input matches `status = 'ERROR'` exactly, audited across both apps - so the pins guard the predicate SHAPE (exact-match, in both daily summaries) against ever being broadened to a not-success form that would re-admit yields. In-memory health treats a yield as neither success nor failure: it does not reset a real error streak (a yield is not proof the collector works) and does not grow one (the guard worked as designed). The re-frame the issue asked for is kept visible: yields get their own counter - a Yields column on both Collection Health grids and a `yields` field on both `get_collection_health` MCP tools - because a cluster of them is evidence about the TARGET's lock contention, mislabeled until now as a monitoring failure. Deliberately unchanged: the timeout itself, no retries, no extension of the guard to other collectors, and the health-band classifier - a collector yielding for hours goes honestly stale with the Yields column sitting right beside the staleness explaining why. - -- **The .NET SDK band is now pinned three ways instead of agreeing by float** ([#1807], closes [#1758]) - the issue predicted CI would break when runner SDKs moved to the 10.0.3xx band, because the committed lock files carried the 2xx-band `Microsoft.NET.ILLink.Tasks` pin. Investigating found the fuse had already half-fired in reverse: the 2026-07-25 maintenance pass regenerated the locks under a 3xx SDK, and CI has been green only because `setup-dotnet`'s floating `dotnet-version: 10.0.x` happens to download the same band - while `global.json` still named `10.0.204`, a version nothing was actually using. Every piece agreed by coincidence, and the NEXT band (10.0.4xx) would have broken it again from either side: a runner image float re-arms NU1004 in CI, a local `latestFeature` roll-forward silently rewrites the locks. Now `global.json` names the band that wrote the locks (`10.0.302`, `rollForward: latestPatch` so a newer installed band is ignored rather than silently adopted), and all four `setup-dotnet` sites read `global-json-file: global.json` instead of floating - so the invariant the issue states (CI SDK band == `global.json` band == the band that wrote the locks) is enforced by one file, and moving bands becomes the deliberate single-PR act it should be. Verified: `--force-evaluate` under 10.0.302 changes zero lock files, `--locked-mode` restore passes on all nine, and the Installer chain builds without the lock-file churn the issue documented on exactly this machine. - -- **The payload-dimension batch upsert deadlocked against itself across concurrent collection cycles** ([#1803], closes [#1801]) - a production field instance logged nine self-healing `40P01`s in its first night on the v38 dedup write path, and zero before it; the box's own `pg.log` captured two full lock graphs showing the same statement shape on both sides, two sessions each waiting on a tuple the other held in `collect.query_text_dim`. The statement upserts a batch through `unnest`, and the arrays came out of a `Dictionary` in enumeration order - per-session ARRIVAL order, which is not a total order across sessions. Two cycles whose batches share two digests in opposite relative order each take the row lock the other needs next, and the cycle closes. **The within-batch digest dedup does not help and was never meant to**: it makes a batch conflict-free with ITSELF (the 21000 guard), not ordered against its siblings - two different properties that happen to concern the same key. - - The arrays are now emitted in ascending digest order from the one place every caller shares, which fixes `query_plan_dim` by the same edit rather than waiting for its name to appear in a log. Deliberately sorted client-side rather than by an `ORDER BY` in the statement: the lock order is then a property of the values bound, owing nothing to how the planner chooses to execute the insert, and it stays checkable without a database. Ordinal sort on the hex key IS byte-wise digest order - equal-length digests, and hex digits ascend in ASCII in the same sequence as the nibbles they encode - though the test asserts the raw bytes so the claim does not rest on the representation the implementation happens to key by. `collect.module_map` was checked and left alone: the `ORDER BY` its `DISTINCT ON` already requires is a deterministic ascending order on its conflict key, so it cannot form the cycle; that reasoning is now a comment there, because the next person to see an `ORDER BY` that looks purely about picking the latest row should not remove it. - - **The ordering is invisible to every behavioural test** - an unordered batch upserts byte-identical data, and only the lock-acquisition order differs - so it is pinned twice: a unit test that feeds the batch in deliberately reversed digest order and requires ascending bytes out, and a source pin that the sort is present at all. Both watched red on a compiling build by restoring the dictionary enumeration. **No live deadlock repro is claimed**: reproducing a two-session lock cycle on demand is timing-dependent, and a test that passes because the race did not happen is worse than no test - the property under guard is the total order, which is exactly what these two pin. - -- **`--backfill-rollups` converged every rollup to raw's oldest row, but the arming gates for the hourly tier measure the SOURCE - so a fully-DONE run could leave retention policies held** ([#1799], closes [#1798]) - all four daily rollups are hierarchical continuous aggregates reading their HOURLIES, not raw, and the [#1680] gate for an hourly-tier policy asks whether the daily covers what the *hourly* holds. On a healthy store - raw purges armed, raw a few days deep, hourlies legitimately holding weeks - a daily converged "to raw" stops well short of that, the gate correctly refuses, and the verb printed `DONE` with its zero-held promise over it; observed on the lab rehearsal box as `16/16 retention policies in place, 14 armed, 2 held` immediately after a fully-DONE run. The sharper edge: a hierarchical daily added AFTER its hourly on such a store enters a hold NOTHING can clear, because the pre-raw region exists only in the hourly and a raw-aimed verb never targets it. Convergence is now SOURCE-relative: `RollupViews` carries each rollup's source relation alongside its raw table (two different questions - `RawTable` answers where a READ falls back to, `Source` answers what the rollup is BUILT from and therefore the most history it can ever hold), and the backfill targets are DERIVED from that one list rather than restated. The read-side tier router deliberately keeps falling back to raw. - - Building it surfaced two more defects in the verb, both fixed and pinned. **The plans were all taken before any slice ran** - necessary, since the disk preflight must total the whole job before committing to any of it - but a daily's source is an hourly this same run deepens FIRST, so every daily was aimed at where its hourly USED to start and then judged against where it now starts: SHORT on every daily, every time, caught by the verb's own end-to-end test. The run loop now re-plans each rollup from live floors immediately before running it (the measured-floor idiom that makes resume work, applied one level up), and the preflight bounds each daily against `EventualSourceFloor` - the deeper of the source's current floor and raw's oldest row, which is as far as backfilling the source can ever take it - so the printed refuse-with-numbers total genuinely bounds the run (pinned on buckets and bytes, and pinned the direct way: the daily's up-front plan must span to the hourly's TARGET, not its current floor). And **a mid-run slice-ceiling refusal was reported as success**: `RollupBackfillPlan.Absurd` sets `IsComplete` as well as `Refusal`, and the re-plan branch tested `IsComplete` first, so a clamp crossing discovered mid-run - after the preflight had cleared it under the ceiling - printed "already covers" and counted toward `DONE`. `Refusal` is tested first now, routed to stderr as a shortfall with a non-zero exit; the guard stays even though today's call graph makes the case nearly unreachable, because that is a property of the call graph rather than of the code. - -- **The retention rollups only ever served what they had materialized, so old windows read empty while raw still held every row** ([#1788], closes [#1759]) - not a data-loss bug: the [#1680] arming gate held the raw purges closed on exactly the affected stores, which is why nothing was lost. The damage was that tier-routed reads returned **empty** for old windows, and that raw then grew without bound because the purges never armed. The premise the rollup tier was built on was wrong twice over. `TimescaleSupport` claimed real-time aggregation was opted into and the view was therefore "correct to query for any window immediately, just un-accelerated" - but TimescaleDB has defaulted `materialized_only` to TRUE since 2.13 (the runtime is pinned at 2.28.1), so naming no option means real-time aggregation is **OFF**; and turning it on would not have helped, because the watermark is a **hard partition** rather than a fallback - `build_union_query` emits materialized-below `UNION ALL` raw-at-or-above with no contiguity guarantee, so history below the watermark that was never materialized is served by **neither** branch. Every rollup's refresh policy starts 3 days back, so on a store that existed before its rollups the materialized span began at roughly creation-minus-3-days and never reached further back on its own. The false premise was not merely a stale comment - it was **pinned by a test** asserting the absence of `materialized_only` for the wrong reason. - - **Reads are fixed with no materialization at all**, because the held purges mean raw still holds every row. A new `RollupCoverage` probe (the companion to [#1665]'s `RollupAvailability`) reads each rollup's `min(bucket)` and each rolled raw table's `min(collection_time)` in one round trip, and `RetentionTierRouter` degrades a window to whichever tier is MEASURED to reach furthest back. **The hard half is not falling back when raw would return LESS**: on a healthy store with armed purges raw keeps ~4 days against the rollups' weeks, so a window older than every floor is the *normal* case there, and a naive "window predates the floor -> use raw" rule would have sent every long window to a 4-day table - a regression introduced by the fix, on every store that was working correctly. The rule is therefore comparative: a tier is abandoned only on a positive measurement that a lower tier is deeper, which is precisely the held-purge signature and is silent everywhere else. Unknown coverage is **inert by construction** - a failed probe, a plain-PostgreSQL store and a partially-built one all produce nulls, and nulls never move a window - so the previous behaviour is reproduced exactly. All six production routing readers are gated, including the MCP `get_daily_health` reader, which is not incidental: it answers the same question as the viewer's Performance Calendar off the same shared SQL, so gating one and not the other would have had them report **different query counts for the same day** on exactly the affected stores, with no way to tell which was right. A source-parsing guard fails the build if a future reader is added un-gated, naming the file and line. Coverage also **expires where availability does not**: availability is cached permanently once complete because a created aggregate is never dropped, which is true of existence and false of coverage - it moves backwards on a backfill and forwards on a retention drop - so keeping the `AllPresent` shortcut would have pinned a pre-backfill floor for the life of the process. And the partial-window notice now prefers the **measured** floor to the retention span, because on these stores the purges are held and raw holds months rather than its nominal 4 days; left assuming, it would have stamped "older points are not included" on precisely the fallback panels that are complete. - - **Materializing the history is a new operator verb, `--backfill-rollups`, deliberately not a startup step.** The arming gate is all-or-nothing, so a store with a year of raw must materialize the WHOLE history before the first purge arms and reclaims anything: **peak disk comes BEFORE any relief**, and doing that automatically at service start, on the worst-affected stores (one already down to roughly 150 GB free), is a plausible disk-exhaustion event. So it preflights and refuses with numbers. The size estimate is CALIBRATED from what the rollup has already materialized (bytes per bucket x buckets to add) - every affected store has that sample, since its refresh policy has been materializing a trailing 3-day window all along - and with no sample it bounds from raw's own size and says so, erring deliberately high because refusing a backfill that would have fit is recoverable and filling a production volume is not. It requires the estimate plus 25% headroom plus a 10 GB reserve, and measures free space on the volume the STORE reports (`current_setting('data_directory')`), so a store on another host refuses rather than measuring the wrong disk; the refusal names the shortfall and both real options, including that **waiting is safe** - nothing is being lost while the purges are held. The run is sliced one source chunk at a time, oldest first (supervision rather than re-implementation: a manual refresh already batches internally, so the slices buy progress on a multi-hour run, a resume point, and a lock window short enough not to sit across a compression job), **judges convergence from DATA** because a batch-cap stop is logged server-side and completely silent to the client, escalates to the forced refresh only on a measured shortfall - the one thing that repairs a hole left by an interrupted pass, whose invalidation records a plain refresh skips straight over - and is idempotent and resumable, re-planning from the measured floor every time. It **arms nothing**: coverage is the whole job, and the arming gate already releases the held policies by itself on the next start. The verb's output says so, and also warns that the hourly rollups' own 21-day policy will trim the coverage a run just built when it next fires, so the restart should not wait. - - **Three findings came out of the gated live leg** against a real PostgreSQL 18.4 + TimescaleDB 2.28.1 store, none of them visible by reading, all three fixed in the product rather than worked around in the test. **`55P03` concurrent refresh**: the verb runs while the service is UP and the aggregate's own refresh policy lands on top of a slice - reproduced immediately, since the ensure sweep attaches the policy and the policy runs at once. Retried, bounded, transient-only, so every other SQLSTATE still fails fast. **`22023` "refresh window too small"**: a day-wide slice's ragged tail is narrower than one bucket for a DAILY rollup, and the range end is a coverage floor or "now" so it lands mid-bucket most of the time - it aborted the entire daily tier; the planned range now closes on a bucket boundary as well as opening on one. **The per-slice convergence check was measuring the wrong thing**: "did the floor reach this slice's start?" fired on the first slice of *every* run, because a slice's range can legitimately hold no source rows (raw's oldest row lands partway into the first slice, and a collection gap does the same mid-run) - a global `min(bucket)` cannot answer a question about one slice, so convergence is judged once at the end against raw's oldest row. The live legs prove the defect by measurement (a window below the floor returns zero rows from the rollup while raw holds rows for it), prove Phase 1 routes it to raw and back to the rollup after the backfill, prove the healthy shape does NOT fall back, drive the **verb itself** end to end through a bring-your-own `darling.json` rather than re-implementing its loop - dry run changes nothing, the `DONE` claim is re-verified against the store per rollup, a second run reports nothing to do - and finish by watching the arming gate arm the held `query_stats` policy by itself through `EnsureRetentionPoliciesAsync`, the real seam rather than its predicate. Every guard verified RED by mutation: floor check dropped, the raw-is-deeper guard removed, a routing reader left un-gated, the preflight bypassed, convergence made call-based over a silently under-covering plan, and the hierarchical order inverted (caught by the pin AND by the live verb reporting the dailies genuinely short). The `materialized_only` pin **stays**, now load-bearing and for the true reason: the coverage probe and the arming gate both read `min(bucket)` to mean "the oldest bucket MATERIALIZED", and unioning the raw branch in would make an empty materialization report raw's own oldest row as the rollup's floor - the router would believe coverage it does not have, and the arming gate would arm a purge over history nothing else holds - - -- **The directories cluttering a field instance's install folder were invisible to the report built to find them** ([#1785], extends [#1770]) - [#1775]'s report verified SILENT on the production instance: it scans siblings of the DATA directory under `%ProgramData%`, and the seven directories are under the INSTALL directory - a different parent entirely. They would have stayed invisible even in the right place, because that report identifies a store copy structurally by its `PG_VERSION` file and the arithmetic rules clusters out (a ~286 GB data directory on a ~500 GB volume with ~175 GB free - seven copies do not fit), so they are small hand-made snapshots with no store semantics at all. A second report now covers the install directory's top level: any directory the product's layout does not account for is named with its size once per start, and **never deleted** - an absolutism that matters more here than for store copies, because this classifies by ELIMINATION against a known layout rather than by any positive test, so it will sometimes be wrong by construction and reporting is the only verdict that stays harmless when it is. It carries no store semantics either: it can only say the product did not put a directory there, so that is all it says. The known layout is enumerated from the shipped artifacts and measured (`viewer` from the packaging step, `runtimes` and the culture directories from an actual publish, `wwwroot` from the csproj, `pg-runtime` and `pg-runtime-prev` from the runtime paths); locale directories are matched STRUCTURALLY - nothing but `*.resources.dll` - rather than by culture name, because that set is decided by which cultures our dependencies localize into and a hardcoded list would have the product reporting its own shipped directory as foreign. Its own 5-second whole-report budget rather than one shared with the store report, so one report's cost can never degrade another report's content; an exhausted budget degrades a directory's SIZE to "at least" and never drops the directory itself. -- **Four test classes shared the live Postgres store without serializing against it, and three more were blamed for it by a substring match** ([#1783], closes [#1776]) - classes that read the shared `DARLING_TEST_PG` database without `[Collection("live-postgres")]` ran in parallel with the sixty-odd that have it, and with each other. Three consecutive full-suite runs against one long-lived database failed on a DIFFERENT unrelated class each time, which is the expensive shape of flake: it looks exactly like the change under test broke something it never touched. **CI cannot see it** - it creates a throwaway cluster per run - so the whole cost lands on local development and surfaces as someone else's problem in someone else's pull request, which is why it survived. The classification was verified against the source rather than taken as given, and it moved in both directions. **The most aggressive offender was not on the list**: `DarlingComposeTests` connects to the shared database and runs `PgMigrations.MigrateAsync` - DDL - against it. It does NOT simply take the attribute, because the class holds 127 tests of which 126 are pure and serializing all of them to protect one is the slowdown the issue itself warns against; the live test is split into its own class instead, the shape more than forty files here already use, which also makes the parent's existing claim that "the live round-trip is exercised elsewhere" true rather than aspirational. **Three of the six named were substring artifacts**: `DarlingManagedPostgresTests`, `DarlingStoreUpgradeTests` and `DarlingPgRuntimeVersionPinTests` never read `DARLING_TEST_PG` at all - they read `DARLING_TEST_PGRUNTIME` (and `_OLD` / `_NEWZIP`), different variables that merely share the prefix - so "do not serialize them" was the right answer reached for the wrong reason, and each now records which reason applies so the next sweep does not re-derive the false positive. Two further classes were missed and are genuinely exempt: they reach the variable only to CREATE and DROP their own database through `ScratchPostgres`. **The "transaction-rolled-back" comments those classes carried read like an exemption and are not one** - rolling back protects the DATA, not the CONCURRENCY, since an uncommitted write still holds its row locks and still fires the `config_version` bump trigger; that is now recorded on each, because the comment otherwise invites exactly the wrong conclusion. A new meta-test requires every class reading the shared store to either carry the attribute or record an `#1776 own-store` exemption - a DECISION, not the attribute, since a class with its own database should not serialize - and it matches the QUOTED literal rather than a substring, which is what keeps the three runtime classes and bare prose mentions out; matching the way the original sweep did would leave it permanently red on three innocent classes. It fails rather than skips when it cannot find the source tree, following the doc-comment pin, because a guard that silently skips is a guard that silently stops guarding. Test infrastructure only; no production code changes -- **The config backups handed a second readable copy of every secret to any interactively-logged-on user** ([#1786], closes [#1769]) - a `darling.json.bak-*` is a byte-for-byte copy of the config: every monitored server's `encryptedPassword`, the MCP bearer token, the web access token, all DPAPI **LocalMachine** blobs with an entropy constant that ships in an open-source repo, so READ access IS the secret. Both the service's CLI verbs and `install-darling.ps1` granted those backups `NT AUTHORITY\INTERACTIVE` read, mirroring the live config. **Nothing reads a backup**: the only references to the `.bak-` name in non-test code are the two lines that construct it, and restoring one is a hand operation that already requires elevation because writing `darling.json` does - INTERACTIVE only ever had Read there, never Write. The grant bought nothing. Both paths now drop it, so the installer and the service cannot disagree, and the installer's summary states which file gets which posture instead of claiming one rule for both. **The live `darling.json` deliberately keeps its INTERACTIVE grant**, and the decision about tightening it moves to [#1792] rather than closing with this: removing it would break the shipped Viewer. `ViewerSettings.TryLoad` reads the config directly, the Viewer manifest is `asInvoker`, and under UAC an interactive administrator's default token carries `BUILTIN\Administrators` as SE_GROUP_USE_FOR_DENY_ONLY - so a DACL of exactly SYSTEM + Administrators + the service identity denies it outright, and the window shows "darling.json could not be read" with no data at all. That is the DEFAULT shipped layout, where the Viewer sits in `viewer\` under the service root and probes the parent for the service's config. The security rationale still holds - the `admin` role's SELECT carves out the secret columns, so the config file really is the only interactive-readable path to the SQL passwords and tokens - but closing it needs the Viewer given a non-secret path to the `postgres` section it actually wants, which is a design decision rather than a one-line ACL edit. Pinned two ways, because the failure modes differ: an exactly-three-identities assertion that NAMES an unintended principal (presence/absence checks pass happily while a fourth rides along, which is the reported defect's exact shape), and an end-to-end assertion through the real `WriteWithBackupAsync` path, because what was wrong was a CALL SITE passing the wrong argument and a test of the ACL shape alone cannot see that -- **The compression self-heal read TimescaleDB's never-ran sentinel as a run that started in year 1** ([#1781], closes [#1760]) - filed as a nightly test flake, and it was a real defect in the detector behind the compression self-heal (#1585/#1586). `timescaledb_information.job_stats.last_run_started_at` is **-infinity, not NULL**, for a job that has never run - measured, along with the fact that a freshly added policy has no `bgw_job_stat` row at all and it is `alter_job` that materialises the row carrying the sentinel. Npgsql maps `-infinity` to `DateTime.MinValue`, so the stuck-Running arm computed an elapsed of roughly 739,000 days, cleared every bound, and reported a perfectly healthy job as "stuck in the Running state for 388,000,000 minutes". **That was reachable because `job_status` and `last_run_started_at` come from independent sources in TimescaleDB's own view**: `job_status` is `CASE WHEN pg_stat_activity.state = 'active' THEN 'Running'` joined on `application_name`, while `last_run_started_at` is `bgw_job_stat.last_start`. A job's FIRST run therefore reads `Running` while its start time is still the sentinel, so the window is structural rather than hypothetical - and every job on a newly provisioned store is in exactly that state. The consequence in the field was the self-heal re-arming a job that was running fine; it is self-limiting, since the sentinel clears once the first run finishes, but a "compression job stuck" alert on a fresh or freshly upgraded store may have been false, and that implausible minute count is the signature. The detection query becomes a const (the `RearmJobSql` idiom already in that file) and `NULLIF`s the sentinel, so BOTH `-infinity` tests run in SQL rather than through Npgsql's infinity mapping - the discipline the code already claimed for `next_start`; the pure predicate rejects `DateTime.MinValue` as a second line of defence. **The test's settle-wait had been hardened once for this exact class and failed anyway, and the reason is worth recording**: it polled `next_start` only - one of the two arms the detector evaluates - and the value it polled was the one the test's own `alter_job` had written one statement earlier. Instrumented, it returned after **POLLS=1, DELAYS=0**: it never waited at all, so no timeout increase could ever have helped, and the alternative hypothesis (a slow runner letting the wait time out quietly) is refuted outright because that path asserts loudly with a different message than the one observed. The wait now polls `ReadStuckCompressionJobsAsync` itself, so there is ONE predicate rather than two copies that drift and settled-according-to-the-wait IS settled-according-to-the-assertion. The live leg also gained the assertion it always claimed to make: the detector is failure-isolated, so a broken query returns an EMPTY list and `DoesNotContain` passed just as happily against SQL that never compiled - it now runs the production const directly, where an error throws, and requires the job to be present. Verified by mutation against a live PostgreSQL 18 + TimescaleDB 2.28.1 store: removing the `NULLIF` turned the **live** test red, which is what proves the sentinel is really `-infinity` and the guard is not dead code -- **A chunk that was already eligible to compress could still sit uncompressed for most of a day** ([#1779], closes [#1778]) - two separate waits stack up before a closed chunk becomes columnar: the *delay* (how old it must be) and the *tick* (how long it then waits for anything to act on it). The tick was **twice daily**, and it was never a product decision - `add_compression_policy` computes `schedule_interval` when the argument is omitted, and on the shipped TimescaleDB 2.28.1 a 1-day chunk interval gets exactly `12:00:00` (measured across six chunk widths: 1h -> `00:30:00`, 6h -> `03:00:00`, 12h -> `06:00:00`, and 1d / 2d / 7d all -> `12:00:00`, so the rule is `min(chunk_interval / 2, 12 hours)` - half the chunk interval **capped** at 12 hours, not floored at it. The cap is what makes 12 hours the default on every store shape this product can produce: the 1-day chunks it creates today, and any 7-day-chunk hypertable an adopted store still carries from before the chunk interval was passed). The extension's own SQL explains why omitting it is not the same as choosing it: the function is declared *"not strict because we need to set different default values for schedule_interval"*. On a field instance a `query_stats` chunk written entirely before the payload dedup of [#1768] reached **81 GB** before its scheduled compression ever came around. Post-dedup chunks are far smaller so that severity was one-time, but the mechanism was on every store: **the newest closed chunk is always the least-compressed data on disk, and the tick is the width of that exposure.** Now passed explicitly as **1 hour** - the natural floor, since eligibility only changes once a day per chunk, and the same cadence the continuous-aggregate refresh policies already use, so the store keeps one background rhythm. No new configuration knob; nothing in the field asked to tune this, they asked for it not to be half a day. - - **Passing the parameter fixes fresh stores only, which is why the converge is the half that matters.** Measured: against a store that already has a policy with a different `schedule_interval`, `add_compression_policy(..., if_not_exists => true)` returns **-1** and skips with a NOTICE - it does not reconcile the parameter. Every store that ever ran an older build would therefore keep the twice-daily tick forever, including the one this was reported from. The service now retunes existing policies via `alter_job` on start, which also re-anchors `next_start` immediately (a job sitting at last-finish + 12h moved to last-finish + 1h), so a converged store honors the new cadence on the next tick rather than after one final half-day. Idempotent by construction - it selects only policies whose interval differs - and scoped to compression jobs alone, because retuning the retention jobs would disturb the armed/paused machinery [#1680] depends on. Failure-isolated per job: an `alter_job` that fails, typically a least-privilege store whose login does not own the job, leaves that one hypertable on its old cadence and converges the rest. - - **A coupled constant moved with it, and would have been a regression if it had not.** The stuck-compression bound is `max(2 x schedule_interval, floor)`, so while the interval was the 12-hour default the first term dominated at 24h and the 2-hour floor never bound anything; shortening the tick collapses that term to 2h. The same field box measured a `query_stats` compression still running at **1h33m** and characterized them as hours-long at ~16 MB/s, so an unraised floor would have had the [#1585] self-heal re-arming compressions that were doing their job. The floor moves to **6 hours**: clear of every legitimately long run observed in the field, still catching a genuinely hung run four times sooner than the accidental 24h did, and `next_start = -infinity` - the dominant failure mode - is caught immediately regardless. - - **Compression is now visible while it happens**, not only in hindsight by its effect on disk: the existing hourly compression-health check also reports a running compression with its elapsed time and the eligible-but-uncompressed backlog the tick exists to keep at zero, at a proportionate level (Information only when something is running or piling up; a store has ~40 policies). Durations come from `timescaledb_information.job_stats` rather than the per-execution history table, which only records successful runs when `timescaledb.enable_job_execution_logging` is ON and defaults OFF - verified live, where a completed run left it empty. **The eligibility delay is untouched**: this changes when eligible chunks get taken, never what is eligible. Proven against a live TimescaleDB 2.28.1 store: the 12-hour default is measured rather than assumed, `if_not_exists` is shown unable to fix it, a legacy 12-hour policy converges to 1 hour and stays converged, a firing takes the eligible chunk and leaves a too-young one alone, one chunk held under `ACCESS EXCLUSIVE` fails while the rest of the run compresses anyway (TimescaleDB reports *"Failed to convert '1' chunks to columnstore. Successfully converted '2' chunks"*), and one row-locked job fails to retune without stopping the others. The **deadlock correlation in [#1778] stays an open watch**: the same chunks and the same bytes are compressed either way, so total exposure is unchanged - what improves is that arrears drain in small hourly runs instead of one half-day batch, and that the policies decorrelate 12x faster in wall-clock terms. - -- **One stuck rollback copy froze the ageing-out of every other one** ([#1775], closes [#1770]) - reported from a production field instance carrying seven multi-GB store copies on a volume with roughly 175 GB free against a 286 GB data directory. The retained-copy sweep's failure handling sat OUTSIDE its loop, so a single copy that could not be measured or deleted - a file a not-yet-exited postmaster or an antivirus scan still holds, an ACL the service account lost - abandoned the sweep for every other copy too, and kept abandoning it for as long as the condition lasted. Their retention counters stopped advancing with it, so nothing aged out at all. Each copy now takes its own turn inside its own handler: a failure costs that one directory, names it in the log, and is retried next start. **The seven directories in the report were not ours** - `_rollback_manual_` appears nowhere in the product, and `DarlingStoreUpgrade.cs`, which owns the whole retain-two-starts contract, was created by [#1718]'s work days after they were made. They were made by hand, so they are now REPORTED rather than deleted: any sibling of the data directory holding a `PG_VERSION` file that this service did not create is named every start with its size and a running total, because a copy someone made by hand is a decision to reverse deliberately, not something a service should silently undo - but tens of gigabytes going unmentioned is how a volume reaches that state unnoticed. Identification is structural (`PG_VERSION`), never a name pattern, and the delete stays scoped to the names the upgrade itself produces, re-checked against the prefix because `Directory.GetDirectories` also matches Windows 8.3 short names. A half-built `-upgrade-NN` cluster left by an interrupted upgrade gets its own line rather than being blamed on someone else - it is one of ours, deleted only best-effort by the upgrade's failure path - and it is still not removed automatically, because the commit point is two directory moves and a process that died between them leaves the UPGRADED cluster under that name. -- **The restart delta seed read the whole store to find rows it was about to throw away** ([#1774], closes [#1772]) - reported from a production field instance, where the collector runner logged *"Failed to seed delta calculator from Postgres store... Exception while reading from stream"* on every start, including the one right after an upgrade. The four baseline seed queries carried no time bound on either half, so the inner `MAX(collection_time)` aggregated every chunk of the hypertable and the outer row-value probe scanned them all again - on a 276 GB store, on the default 30-second command timeout. The seed catches its own failures, so what the field actually lost was invisible: restart continuity, silently degraded to first-cycle-zero deltas on every service start. **The bound costs nothing that was being used.** All 36 delta call sites pass `maxGapSeconds: 300`, and the gap policy discards any baseline older than that and returns 0 - the same value an unseeded key returns - so every row found outside a five-minute window was work whose result was already being thrown away. The read is now bounded to a 15-minute window on **both** halves of each query (either one left open reads the whole table), with the cutoff carried as a reused `$1` that both Npgsql and DuckDB resolve to a single parameter, keeping the two apps' queries byte-identical. No timeout was raised. Lite carries the same bound. `memory_grant_stats` also gains `collection_time` in its SELECT list: it was the one family seeding a null timestamp, which disarms the gap policy for two monotonic counters where a stale baseline reads as a fabricated spike. Pinned by mutation on both sides, including against a live PostgreSQL 18.4 + TimescaleDB store, where dropping just the outer bound puts a two-day-old chunk back in the plan. -- **The service tried to open its own firewall on every start, could never succeed, and a fresh networked install got no rule at all** ([#1773], closes [#1771]) - a production field instance logged both halves of the firewall self-heal failing for BOTH endpoints on EVERY service start: `Remove-NetFirewallRule : Access is denied` and `New-NetFirewallRule : Access is denied`, for 5152 (MCP) and 5153 (web). It was harmless *there* only because the rules already existed from an earlier manual setup. The real defect is what happens without that accident: on a genuinely fresh networked install nothing ever creates the rule, so remote MCP and dashboard clients are firewalled off with only a warning in a log file to say why - and **networked is the normal deployment mode for this product, not the exception**. The cause is structural rather than a bug in the reconcile: the service runs as the `NT SERVICE\PerformanceMonitor Darling` virtual account, which cannot modify Windows Firewall **by design**, and granting it that privilege is the wrong fix - a monitoring service should not hold it. - - **Rule management moves to the elevated context that already existed.** A new `--configure-firewall` verb reconciles all three scoped rules - store, MCP, web - against `darling.json` in one pass, and `install-darling.ps1` runs it (before the first start, and again after `-Network`, since the wizard rewrites the exposure it just acted on); `uninstall-darling.ps1` removes them. The verb reads **only** `darling.json` - no store connection, no credentials - which is what lets it run at install time before the store has ever booted, unlike `--enable-mcp`/`--enable-web`, which write the control-plane store and need it initialized. It is idempotent, so an upgrade re-running it is a no-op. - - **The exposure decision is delegated, not re-derived, and that is the load-bearing choice.** The planner asks the same resolvers the running service fail-closes on (`ResolveNetworkExposure`, `ResolveMcpBind`, `ResolveWebBind`) rather than reading `listen` at face value. A config the service degrades to loopback - an unparseable `listen`, a missing or invalid `allowFrom`, a mismatched address family, a missing token, BYO mode - gets **no open port**, and the CIDR that reaches the rule is the parser's canonical output, never the operator's raw string (`IPNetwork.TryParse` *masks* host bits rather than rejecting them, so `192.168.1.77/24` opens `192.168.1.0/24`). Re-deriving it would have been worse than duplicated logic: the installer would open a port for an endpoint the service binds loopback-only, and then the service's own start-up check would flag, every start, a rule the installer had just deliberately created. - - **At runtime the write degrades to a read.** `Get-NetFirewallRule` needs no elevation - verified on Windows 11 26200 under a restricted token, where reads succeed and `New-NetFirewallRule` returns `PermissionDenied: Access is denied`, the exact field signature - so the service still tells the operator the truth without holding the privilege. It probes, then reports **once per state, not once per attempt** (the MCP and web hosts retry a failed start on a 30-second backoff, so a check wired where the reconcile sat would have traded access-denied spam for warning spam). A loopback-only install with no rule is the healthy default and logs **nothing at all**; an exposed endpoint whose rule is present logs one INFO; an exposed endpoint with no rule logs one WARN naming the exact `New-NetFirewallRule` to run, built by the same builder the elevated verb runs so the printed and applied commands cannot drift; a loopback-only endpoint with a rule still open logs one WARN naming the removal. An unreadable probe result is its own verdict rather than collapsing into "absent", which would manufacture a missing-rule warning out of a broken probe. The probe is also shaped so "absent" is an ANSWER and not a failure: an exact `-DisplayName` matching nothing is an `ObjectNotFound` **error**, and a suppressed error still exits 1, so the explicit `exit 0` is load-bearing - the same trap `BuildFirewallDisableCommand` already documents. - - **The bundled store had the identical bug on its own port** and is fixed the same way, though the issue named only 5152/5153. Two behaviors deliberately end: a service stop no longer removes the rule (it could never succeed, and the rule is install-managed now - it is scoped to a port nothing is listening on, and the next start reports it as stale), and `_runningNetworkMode`, which existed only to drive that removal, is gone. - - **The port is part of the rule NAME**, so changing a port does not update a rule - it makes a different one and strands the old as an inbound allow rule on a port nothing serves. Reconciling by exact name could never reach that, so the verb sweeps a per-surface `... (port *)` wildcard before ensuring the current rule. The wildcard is derived, never assembled from an operator string, and a name with no port suffix is returned UNCHANGED rather than gaining a `*` - a wildcard built from a malformed name could match, and therefore delete, rules this product does not own. Verified live: with a leftover `MCP (port 5152)` rule present and `mcp.port` moved to 5199, one run left exactly one rule, on 5199. The sweep runs as its OWN step rather than concatenated ahead of the open, because it ends in `exit 0` and would otherwise terminate the shell before the rule was created - a bug written and caught here, now pinned by its own call-shape test. - - **Verified live, not just green.** The verb was run against real configs on a real firewall: an exposed MCP config created exactly one rule (`TCP 5152, inbound, RemoteAddress 192.168.1.0/24`) and left the other two absent; a fail-closed config (`allowFrom: "not-a-cidr"`) opened nothing, said why, and removed the stale rule; a loopback config removed all three; the machine was left with zero Darling rules. The exact probe command was run under a **non-elevated** token against both a present and an absent rule, returning `1` and `0` with a clean exit each time. Every guard was then verified RED by mutation, including one that first came back GREEN and exposed a weak test: restoring the runtime `New-NetFirewallRule` write; planning exposure from `listen` alone instead of the resolver; dropping the probe's `exit 0`; treating an unparseable probe as "absent"; reporting on every tick; emitting an open command with no CIDR; renaming the shared rule prefix that uninstall sweeps by; deleting the installer's invocation; dropping the verb from the dispatch allow-list; and reverting the store to writing its own rule. The installer pin **survived** its first mutation - `Contains("--configure-firewall")` stayed green with the real call deleted, because the script also names the verb in its help text and its error message - so it now pins the invocation form and proves the wrapper is called somewhere other than its own definition, and goes red both when the call is replaced and when every call site is removed - -- **Anomaly baselines were being computed from a fraction of their intended history, silently** ([#1762]) - closing [#1757]. `PgBaselineProvider` asks for 30 days. `query_stats` keeps 4 under tiered retention, and the other eight baseline sources keep exactly 30, so on every tiered store the thresholds that decide what counts as an anomaly were built from far less data than designed with **no error raised**. The client-side timeouts that got this reported were the loud phase a store passes through on its way to that silence: once retention reaches a box the query stops timing out, because there is only 4 days left to scan. Severity was corrected during implementation - only `query_stats` sits behind the 4-day horizon (the TimescaleDB raw retention policy covers just `query_stats`, `procedure_stats` and `query_store_stats`, and only the first is a baseline source), so `query_duration` is a cliff at 4-vs-30 and the other ten families are a knife edge at 30-vs-30 - not broken today, but racing the purge at the oldest bucket and one config change away from the same degradation. **Nine continuous aggregates at the COLLECTION grain** now serve all eleven families: the hourly bucket is purely a partitioning and retention key and `collection_time` is a second GROUP BY column carrying the grain, so `AVG`, `STDDEV_SAMP` and the restart-exclusion `LAG` stay numerically identical instead of becoming a different statistic at a different scale. The existing hourly CAGG ladder **cannot** serve this for two independent reasons - wrong unit of observation, and it stores no sum-of-squares, so `AVG` reconstructs from sums and counts but standard deviation does not, and standard deviation is what sets the threshold. `cpu` and `file_io` therefore store sufficient statistics rather than an average: neither is one row per collection, because `CpuUtilizationCollector` issues `SELECT TOP (60)` over `RING_BUFFER_SCHEDULER_MONITOR` and appends every row under a single `collection_time`, so averaging first would silently redefine `sample_count` from samples to collections. The tier carries its own 35-day retention, deliberately longer than the 30-day window, and that relation is **pinned statically** because Storage cannot reference Analysis and it has no compile-time home - drop it below the window and the bug returns, silently. Two properties are required for the aggregates to actually hold history, both verified against TimescaleDB source rather than the docs: real-time aggregation is **opted into explicitly** (it has defaulted OFF since 2.13), and a coverage-gated backfill materializes whatever history the store already had. That backfill **verifies coverage after refreshing** instead of trusting that a successful `CALL` filled the aggregate, escalating to a forced refresh only on evidence: a refresh consumes invalidations, so a backfill cut short by a shutdown leaves a hole that a later plain refresh no-ops over while reporting success. That matters because the failure mode is **silence, not an error** - with real-time aggregation the watermark is a hard partition (`materialized WHERE time < watermark UNION ALL raw WHERE time >= watermark`), so an un-materialized region older than the watermark reads as absent rather than falling back to raw. No hand-rolled time slicing: manual `refresh_continuous_aggregate` already batches internally, each batch its own transaction, so partial progress survives an abort. The backfill is **launched, not awaited** - the composer tuning, delta re-seed and collection loop are all sequenced behind the TimescaleDB block, so awaiting it would take a restarted service dark for its whole duration, exactly when an operator is most likely to be restarting it - and a source-level test pins that, because adding an `await` there compiles, passes every functional test, and is invisible except on a large store. **No `CommandTimeout` bump anywhere**: the baseline queries keep the default, the fix is that they now read a tier sized for the question. Lite has no horizon bug (all nine of its baseline sources retain 30 days) so it keeps its raw-scan shape, but it gains the warning - lowering a source collector's retention below the baseline window would reintroduce the same silent degradation, and now says so once per metric. The mutation check is the test that earns its keep: point any family back at a raw `v_*` passthrough and the suite goes red, verified by actually reverting one. One more failure mode was caught by the live-PostgreSQL CI job, which is the only place it could show: without TimescaleDB the aggregates cannot exist, and the provider reads those relations **by name**, so a missing one throws, gets swallowed, and logs that same "Failed to compute baselines" line for **every** family - strictly worse than the bug being fixed. The nine relations are therefore also creatable as ordinary views, **derived from the aggregate's own select body** rather than written out again so the two supplies cannot drift, with `date_trunc('hour', ...)` standing in for the one TimescaleDB-only construct. The reverse transition is handled too - a store that later gains TimescaleDB has the stale fallback dropped first, guarded, because a continuous aggregate is itself a `relkind='v'` view and an unguarded `DROP VIEW` would destroy a materialized tier. -- **The Lite AG alert guard now captures the data service before checking it** ([#1756]) - carries the guard refinement from [#1754], which was closed as superseded and would have lost it; the improvement is that PR author's. `EvaluateAvailabilityGroupAlertsAsync` runs UNAWAITED off a UI-timer tick and awaits twice, so checking the nullable field and then dereferencing it leaves a gap where it can be reassigned - and an NRE on a fire-and-forget task is an unobserved exception nobody sees, so the symptom would have been AG alerts silently not evaluating on some launches with nothing in the log. Capturing into a local first also matches the idiom the same file already uses for the tray service. ([#1755] carries the remaining `xUnit2031` holdout separately, under its own author.) -- **A package with an OLDER PostgreSQL major no longer replaces a working newer runtime and takes the store down** ([#1752]) - closing [#1738] and [#1737] - `TryAdvanceRuntimeAsync` treated "the package's major DIFFERS from the extracted runtime's" as a reason to update, without checking which direction the difference went. A PostgreSQL 17 package landed beside an 18 store on DARLING01, the runtime was swapped, and the store was down about seven minutes until the previous runtime was restored by hand: no older postmaster can open a newer cluster. Nothing reached pg_upgrade, so no data was ever at risk - the store simply could not start. There is now a **direction check** against the data directory's own `PG_VERSION`, which is the authority on what the store needs AND is readable without executing anything - the reason the old checks all failed open is that they asked the binaries what they were, and on that host the binaries could not launch at all (`STATUS_DLL_NOT_FOUND`). It sits deliberately OUTSIDE the no-stamp branch, because the stamp records only that the zip CHANGED, never which way, so a stamped host receiving an older package would downgrade exactly the same. A refused package is left in place and its hash is deliberately NOT recorded, so the critical log repeats every start until the right zip is shipped rather than going quiet on the second one. **The second half of the same incident is also fixed**: the version check that should have caught it logged "data directory: 18, bundled runtime: unreadable - skipping the runtime version check. The store starts normally" one second before the bootstrap died. That degrade was backwards - the check could not run BECAUSE the binaries could not run, which is the strongest possible evidence they must not be used - so a KNOWN store major paired with an unidentifiable runtime now refuses to start, with a message naming the rescued runtime to restore instead of a raw Win32 status code. Also lands [#1737]'s deferred item: `RevertRuntime` now takes the data directory and the major it assumes, and refuses when `PG_VERSION` has already moved forward - redundant against today's two callers, and defence in depth against the third that forgets the flag, which is how #1738 happened by a different route. Every one of these is pinned, and the pins were mutation-checked: deleting the guard call, moving it inside the no-stamp branch, inverting either comparison, and deleting the revert guard each turn a test red - the direction comparison and the unidentifiable-runtime rule were extracted as PURE predicates precisely because inverting them in place left the entire suite green. -- **The Lite AG alert sweep could throw instead of quietly doing nothing before the store opened** ([#1753]) - `MainWindow.AlertEngine.cs` dereferenced `_dataService` unguarded, which the compiler had been reporting as `CS8602` since [#1726] landed the Lite AG alerts. The field is `LocalDataService?` and stays null until the local store opens, so a sweep that ran first would have thrown a `NullReferenceException` out of a background evaluation rather than passing with no alerts - the warning was pointing at an ordering hazard, not just a missing annotation. Guarded the way the rest of `MainWindow` guards the same field, deliberately not with `!`: there IS a correct behaviour before the store is open, and the null-forgiving operator would have asserted the opposite while silencing the compiler. Restores the zero-warning build - which turned out to need two fixes, not one: [#1750]'s new `AlertFiringLog.Fired` takes a non-nullable `shortMessage` while `AlertOutcome.ShortMessage` is optional (`string? = null`), so the engine's call site raised `CS8604` and a message-less alert would have logged a line ending in a dangling `": "` - the greppable shape breaking exactly when there is nothing to read. Fixed at the helper rather than the call site, since null is a normal input there and every other caller would otherwise have to paper over it; the line now simply stops after the server name. Twin tests in both suites cover null, whitespace and the muted combination. -- **Alert FIRINGS are logged now, in the shared engine and in Lite - not just resolutions** ([#1750]) - completes #1681. [#1685] fixed the Darling self-alert evaluator, but the issue's actual complaint - a log showing "... Recovered" with nothing before it - was still true for the NINE shared-engine families (CPU, blocking, deadlocks, poison waits, long-running queries, tempdb, volume free space, long-running jobs, failed jobs), whose resolution callback logs at Information while the firing logged nothing. Lite was worse: it logged NEITHER half anywhere, so an operator reading its log file after the fact found no alert history at all, because its resolution toast is transient and never reaches a log. `AlertEngine` now has a single `FireAsync` funnel and all nine families route through it - they previously each built an outcome and called the deliverer directly, which is precisely how they all ended up silent together - and it logs BEFORE delivering, because delivery does I/O and swallows its own failures, so logging afterwards would lose the record of exactly the alert someone later goes looking for. Lite's connection and Availability Group senders bypass both the engine and the self-alert evaluator, so they log at their own send sites, and Lite's resolution callback logs the cleared half for the first time. Darling's self-alert firing log is deliberately NOT duplicated: self-alerts do not pass through the engine, so there is no overlap to stack. The line shape is one definition shared by every path - both halves name the server and metric so one server's story greps together, and both spell out TRIGGERED / RESOLVED so they are distinguishable without knowing how each app maps severity to log level, which matters because the two halves are written at different levels by different code paths in two different apps. A muted alert still logs, flagged: muting suppresses the notification CHANNELS, not the operator's ability to find the event afterwards. Guarded by a source scan that fails if any direct deliverer call survives outside the funnel, since the way a tenth family skips the log is by copying an existing block. -- **The config backups were world-readable copies of the credentials they back up** ([#1749]) - [#1721] reported that the service's hardening of `darling.json` fails on every start; tracing the write paths found a second exposure with the same effect. The CLI's config verbs back the file up before rewriting it, and `File.Copy` does NOT carry the source DACL - the backup takes the DIRECTORY's inheritable ACEs instead, measured rather than assumed (a source protected with one ACE produced a backup unprotected with three). On the documented install location, a folder created directly under `C:\`, those include `BUILTIN\Users: Read`, so every `--rotate-token` or `--disable` dropped a full copy of every encrypted SQL password and access token that any local user could read - defeating the ACL that is the whole protection boundary for LocalMachine-scope DPAPI blobs. Backups are now hardened at the single chokepoint that creates them, and `install-darling.ps1` sweeps any already on disk. The installer also makes the service account the OWNER (ownership carries `WRITE_DAC`, so the service can re-assert its ACL even if a later edit drops the explicit `FullControl` grant; applied after the DACL and in its own `try`, so a privilege failure cannot take the DACL with it) and VERIFIES the result instead of assuming it - the original bug was a permissions call that silently did nothing for months. For hosts already in that state the service cannot self-heal, because re-ACL needs `WRITE_DAC` it does not have and taking ownership needs a privilege a virtual service account is not granted; the log now carries the three runnable `icacls` commands rather than describing the problem. -- **The daily-summary MCP live test no longer fails in the five minutes after UTC midnight** ([#1748]) - closes #1736. `HealthTools_ReadPlantedRows_AgainstDevPostgres` planted rows at `UtcNow - 5min` and then asked `get_daily_summary` for its implicit default of "today": between 00:00 and 00:05 UTC the rows carry yesterday's date while today has already rolled over, so the tool correctly returned its empty envelope and the test failed - turning any `darling-pg` run that landed in that window into a red required check that looked like a real regression. A test bug, not a product one, so the product default is deliberately NOT widened; the test simply stops assuming the planted date and "today" agree, and passes the date its own rows carry. **Verified by simulating the boundary rather than asserting it**: a new live test plants at an ABSOLUTE 23:57 UTC on a fixed past day, so the "row belongs to the previous day" condition holds on every run at every hour instead of only during the 0.3% of the day that straddles midnight - which is exactly why the original bug survived unobserved. It pins the mechanism too, asserting that the same rows queried without a date still return the empty envelope, so a regression that reintroduced an implicit-today call fails on every run. Detection power confirmed by pointing the query at the wrong day and watching it go red. -- **Darling: the TimescaleDB ensure-summary log stopped lying about which continuous aggregates exist** ([#1747]) - closes #1746, reported from a client-site upgrade. The line's COUNTS were computed (`{Ready}/{Total}`, so it correctly said 8) but the parenthetical text was hand-written and still read "3 hourly: query_stats, procedure_stats, query_store_stats; 3 daily: ..." from before #1664 added the db-grain pair. An operator mid-upgrade spent real time reconciling 8-against-6 before concluding it was not a bug - false operator-facing text costing more than the thing it described. The names are now DERIVED from the same array the counts come from, so the summary quotes its own source and cannot drift from it again; the fix is the shape, not a corrected number that would go stale on the next pair. Swept the sibling summary lines in `TimescaleSupport` for the same defect: the retention line and the hypertable/compression lines already interpolate computed counts and interval constants, and the `RetentionPolicyCount = 7` comment (three raw plus four hourly) is accurate, so this was the only instance. -- **The in-place store upgrade could not authenticate anywhere but a developer's box, and one of its new guard tests could pass with the bug present** ([#1739]) - the gated upgrade fixture went red on its FIRST CI execution, which is exactly what wiring it into nightly was for: `pg_upgrade` reported `password authentication failed for user "darling"` and the upgrade correctly aborted at the dry-run step, reverted, and left the store on 17 with its data directory untouched. The fail-safe worked; the upgrade did not. Cause was the credential HANDOFF, not the credential: it rode a hardened temporary `PGPASSFILE`, which has to get four separate things right on every host - the ACL (and `pg_upgrade` re-executes itself under a RESTRICTED token on Windows, so the reader is not quite the writer), the encoding, a path that restricted child can reach, and cleanup. It now rides `PGPASSWORD` in the child's environment instead, which is also the SAFER option rather than a trade: a process environment block is readable by the same user and by administrators, the identical audience that can already read the DPAPI credential file the password comes from, except nothing touches disk where a killed process could strand a cleartext temp file its `finally` never ran for. Separately, the cancellation-path guard test asserted textual ORDER rather than CONTAINMENT, so moving the revert OUT of its `if (!swapped)` block left it green while a post-swap shutdown would brick the store again - the same vacuous-pass hole that test was written to close, one level in. It now also asserts no closing brace falls between the guard and the revert, verified by applying that exact mutation and watching it go red. -- **The store-upgrade commit point is now pinned, and the degraded-upgrade alert stops asking for cleanup the product does itself** ([#1739]) - [#1718]'s review round hardened the failure paths; this pins them and corrects the one claim they got wrong. **The wiring had no test.** Mutation-checking the new pins found that deleting `swapped = true`, reordering `catch (Exception ex) when (swapped)` after the unfiltered catch so the filtered clause becomes unreachable, dropping the `!swapped` guard on the cancellation path, or deleting the `AssertUpgradePortsFree()` call ALL left the suite green - the logic tests pass because the logic is right, and the gated end-to-end test is a happy path that structurally cannot reach a failure branch. So the hardest-won guard of that round, the one that stops a post-swap failure putting PostgreSQL 17 binaries in front of an 18 data directory, could have regressed in silence. Closed with source-parsing pins in the `HostHeaderGuardTests` idiom this repo already uses for wiring invariants. Independent re-running of the mutations sharpened two things worth recording rather than smoothing over: reordering the catch clauses does not merely fail a test, it does not COMPILE (`CS0160`), so the compiler is the real guard there and the test comment claiming otherwise was corrected; and a FIFTH mutation was found that the pins missed - moving `RevertRuntimeForCancel` out of the `if (!swapped)` block, which compiles, is exactly the store-bricking bug, and passed green, because the pin asserted textual ORDER rather than CONTAINMENT. Closed with a containment assertion, and that mutation now turns the pin red. **And the alert was wrong.** Its degraded branch told the operator to delete the pre-upgrade data directory by hand "because it will not age out on its own" - but `RollbackRetentionStarts` is 2 and `SweepRetainedDataDirectories` reads a MISSING `.starts` marker as 1, so it writes the counter, keeps the copy, and deletes it on the following start. A failed marker write costs exactly one extra service start, which is the same reasoning that made wrapping that write safe in the first place. It now says so, with the real conditional: the copy only stays put while whatever blocked the write is still blocking it. Same defect class as the round that produced it - operator-facing text asserting something the code does not do - erring toward unnecessary work rather than false comfort. -- **Azure SQL Database: `database_size_stats` stopped touching `master`, and error 300 now explains itself** ([#1732]) - the remaining half of #1631, reported by TrudAX on a Dynamics 365 elastic-pool database reached through a DATABASE-level firewall rule. **`database_size_stats`** kept throwing `40615 Cannot open server ... not allowed to access the server` even after [#1634]/[#1642]. The query was never the problem - it was already database-scoped - but the collector declared `RunsPerDatabase` on Azure, and that made the host ENUMERATE databases first, which connects to `master`: the one database that login cannot open. The enumeration bought nothing there, because the connection already points at the database being monitored, the query reads only `sys.database_files` + `FILEPROPERTY(SpaceUsed)` (satisfied by the `public` role), and a contained Azure user cannot see sibling databases anyway - so the databases it went to `master` to discover were never readable. It now runs once on the existing connection and `master` is out of this collector's path entirely, rather than relying on #1634's fallback to recover from an error it never needed to provoke. Pinned so `sys.master_files`, `dm_os_volume_stats` and `dm_db_file_space_usage` cannot creep back in - the last of those looks like the natural Azure choice but carries the very trap described next. **`waiting_tasks`** answers the question TrudAX actually asked ("does it really require master DB access?"): **no, and no amount of `master` access would help.** Per MS Learn, `VIEW DATABASE STATE` covers `sys.dm_os_waiting_tasks` on every Azure SQL Database service objective EXCEPT Basic, S0, S1, and any database in an **elastic pool** - on those, only the server admin, the Entra admin, or a login in `##MS_ServerStateReader##` can read it, and `VIEW SERVER STATE` is not grantable at the server on that platform at all. So the collector is NOT gated off Azure (that would break it for the majority of Azure users, on S2+/vCore, for whom it works); instead the stored error now names the real cause and both remedies, appended to the raw SQL error so it stays searchable. Lite and Darling both, identical wording. -- **Lite: prove the Agent-status SQL, not just the decision it feeds** ([#1730]) - [#1725]'s tests all run against constructed `AgentStatusRow` values, which is the right shape for pinning what a row MEANS but leaves the query that produces those rows unproven. It is not trivial SQL: `ever_seen_running` is a window aggregate that has to see the whole retained partition while the surrounding query collapses to the newest row per server, and the two ways that goes wrong both return a plausible boolean and both pass every model-level test. If it collapsed to the newest row, a server that ran Agent yesterday and stopped today would read "never ran Agent" and a **genuine outage would render neutral** - the exact failure the fix exists to prevent, arrived at from the other direction. If the partition leaked, one real server's history would make every Agent-less container read as a server that runs Agent, putting the red "Stopped" straight back where [#1725] removed it. Six real-DuckDB round-trips now cover both, plus the newest-row-wins tiebreak, `collection_time` driving staleness, and the fresh case NOT being suppressed. Confirmed non-vacuous by mutation: dropping `PARTITION BY server_id` from the shipped query fails two of them. Also pins that the stale window stays longer than the collector's own cadence, read from `CollectorScheduleDefaults` so it follows a retune - a live trap rather than a hypothetical, since the shared `ServerHealthThresholds.StaleThreshold` is two minutes derived from the FASTEST collector's one-minute cadence, looks like the obvious thing to unify this with, and would render a perfectly healthy Agent as "unknown (stale)" most of the time because `agent_status` collects every five. Tests only; no production code, and the shipped query passes as written. -- **Credential-file ACL failures now name the owner, and the DPAPI credential files verify the result instead of assuming it** ([#1727]) - hardening #1721. When the service cannot re-ACL a file it protects, the error said "fix the file permissions by hand" and left the operator to work out which permissions, held by whom, and why. It now names the file's OWNER and the ordinary-user group that can read it, and states the part that actually resolves it: `SetAccessControl` needs WRITE_DAC, which comes with ownership or FullControl, so a service account holding only inherited Modify on a file owned by someone else can NEVER succeed - restarting will not clear it, and only granting FullControl or ownership will. That was the exact live condition on a field box, where the same error repeated every start for a day. The bigger half: the post-harden verification that `darling.json` already had - attempt, then CHECK whether ordinary users can still read the bytes, and log Critical if they can - is now applied to the machine-scoped DPAPI **credential** files too (the managed-Postgres owner/admin/viewer/MCP credentials and the least-privilege role credentials), which previously only logged the attempt. For LocalMachine DPAPI, read access to the file IS the secret, so "we tried to harden it" and "the secret is protected" are different claims and only one of them is worth logging. -- **Lite: the Job History Agent indicator stops calling absence of signal a problem** ([#1725]) - the display-tier companion to the Agent-alert capability gate, closing the same misleading-by-platform class on the surface a user actually looks at. The header rendered a red **"Agent: Stopped"** in exactly two situations where nothing is wrong. A server where SQL Agent has NEVER been observed running - a container built without it, Express, a Linux-minimal image - is not stopped; nothing stopped, the target simply does not run Agent, and Lite collects in-process so nothing here depends on it. That now reads a neutral **"not present"**. And a STALE snapshot is not evidence Agent is down: a server nobody has collected from in days was rendering its last-known state in red forever. That now reads **"unknown (stale)"**, using the same 30-minute refusal window the headless service's alert path already applies, pinned equal by test so the two surfaces cannot drift into disagreeing about whether a reading is old enough to judge. Red is now reserved for the one case that means something: a FRESH reading of a genuinely stopped Agent on a server that runs one. The fleet roll-up counts only the servers the question applies to, so "2/2 running" no longer becomes "2/5 running, 3 stopped" because three containers were counted as failures. The backing read gained `collection_time` and an ever-seen-running flag to make any of this decidable - it previously returned neither, so the header had no way to tell the three cases apart. Also corrects the collector-schedule description shown in Lite's own UI, which told users `agent_status` "drives the Job History tab header and the Agent Not Running alert"; Lite raises no such alert, and now the text says so. -- **Darling Web: a capped time-series panel kept the OLDEST slice of the window and said nothing** ([#1724]) - closes #991's sibling #1687, found live on DARLING01 while building the #1606 demo view. A grouped panel whose buckets x groups product exceeds the 10,000-row hard cap got `ORDER BY bucket LIMIT 10000`, which keeps the EARLIEST rows: a 24-hour wait_stats panel grouped by `wait_type` at minute grain rendered its first ~87 minutes as though that were the whole window, and because the truncated domain never crossed midnight the x-axis showed no dates, so nothing looked wrong. Same silent-truncation class as the retention-routing gaps (#1661/#1664/#1665) - the numbers are right and the WINDOW is wrong, which a monitoring product must say out loud. **Two halves, both needed.** The cap now applies `ORDER BY bucket DESC` inside a subquery and re-sorts the survivors ascending for the renderer, so a truncated chart shows the RECENT end - what a monitoring chart is for - instead of the start of the window. And when a time-series panel comes back sitting exactly on the cap it attaches a notice through the existing #1665 `notice` plumbing ("row cap reached - showing the most recent 10,000 buckets of the window, not the whole of it. Coarsen the time bucket or narrow the group-by..."), rendered by the `noticeStrip` already on every web surface and riding verbatim through MCP. Exactly-at-cap is the only signal available, because group cardinality is a property of the data rather than the spec - so a panel that legitimately produces exactly 10,000 rows is reported as capped, a false positive taken deliberately over the reverse. Retention and row-cap notices are JOINED rather than one winning, since the two truncate a panel from the same end and a long grouped panel on a retention-active store hits both. Ranked and Scalar panels are untouched: a Ranked LIMIT is the caller's own topN, so reaching it is the request being honored, not truncation. The issue's optional third item (a write-time hint) is deliberately not implemented - `MaxBuckets` (5,000) is already below the row cap, so buckets alone can never trip it and any pre-flight warning would have to guess at group cardinality it cannot know. -- **Darling: "Agent Not Running" no longer nags servers that never ran SQL Agent** ([#1719]) - a capability gate for the field-noise class Erik hit within minutes of pointing the fleet at real mixed targets. The alert fired the instant a stopped Agent was first seen and re-fired every cooldown forever after, so any target where Agent is off BY DESIGN - a container built without it, Express, a Linux-minimal image - produced an endless stream of identical Critical alerts telling the operator to start a service they deliberately declined. Observed on the AG fixture containers: byte-identical alerts every five minutes on both nodes from first contact. **The framing matters more than the fix.** This alert is a full-Dashboard-era holdover: the Dashboard collected THROUGH SQL Agent jobs, so Agent down meant collection down and firing on sight was correct. Lite and Darling collect in-process and have no Agent dependency at all, so all that survives is a customer-WORKLOAD signal - "the jobs you rely on stopped running" - which is meaningless on a server that runs no Agent jobs. So a stopped Agent now alerts only where Agent has been OBSERVED RUNNING at least once; everywhere else it is permanently silent, and it re-arms by itself the moment Agent genuinely runs. That is the same first-sighting-silent discipline the connection edge three methods away already applied, restored to the one condition that was missing it. The baseline is DERIVED from collected `agent_status` history rather than remembered in process, deliberately: an in-memory flag would be forgotten on restart, which is precisely when a genuinely stopped Agent would go quiet instead of alerting. Failed-job and long-running-job alerts are untouched - real jobs failing on a real Agent are still real. No new configuration: the gate is evidence, not a knob. Lite raises no Agent service-state alert at all, so there was nothing to gate there; three Lite comments and one collector description that claimed otherwise (the schedule UI told users agent_status "drives ... the Agent Not Running alert") are corrected to say the alert is Darling's. -- **Darling: retention policies were silently not being created AT ALL, on every store** ([#1711]) - closes #1705, and it is worse than the version-compat problem it was first reported as. `add_retention_policy` has **never** accepted a `scheduled` argument on any TimescaleDB 2.x - the parameter exists only on `add_job` / `alter_job` - so the statement introduced with the pause-then-arm work failed with `42883: function add_retention_policy(...) does not exist` on **every** store, fresh or upgraded, not just older ones. Because each policy is created inside a per-policy `try/catch` that downgrades a failure to a warning, this surfaced only as a line reading `TimescaleDB: 0/7 retention policies in place` buried among seven warnings, and every retention tier has been growing without bound since. Verified against a live 2.28.1: the accepted signature is `(regclass, "any", boolean, interval, timestamptz, text, interval)`, and the shipped statement is rejected there too. The fix drops the bogus argument and preserves the paused-at-creation guarantee a different way: creation and `alter_job(job_id, scheduled => false)` now run in **one transaction**, so the TimescaleDB job scheduler - a separate backend - cannot observe the `bgw_job` row until it already reads `scheduled = false`. That closes the same window without needing an argument the API does not have, and works identically on fresh and upgraded stores. `add_retention_policy` returns `-1` when `if_not_exists` matches an existing policy, so that case is skipped rather than fed to `alter_job`, which keeps a restart converging instead of re-pausing a policy the store already armed. The arming path is untouched. **The pin that let this ship is replaced**: the old test asserted the generated string *contained* `scheduled => false`, so it passed against SQL no TimescaleDB version can parse; there is now an inverted string pin plus a gated-live test that actually EXECUTES the policy creation and asserts all seven apply - confirmed to fail against the old statement and pass against the new one. Validated end-to-end on a real 2.28.1 hypertable holding 30-day-old rows under a 4-day policy: the rows survive creation, proving no immediate drop. -- **Darling: state the rule that decides how a new AG measure gets handled** ([#1710]) - the AG sync doc block ended up carrying two superficially similar measures with opposite prescriptions (gate drain-time, never use commit deltas) and no stated reason for the difference, which is the setup for someone applying the wrong precedent. The discriminator is now written down, and it is the failure DIRECTION rather than the column: a reading that fails in ONE direction can be gated, because the may-fire-never-clear rule suppresses exactly the direction that would wrongly clear an alarm; a reading that fails in BOTH directions under different conditions has to be abandoned, because gating it converts a silent failure into a noisy one rather than a correct one. Worth recording because every measured VALUE in this area moved at least once during the work - the documented lag behavior, the latch timing, the drain estimate, the lag baseline under load versus idle - while the direction-based rules never moved. Documentation only. - -- **Darling: corrected the AG doc guidance on commit-time deltas** ([#1708]) - [#1703] lumped commit-delta triggers in with drain-time triggers and told a future implementer that both "fail silent rather than loud". That is right for drain-time and wrong for commit deltas, in the direction that matters: it would send someone to guard the suspended case and then ship a trigger that pages about every idle database. `now - last_commit_time` fails BOTH ways - it stops growing on a suspended row (the silent half), and it grows without bound on a quiet but perfectly HEALTHY replica, because nothing has committed. Measured at 1757 seconds, and separately at around 7 minutes, on secondaries that were `SYNCHRONIZED`, not suspended, and reporting `secondary_lag_seconds = 0`. The loud half is the one more likely to actually ship, since it shows up immediately in testing against any database nobody is writing to. The guidance is therefore no longer "route it through the gate" but "do not derive lag from commit times at all" - `secondary_lag_seconds` is the lag measure, and the commit times are for showing an operator WHEN something last happened rather than for judging whether it is late. Documentation only; this evaluator has never read those columns. - -- **Darling: the AG lag alert now says what its number actually measures** ([#1703]) - follow-up to [#1700], which got the rule right but the mechanism wrong, and the mechanism is what an operator needs to tune a threshold. While data movement is suspended `secondary_lag_seconds` reports the **staleness of the last hardened log** (roughly `now - last_hardened_time`), which then grows at wall-clock rate; it is NOT time since suspension. That reconciles two measurements that looked contradictory: under write load the last hardening is near-now, so it starts around 0 and climbs, while on an idle group it starts at however long since the last write and can jump straight to a large number. Chasing that turned up something sharper, measured across a 60-second suspend on an idle group sampled every 15 seconds: lag read **0 at every single sample** while the replica was already `NOT SYNCHRONIZING` and its last hardened log aged from 262 to 322 seconds. It did not latch late - it never latched. So a suspended replica can report zero lag for an entire outage, which makes [#1700]'s "a sub-threshold reading on a suspended row is never *caught up*" load-bearing for a far more common case than the inverted-documentation one it was written for. The corollary is now stated plainly in the code: **the lag trigger alone cannot detect suspended data movement on a quiet group** - "AG Database Suspended" is the alert that owns that case, which is why the family has both. The alert detail text also now explains that the figure is staleness rather than volume of queued data, since on a quiet group a large value can simply mean nothing has been written recently, and the volume measure (`log_send_queue_size`) reports nothing at all while suspended. Documentation and alert text only; no logic change. The rule is also now stated as a property of suspended ROWS rather than of the two columns it started on: ag-collector-builder's measurement of the four `*_time` columns (#1702) found they all FREEZE at their last pre-suspension instant, so a cross-replica commit-time delta stops growing exactly when replication stops, and any drain-time measure (queue divided by rate, both frozen) holds a small, static, healthy-looking number - measured at a flat 0.0144 minutes across an entire suspension - when the true answer is “never”. Three different columns, three different mechanisms, all erring in the REASSURING direction, which is the half that fails silently. Anything added later that judges a suspended row has to route through the same may-fire-never-clear gate. -- **AG suspension semantics: one rule that holds whether or not the vendor docs are right** ([#1702]) - the AG collector's doc comments described the SUSPENDED state as a single quirk about `secondary_lag_seconds`. Measuring the rest of the surface on the live Docker AG fixture showed it is broader, and that the columns disagree with each other in OPPOSITE directions, which is the part that could bite. While a replica is suspended: `secondary_lag_seconds` ACCRUES (measured twice - 0/15/31/46/62s and 30/45/60/75s - though MS Learn claims it reads 0), `log_send_queue_size` goes NULL, `redo_queue_size` FREEZES at its last value, and all four `*_time` columns FREEZE at their last pre-suspension instant. So a cross-replica commit-time delta stops growing exactly when replication has stopped, understating the problem at its worst, while the lag column overstates it. **The sharpest edge is a measure this release shipped**: `est_redo_completion_time_min` is queue / rate with both frozen, so it holds a small, static, reassuring value (0.0144 min, flat across a 45s suspension) when the honest answer is "never, movement is stopped" - a suspended replica makes a drain panel look HEALTHIER than reality. All of it now reduces to one rule stated on the collector and mirrored on the compose measures: **a suspended row may RAISE an alarm but may never CLEAR one**, which is correct under both the documented and the measured behavior, so nothing downstream has to bet on which is true (WSFC and other builds remain untested). Also documented: `last_received_time` read NULL in every sample, healthy or suspended; and `last_commit_time` / `last_redone_time` sit still on an IDLE database, so `now - last_commit_time` is not a lag measure - on a quiet healthy replica it grows without bound. Doc-only; no behavior change. -- **Darling: a suspended secondary that is falling behind now raises the sync alert** ([#1700]) - the "AG Sync Fell Behind" lag trigger shipped in [#1692] made its seconds check ABSTAIN on a suspended row, written to MS Learn's statement that `secondary_lag_seconds` "shows as 0 if the data movement is suspended" - abstaining looked like the careful reading, since a zero would otherwise report the database that is furthest behind as caught up. **The documentation is wrong.** Measured against a live Availability Group (SQL Server 2022 16.0.4265.3, clusterless AG, write load, sampled across a `SUSPEND_FROM_USER` on the secondary), lag ACCRUES monotonically at wall-clock rate while suspended - 3993, 4005, 4017, 4029, 4041 across four 12-second intervals - and returns to 0 on resume. So the abstention was not caution, it was silencing the alert on suspended data movement: the single most common way a secondary falls behind, and the case an operator most needs paging for. A suspended secondary could drift arbitrarily far behind while only "AG Database Suspended" fired once, on the edge. Replaced with an asymmetry - **a suspended row may raise an alarm but may never clear one** - which is deliberately correct under BOTH behaviors rather than betting on the measurement: if lag accrues it crosses the threshold and fires, and if it ever did read 0 that zero is under the threshold and yields "not measurable" rather than "caught up", so it still cannot resolve a standing alert. The same rule now protects the redo-queue trigger, whose value FREEZES at its last reading while suspended (also measured): frozen and over the threshold is a real backlog worth firing on, frozen and under it is stale data that must not clear anything - previously a small frozen queue could resolve a live alert. Two more measured behaviors are documented rather than coded around: `log_send_queue_size` reads NULL while suspended instead of growing, so it is useless as a fell-behind signal (this evaluator never used it), and on RESUME the secondary has a genuine backlog to drain (388,620 KB after a 60-second suspend under load), so a single-sample redo threshold fires during legitimate catch-up - not wrong, since the data-loss window really is open until it drains, but it is why that trigger ships off. Evidence from the Docker AG fixture; the suspend/resume cycle was re-run independently before the shipped logic was changed. -- **Darling: the Availability Group store reads are now executed against a real Postgres in CI** ([#1697]) - the two AG reads added in [#1692] had no test that ran their SQL, and that was the one gap that mattered: the defect they were corrected for during review is invisible to a unit test. Both grains were briefly read as two statements over a SINGLE command to save a round trip, which PostgreSQL rejects - Npgsql only splits multi-statement text into a batch when it parses the SQL for NAMED placeholders, so with the positional (`$1`) parameters those reads use it sends one extended-protocol `Parse` and the server answers `cannot insert multiple commands into a prepared statement`. Every AG path is failure-isolated, so it would not have crashed anything; it would have logged one error per server per sweep with the whole alert family silently dead, which is the worst failure mode a monitoring product has - the thing that is broken is the thing that tells you something is broken. The new `DARLING_TEST_PG`-gated test seeds both collector tables and asserts what only real SQL can prove: that each query executes, that the newest-snapshot predicate excludes an older one, that a row with a NULL identity column is dropped rather than keyed under a placeholder, that NULL lag and suspend-reason columns round-trip, and that the path reaches the sweep entry point and fires exactly one alert off freshly seeded rows. - -- **Lite + Darling: recovery notices were being styled and counted as live alerts in Alert History and the Daily Summary** ([#1692]) - `AlertMetricClassifier.IsResolution` is the shared source of truth for "this row is good news, not something to act on", and it recognized only the `Cleared` / `Resolved` / `Restored` suffixes. Darling's self-alert recoveries have been emitting `Collection Resumed`, `Agent Restarted` and `Compression Job Recovered` - written by the very same recovery path as the recognized `Capture Restored` - and every one of them landed in BOTH apps' Alert History grids styled as an actionable alert, and was counted as one in the Daily Summary's per-day alert totals. This is the same drift #1225 fixed one layer up, recurring one layer down. Widened the suffix set to `Resumed` / `Restarted` / `Recovered` / `Reconnected` (no actionable metric name in either app contains those words, so nothing real turns green), and widened both hand-maintained SQL copies of that list - Darling's `DailySummarySql` and Lite's `LocalDataService.DailySummary` - which is where the miscount came from. -- **Darling: the MCP `get_alert_settings` tool under-reported the store by five columns** ([#1692]) - its SELECT stopped at 36 columns while `config_alert_settings` grew to 41, so an MCP client could not see the [#1674] connection opt-ins at all, and would not have seen the new Availability Group knobs either. Extended to the full 41. The parity test that pins the alert-settings column list against the upsert's placeholders, the bind order and the reader ordinals - the guard for the highest-risk defect class in that plumbing, since a mismatch only fails against a live Postgres - had drifted to 36 the same way; it now covers all 41 and drives its placeholder loop off the list length so the literal cannot drift again. -- **Darling test suite: a genuinely intermittent failure, roughly one full-suite run in six** ([#1692]) - `ViewerTimeHelper` keeps the timestamp display mode and the active server's UTC offset in process-wide statics, and three test classes have to mutate them to exercise the production code that reads them. They share a `viewer-time-statics` collection and each restores the previous value in a `finally`, which makes them safe against each other - but the collection had **no `CollectionDefinition`**, so it grouped its members without constraining them and xUnit ran it in parallel with every other collection. Any viewer test that renders a timestamp could observe a swapped display mode mid-assertion, and the victim differed run to run (`ViewerSystemEventsTests`, then `ViewerWave3DisplayTests`) - precisely the shape that reads as "flaky test, just re-run it". Added the definition with `DisableParallelization`, constraining the three mutators rather than chasing an open-ended set of readers; verified with 12 consecutive full-suite runs. -- **Lite + Darling: the query tools no longer present `max_dop` as current parallelism** ([#1682]) - `sys.dm_exec_query_stats`' DOP columns are lifetime-max values that persist for a cached plan's whole time in cache, so a plan compiled before MAXDOP was lowered keeps reporting the old higher value until it is evicted or recompiled. Field-confirmed on two servers running MAXDOP 1 where the tools reported `max_dop` up to 37 while the compiled plan showed `"dop": 0` and `"serial_reason": "MaxDOPSetToOne"` for those exact query hashes. Worse than a misleading column: `parallel_only` filters on that same stale value, so asking for parallel queries on a MAXDOP-1 server returned queries that cannot run in parallel at all. Both parameter descriptions now state plainly that this is a lifetime-max for the cached plan rather than current parallelism, and point at `analyze_query_plan` - which reads the actual plan - for confirmation. Byte-identical in both apps. This is labelling, not a data fix: deriving DOP from the stored plan XML at collection time is the authoritative version and is tracked separately, since it needs a collector and schema change. - -- **Darling: self-alert FIRINGS are logged, not just their recoveries** ([#1681]) - `RecordResolutionAsync` had always logged at Information while the firing path called the deliverer with no logger call at all, so the service log showed "Compression Job Recovered" with nothing before it. That reads as a spontaneous recovery from a condition that never happened, which is worse than logging neither half, and it applied to every self-alert - compression-job stuck, disk pressure, capture-down, agent-not-running, collection-health - not just the one that surfaced it. Firings now log at Warning (a firing self-alert means something is wrong with the monitor itself, and it must stand out from the Information-level recovery it will pair with), including muted ones, flagged: muting suppresses notification channels, not the operator's ability to find the event afterwards. Found on a field store where a compression job stuck 9 times in 6 days - always the two highest-write hypertables, never the other 35 - and the only record was `config.config_alert_log`, which you had to already suspect the problem to query. -- **Darling: the managed PostgreSQL now records a per-run audit trail for TimescaleDB background jobs** ([#1681]) - `timescaledb.enable_job_execution_logging` is off by default, which meant `timescaledb_information.job_errors` and `job_history` returned ZERO rows for every job on the field store, including jobs with dozens of confirmed successful runs. When a compression job hung there was no captured error, no worker PID, no start/finish record - nothing to diagnose from, which is why that hang still has no confirmed root cause. Now enabled in the conf append; the rows are written once per job run, not per row processed. -- **Darling: retention policies no longer destroy history on the deploy that creates them** ([#1680]) - TimescaleDB runs a newly-created retention policy's first check IMMEDIATELY at creation, not on its next scheduled interval. Confirmed on a field store: all six new retention jobs fired and completed within the same minute as service startup, before any external session could pause them per the runbook's pause-before-backfill ordering - permanently dropping ~2 days of query_stats/procedure_stats history and all of query_store_stats. There is no window to win that race, so being faster was never the fix. Policies are now created `scheduled => false` and armed as a separate step, and only when arming is provably non-destructive: the tier below must already reach at least as far back as the source does (raw covered by its hourly rollup, hourly by its daily). A store with nothing yet collected arms immediately, since there is no history to lose; a store whose rollups have not caught up is held paused with a warning naming the rollup to backfill, and arms itself on the first start afterwards - no manual step, and no operator racing a scheduler. The coverage probe fails CLOSED: if it cannot establish coverage for any reason, the policy stays paused rather than being armed on an assumption. - -- **Lite + Darling: a permission-denied collector is now visible without opening the Collection Health tab** ([#1591]) - collectors already classified permission failures as `PERMISSIONS` and the health grid already showed a `NO_PERMISSIONS` status, but both apps load that tab lazily, so the signal only existed if you already suspected the problem and went looking. That is why the same gap kept getting hand-patched one surface at a time after users reported it. The Collection Health tab header now carries a count ("Collection Health (3 no permission)"), refreshed on every pass in the slot each app already reserves for always-current chrome - Lite's alert badge, the viewer's freshness readout - so an empty tab caused by a missing grant is discoverable from anywhere. Counts DISTINCT collectors rather than log rows, so one collector denied every cycle for a week reads as "1" instead of a four-figure number nobody would trust, and filters to `PERMISSIONS` only so ordinary collector errors are never badged as a grant problem. Both apps, same wording, best-effort: a failed badge read leaves the plain header rather than disturbing the refresh. -- **Docs: which collectors run on which platform** ([#1591]) - the per-collector x per-platform matrix, derived from each collector's own `AppliesTo` declaration rather than hand-written, which is why it is five rows instead of thirty-six. Verified complete by arithmetic against `CollectorCatalog.All`. The part that answers the question people actually ask: a collector skipped for platform reasons shows NO runs in Collection Health, while one denied by permissions logs `PERMISSIONS` - identical from an empty grid, different problems, and only the second is a grant you can add. -- **Darling viewer: Server Inventory now explains missing hardware instead of showing zeros** ([#1591]) - a monitoring login without `VIEW SERVER STATE` (`VIEW DATABASE STATE` on Azure SQL DB) leaves the collector unable to read `sys.dm_os_sys_info`, so the stored hardware columns are NULL and the grid's coalesce rendered them as a literal `0` - indistinguishable from a real zero, and reading as "this server has no CPUs and no memory" rather than "we were not allowed to look". The Server Inventory grid gains the **Hardware Note** column Lite has had since [#1535], naming the missing grant. Lite sets that note by catching the `SqlException` from its own live DMV read; the viewer cannot, because it reads the collected store and the failure happened in the collector minutes earlier - so it derives the note from the NULL columns instead, which is only possible because [#1663] made them nullable (before that a missing grant lost the ENTIRE `server_properties` row, leaving nothing to annotate). Keyed on cpu_count AND physical_memory_mb both being absent, since they come from the same guarded read, so one odd NULL is never annotated as a permission failure. Closes the Lite/Darling parity half of the "surface WHY a tab or column is empty" work; the note derivation is a pure function pinned by tests, including one asserting the collector's hardware columns stay nullable so the note cannot silently become unreachable. - -- **Upgrade scripts (deprecated path): the 2.1.0-to-2.2.0 compression migrations are now genuinely idempotent** ([#1676]) - the three rename-swap migrations (`01_compress_query_stats`, `02_compress_query_store_data`, `03_compress_procedure_stats`) guarded themselves with `RETURN` in separate `GO`-terminated batches, but `RETURN` only exits the batch it is in - under sqlcmd (and the CLI Installer, which splits on GO and runs each batch as its own command) the migration body still ran after the guard printed its skip message. Re-running a script against an already-migrated table re-COMPRESSed the already-compressed LOBs (single `DECOMPRESS` reads garbage; the data is recoverable with a double `DECOMPRESS`), left a stray `*_old` table behind, and a second re-run then failed mid-swap and stranded the live rows in `*_new` - an empty live table. Fixed with the `SET NOEXEC ON` pattern: the guards flip NOEXEC instead of RETURNing, so every later batch parses but does not execute, and a final batch resets it (per-file connections in the Installer mean it cannot leak across scripts). Validated against a real SQL Server 2022 on a scratch store shaped both ways: the original script reproduces the reporter's corruption exactly; the fixed scripts skip byte-for-byte-inert on a migrated store (create_date unchanged, text intact, no stray tables, twice in a row) and still perform the full migration correctly on a genuine 2.1.0-shaped table. Reported with a superb Docker repro in [#1673]. Deprecated surface, bug-fix-only lane. -- **Full Dashboard (deprecated): `report.daily_summary_v2` no longer garbles `worst_query_hash`** ([#1667]) - the view boxed the raw `binary(8)` `query_hash` straight into its `sql_variant` pivot column, so anything that stringified that value reinterpreted the 8 bytes as UTF-16 and printed mojibake (`稟렿坧譈`) instead of a hash. It is the only non-scalar payload among the view's 25 pivoted metrics - every other row boxes a number or a string and round-trips cleanly. Now hex-encoded before boxing (`CONVERT(nvarchar(20), ..., 1)`), yielding `0x1F7A7FB76757088B`. Community fix by @argpna ([#1666]), verified live against SQL Server 2022. **Note for anyone querying this view directly:** the `sql_variant`'s base type for that one row changes from `binary(8)` to `nvarchar(20)`, so a caller doing `CONVERT(varbinary(8), metric_value)` needs updating - though in practice the old value was unusable, which is the bug. -- **Darling Web: composed custom-view panels no longer route onto rollups the store doesn't have** ([#1675]) - the composer's arm of the #1664 availability guard, and its pre-existing sibling: `ComposeSourceRouter` picked `query_stats_hourly` / `procedure_stats_hourly` / `query_store_stats_hourly` (or the `_daily` tiers) purely by window age and dimension coverage, never checking the rollup **exists in the store**. The continuous aggregates are runtime TimescaleDB setup, so on a bring-your-own **plain PostgreSQL** store any composed panel with a window past ~3 days compiled SQL against a missing relation and failed at run time with `42P01` - and a TimescaleDB store whose failure-isolated ensure sweep only built some aggregates routed onto the missing ones the same way. The #1664 probe now covers all three composer catalog tables' rollup pairs (per-table flags, since a partial build can lose one table's pair and not another's), the compile-and-run endpoint probes it lazily with the viewer's exact caching rule (permanent once fully built, 5-minute re-probe while partial, raw on a failed probe), and the route degrades through the SAME shared `RetentionTierRouter` ladder the built-in tabs use (daily missing -> hourly capped, hourly missing -> raw) so the two readers can never drift. On plain PG the raw fallback is complete, not degraded - nothing ever drops raw there - so it stays silent; on a **retention-active** store, a route that lands on a tier whose retention cannot reach the window's start (a forced raw fallback truncates at ~4 days, a daily-less hourly route at ~21) now returns a `notice` alongside the rows - "partial window, and says so," the same treatment the expensive-queries panel got in #1664 - rendered above the chart on every web surface and riding along verbatim through the MCP `run_custom_view_panel` tool. Pinned by router availability tests (per-table degrade), a compile-level raw-route test, notice-boundary tests, an 8-view probe/`Has()` drift guard, and a gated-live end-to-end that runs a 10-day window through the real runner against plain PostgreSQL - the exact 42P01 repro, now green. -- **Full Dashboard (deprecated): the Trace Flag Changes grid and the `get_trace_flag_changes` MCP tool throw `InvalidCastException` now that the view returns rows** ([#1668]) - `report.trace_flag_changes` projects `previous_status`/`new_status` as BIT, and the Dashboard reader called `SqlDataReader.GetString` on them, which performs no conversion. Before #1635 the view itself failed (Msg 245) before the reader ever ran; fixing the view exposed the cast. Fixed in the READER (GetBoolean + ON/OFF/empty formatting, matching the shared `ConfigChangeDiff.StatusText` wording) - the view's column shape is a public contract that external consumers (e.g. a Grafana companion) already query directly, so converting to text in SQL would have fixed the Dashboard by breaking them. Deprecated surface, bug-fix-only lane. -- **Darling: managed PostgreSQL's `pg.log` finally rotates** ([#1670]) - closes the deliberately-left gap 1 of #1652 (#1654 fixed gaps 2-3). The server log was a single pg_ctl `-l` file with no rotation of any kind, growing unbounded across restarts on the data volume. The v6 conf block hands steady-state logging to PostgreSQL's own logging collector as a SELF-CAPPING weekday ring (`postgresql-%a.log`, rotate 1d, truncate-on-rotation - seven files, one week, zero sweep code; size rotation deliberately 0 because size rolls append rather than truncate). `pg.log` keeps exactly its diagnostic job - pg_ctl chatter plus anything the server says BEFORE the collector starts, which is precisely the startup-failure window - and now gains a few lines per restart instead of the whole log; an oversized pre-rotation `pg.log` is rolled to `pg.log.old` once (two files, bounded forever). The failure-diagnostics tail follows the log wherever Postgres last wrote it (newest of pg.log and the ring). Existing clusters heal on their next service-owned start, same as the v2-v5 blocks, and since the append runs before pg_ctl start the restart-only `logging_collector` applies immediately. Live-validated by the gated bring-up E2E: a real server boots with the block and the ring file appears. -- **Darling viewer: built-in tabs no longer silently lose history past the raw retention horizon** ([#1661]) - the three-tier retention from [#1623] drops raw `query_stats` at 4 days and runs automatically on every service start, but the CAGG read-routing from [#1625]/[#1626]/[#1627] only ever covered the Custom Views composer. The viewer's built-in tabs read `v_query_stats`, a plain passthrough over the raw hypertable, so the Performance Calendar reported zero distinct queries for every day older than ~4 days and the FinOps panels aggregated a 4-day sample while the picker said 30. Nothing errored - the numbers were just quietly wrong, which for a cost view whose whole purpose is trend over time is the worst failure mode available. Each PR was correct in isolation: the write side and read side were designed as a pair and the read side was scoped to the surface being built at the time. **Never released** - caught in `dev` before 3.3.0. Fixed by giving the tier decision one definition (`RetentionTierRouter`, in Storage because the composer lives in the service and the tabs live in the viewer and neither can see the other; the composer now delegates to it and no longer keeps its own thresholds) and routing every reader that can be routed: the calendar's query count, the `get_daily_health` MCP reader, and three FinOps aggregates. The FinOps database-grain view needed I/O sums no rollup carried, so it gets a NEW `query_stats_db_hourly`/`_daily` pair rather than a widened existing one - TimescaleDB cannot ALTER columns into a continuous aggregate, and with retention live a drop-and-recreate would rebuild from 4 days of raw and permanently destroy the very history the tiers exist to preserve. Readers that project per-row `query_text`/`query_plan` cannot be routed at all (no rollup carries text), so the expensive-queries panel clamps to the text horizon and **says so** instead of presenting a few days as a month. Routing that also exposed a second copy of the entire daily-summary query in the MCP health reader, documented as the viewer's "verbatim" with nothing enforcing it - unified into one definition, since routing one and not the other would have had the calendar and the MCP tool disagree about the same day. The routing is gated on the rollups actually existing (a `to_regclass` probe, cached in the viewer): the continuous aggregates are runtime TimescaleDB setup, so a bring-your-own **plain PostgreSQL** store never has them - by-age routing alone threw `42P01` at the user there (caught by the gated-live CI, which runs plain PG). On plain PG the raw tables are never dropped either (retention policies need the extension), so routing everything to raw is complete, not degraded; a partially-built TimescaleDB store degrades one tier at a time (daily missing -> hourly, hourly missing -> raw), mirroring the composer's existing rule. - -- **Lite + Darling: a monitoring login without `VIEW SERVER STATE` no longer loses its entire `server_properties` row** ([#1591]) - `ServerPropertiesCollector` read hardware facts straight out of `sys.dm_os_sys_info` in the FROM clause of its main SELECT, alongside a dozen permission-free `SERVERPROPERTY` scalars. That DMV requires VIEW SERVER STATE (VIEW DATABASE STATE on Azure SQL DB), so a login lacking it failed the whole statement and collected **nothing** - no edition, no version, no patch level, no service objective - when only the six hardware columns were actually gated. Azure SQL DB users hit this hardest, since the grant is database-scoped there and monitoring logins routinely lack it on `master`. The DMV read now happens up front into variables inside TRY/CATCH - the exact pattern the supplemental health query ten lines below already used - and the projection reads no table at all, so a permission failure costs only `cpu_count`, `hyperthread_ratio`, `physical_memory_mb`, `socket_count`, `cores_per_socket` and `sqlserver_start_time`, which come back NULL. The first three were `NOT NULL` in DuckDB and are now nullable (Lite schema **v48** migrates existing databases with `ALTER COLUMN ... DROP NOT NULL`; Postgres was already nullable), and the collector's `Row` types them `int?`/`int?`/`long?` so "unknown" is representable rather than a fabricated zero. Downstream is unaffected: the FinOps and MCP readers consume the stored row and already null-guarded these columns. Pinned by three guards - the projection must reference no table, the hardware columns must bind to the TRY/CATCH variables while identity columns stay direct `SERVERPROPERTY` reads, and the `Row` fields must stay nullable - plus an explicit, documented exception in the schema-equivalence proof so the deliberate NOT NULL drop is reviewable instead of silently rewriting the frozen golden snapshot. - -- **Lite + Darling: the `get_active_queries` MCP tool no longer tells the model its data comes from sp_WhoIsActive** ([#1650]) - both apps shipped the tool description "Gets active query snapshots captured by sp_WhoIsActive", and Lite's MCP instructions table said the same. Neither app has ever called sp_WhoIsActive: that is the deprecated Full Dashboard's collection path (`install/11_collect_query_snapshots.sql`), while Lite and Darling share `QuerySnapshotsCollector`, which queries `sys.dm_exec_requests` (plus the session-space, task-space and transaction DMVs) with its own T-SQL. The wording was inherited from Dashboard and never corrected when the shared collector replaced it. This is worse than an ordinary stale comment because MCP tool descriptions are fed to the model as ground truth - an agent reading it would reason about column provenance, data availability and permission requirements as though sp_WhoIsActive produced the rows, when the real source has a different column set and different permission needs. Corrected in both apps (byte-identical, as the tool descriptions are meant to be) plus Lite's instructions table. The one remaining sp_WhoIsActive mention in Lite is accurate and left alone - it documents which columns the *Dashboard* collects that way, as the origin of the column set Lite later adopted. -- **Darling Web: a LAN-exposed dashboard no longer lets any local process in without a credential** ([#1649]) - the browser auth gate passed every loopback request tokenless, on a rationale written into the code: "the web surface is read-only." That was true when it was written and stopped being true at Custom Views v2, which added view create/update/delete plus `/api/compose/run`. So on a host with `web.network` enabled, anything that could open a socket to `127.0.0.1` - a scheduled task, SSRF'd code in another service, a second user's session, a sandboxed process - could read the entire monitoring store and create, modify, or delete custom views without presenting anything. The MCP host had already decided this exact question the other way (`NO loopback exemption - in exposed mode even a local client must present the token; that IS the loopback guard against SSRF/sandboxed sockets`), so the two surfaces on the same box disagreed. The web host now mirrors MCP: loopback is exempt from the **CIDR test only** - 127.0.0.1 is not inside a LAN CIDR and testing it there would 403 the operator's own browser - and still needs a session cookie or `?token=`, taking the identical token-to-cookie exchange a LAN client takes. **Loopback-only dashboards are completely unaffected**: the auth middleware is registered inside `if (networkMode)`, so with network mode off there is no gate to pass and browsing stays frictionless; the cost of the change is one login on a host you deliberately exposed, then a 12-hour cookie. The route matrix pins the change directly (same loopback inputs, `ShowLogin` instead of `Allow`, plus the cookie and token paths and the IPv4-mapped `::ffff:127.0.0.1` form so the unwrap can never drift), and a null/unverifiable remote still fails closed to `Forbid`. Also corrected the three comments that had gone stale and let the drift happen unnoticed - they still described the surface as read-only and the loopback arm as "tokenless". -- **Darling + Lite: security hardening - firewall command injection via `allowFrom`, an unprotected `darling.json`, and MCP hosts with no DNS-rebinding guard** ([#1646], [#1647], [#1648]) - four findings from the 2026-07 maintenance security review, all in the same blast radius: what a local user, or a browser running on the monitoring host, can reach. **(1) Command injection in `--enable-web` / `--enable-mcp` ([#1646]).** `ReconcileEndpointFirewallAsync` read `web.network.allowFrom` (or the MCP equivalent) out of darling.json and interpolated it UNQUOTED into the PowerShell `-Command` string that creates the scoped firewall rule, gated by nothing but a blank check. It was the one `BuildFirewallEnableCommand` caller that never parsed the value first - every other call site passes a canonicalized `IPNetwork.ToString()`, and the `--configure-network` wizard validates before writing, but the toggle verbs load the config with `DarlingConfig.Load`, which deserializes and never calls `Validate()`, so nothing upstream caught it either. An `allowFrom` of `10.0.0.0/8; ` therefore executed that command - directly, when the verb runs elevated (the documented way to run it), or via the operator, when it is not, because the non-elevated path PRINTS the fully-injected command and tells them to paste it into an elevated PowerShell. Fixed in two layers, because either alone leaves a sharp edge: the verb now parses `allowFrom` as a CIDR and passes the PARSER'S canonical output (so nothing unvalidated is carried through even on the valid path), refusing outright on a parse failure - no firewall change, and deliberately no printed command, since a handoff executes the injection just as surely; and `BuildFirewallEnableCommand`/`BuildFirewallDisableCommand` now single-quote and escape every value they interpolate (PowerShell escapes `'` by doubling), so the builders are safe regardless of what any present or future caller hands them. Real rule names contain no quote, so the emitted commands are byte-for-byte unchanged. **(2) `darling.json` was never ACL-hardened ([#1647]).** It holds every monitored server's `encryptedPassword`, the MCP bearer token, the web dashboard access token, and in BYO mode the store connection string - all DPAPI **LocalMachine** scope with an entropy constant published in this open-source repo, which means anything that can READ the file can unprotect the lot. That is by design; `DarlingFileSecurity` says so itself ("the file ACL is therefore the real access boundary"). The config just never got that ACL: every harden call site targeted the credential files under the data directory, while the config sat beside the binary, and the README's recommended install extracts the zip to `C:\PerformanceMonitorDarling` - a folder created directly under `C:\` inherits `BUILTIN\Users: Read & Execute` from the root DACL. Any local unprivileged user could read it, decrypt every SQL Server password, and lift the tokens that unlock the MCP write surface and the web dashboard - and, where the install directory was user-writable, plant the payload for (1). The service now hardens the resolved config path at startup with the same posture the admin/viewer credentials get (`allowInteractiveRead: true` - the Viewer and the CLI verbs run as the interactive operator and must still read it), `install-darling.ps1` applies the identical ACL right after service creation (which is when the `NT SERVICE` virtual account first has a SID to grant), and a file that is STILL readable by Users/Authenticated Users/Everyone after the attempt logs Critical rather than crashing a monitoring service over a permissions problem. The check keys on `ReadData` specifically, not the composite `Read` mask, so a metadata-only ACE does not cry wolf. **(3) Neither MCP host checked the `Host` header ([#1648]).** The web dashboard got this guard in [#1576], with the reasoning written into the code: a loopback surface is tokenless, so a browser ON the host that loads attacker content can be DNS-rebound to `127.0.0.1` and read and write the whole surface same-origin. That applies identically to the MCP host at :5152 in its default loopback configuration, which installs NO auth middleware at all - and that surface is no longer read-only, having gained custom-view CRUD, `add_servers`/`remove_server`, and alert-config writes. The `application/json` content type does not save it: under a rebind the browser treats the request as same-origin, so no CORS preflight applies, and `ModelContextProtocol.AspNetCore` does not add the check itself (per the MCP spec it is the application's job). The decision moves into a shared `HostHeaderGuard` in `PerformanceMonitor.Common` - the library both apps already reference - and is installed as the FIRST middleware, in BOTH bind modes, on Darling's MCP host and on Lite's, mirroring the web host. Lite is loopback-only, single-user, and holds no service-account privilege, so its exposure is informational; it is fixed in the same change anyway, because a guard that exists in one app's copy and not the other's is the drift this codebase keeps paying for. The web host's behavior is byte-for-byte unchanged (its forwarder and existing tests are untouched, which is the proof). **(4) Lite: an unescaped export path ([#1648]).** `DataImportService.FlushOldDatabase` built `COPY (...) TO '{path}'` by interpolating the import folder raw while every sibling COPY routes the same value through `DuckDbInitializer.EscapeSqlPath`. An apostrophe is legal in a Windows path (`C:\Users\O'Brien\...`), so it closed the SQL literal, the COPY failed to parse, and the per-table catch swallowed it as a warning - the import reported success having silently flushed nothing. Tested throughout, and the tests pin the things that actually regress: a non-CIDR `allowFrom` can never reach a firewall command (proven as a property of the canonical output, not by enumerating payloads); the builders keep any hostile value inside one balanced single-quoted literal; the Host guard's full accept/reject matrix in both modes; that the guard middleware is really installed FIRST and ahead of the network-only gates in all three hosts, parsed from the shipped sources, because #1648 was a WIRING omission that a pure-function test would have passed on the vulnerable build; the ACL check against real Windows DACLs; and the parquet export end-to-end under a directory whose name contains a quote. -- **Lite + Darling: four things that grew forever - two tables and two log directories** ([#1651], [#1652]) - found by the retention/archival audit in the 2026-07 maintenance pass, which confirmed everything else in both apps' retention is correctly wired. These were the exceptions, and each one had grown unbounded since the day it shipped. **Lite's `dismissed_archive_alerts`** - the sidecar that remembers you dismissed an alert that has since been archived to parquet - was INSERT-only with no purge path anywhere in the repo, and it is deliberately in `ArchiveService.PreservedConfigTables`, so it is RESTORED after the 512 MB emergency reset that incidentally bounds everything else. It was the one table designed to survive every reset with nothing to ever shrink it. It now purges on the same daily tick as the analysis-findings cleanup, at a 180-day horizon keyed on `alert_time`. The horizon is deliberately generous rather than tight: suppressing an archived alert is the row's entire job, so purging one while its parquet is still readable would make an alert you already dismissed reappear - and parquet retention is 3 months measured on the FILE's date prefix, which is the archive date and therefore always NEWER than the rows inside it. It stays preserved through the reset; it just finally has an age limit. **Darling's `config.config_command`** - the imperative command queue the viewer, MCP tools and CLI enqueue into - had a service-side purge that the viewer's own code already ASSUMED existed ("the service-side purge is the backstop", `ViewerDataService.RunTestConnectAsync`) but which nothing had ever implemented. The only DELETE was viewer-side, per-command, wired into exactly four self-cleaning flows, and best-effort with the exception swallowed; every other command type - pause/resume, `snapshot_now`, `analyze_now`, `purge_now`, the enable/firewall verbs, collector toggles, anything issued from MCP or the CLI - left a terminal row and its `result_json` behind permanently, as did those four whenever the delete failed or the viewer died mid-poll. It now purges alongside the `collection_log` (60d) and `config_alert_log` (90d) sweeps at a 30-day horizon, using the same compressed-chunk-safe time-sliced DELETE. It is keyed on `created_at` (NOT NULL, so no row can slip the horizon by never being stamped) and filtered to terminal rows only, in the DELETE **and** in both `min()` subqueries - a slice anchored on an ineligible row would delete nothing and the drain loop's "an empty slice means we are done" termination would stop the purge early with older rows still there. A `pending` or `in_progress` row is never purged however old: deleting a live command strands whoever is polling it. The statement is schema-qualified deliberately - unlike `collection_log` and `config_alert_log` (created bare, so they live in `collect`), this table really is in `config`, and a bare name would have resolved to a nonexistent `collect.config_command` and failed every night into a warning nobody reads. **Lite's application log** rotated daily but never pruned - "daily rotation" only ever meant a new file per day - so an install kept every `lite_yyyyMMdd.log` it had ever written. This was a parity gap inside Lite itself: both sibling loggers already swept at 7 days, and this one now sweeps the same way, on the same window, scoped to its own file name so it cannot touch theirs. **Darling's service log** had the sweep and the 14-day window already, but called it only from the logger provider's constructor - so a service left up for months, which is the entire point of a service, swept exactly once at startup and then never again while writing a file a day forever. The sweep now also rides the worker's daily maintenance tick beside the retention purge. Not addressed here and tracked separately: `pg.log` still has no rotation, because fixing it moves where Postgres writes its log and both `ReadServerLogTail` and the startup failure-diagnostics path depend on the current location - it needs its own change and a live bring-up test, not a drive-by. -- **Lite: the Excluded Databases picker throws a raw firewall error on an Azure database-level-firewall server** ([#1642]) - the picker lists databases by connecting to `master` and reading `sys.databases` with no fallback, so the exact setup #1631/#1634 fixed for collection - an Azure server blocked at the server-level firewall but reachable through a database-level rule (error 40615) - still hit a bare "Cannot open server ... is not allowed to access the server" error box here. It now catches the fallback-eligible error via the shared `SqlErrorClassification.ShouldFallBackToSingleDatabase` (the same classifier the collectors use) and falls back to the connected database with an explanatory status note instead of failing outright. Lite-only: the Darling viewer reads its database inventory from the Postgres store and never opens master. -- **Darling: a bring-your-own-Postgres `viewer` seat gets `permission denied` on the Manage Servers list and the Settings notification prefill** ([#1639]) - the fail-closed viewer column ACL (#1262) is authored ONCE in C# (`DarlingManagedRoles.ViewerRestrictedConfigTables`) and applied two ways: managed mode GENERATES the `REVOKE`-then-`GRANT SELECT (columns)` carve from that list on every service start, so it is correct by construction, while BYO mode has no service to generate anything - the operator runs `Darling/tools/provision-roles.sql` by hand, and that file's three `GRANT SELECT (...)` lists are a HAND COPY of the same columns. Nothing pinned the two together, and they drifted across three releases: `config_notification.generic_body_template` / `generic_proxy` arrived with the generic webhook channel ([#1506]) and `config_monitored_servers.alert_delivery_mode_override` was added to the C# list by [#1551] - where the gated-live ratchet caught the MANAGED half of exactly this bug and the BYO hand mirror was the half that got missed - and in neither case did the .sql follow. Because the carve is fail-CLOSED by design - `viewer`'s table-wide SELECT is revoked and only the enumerated columns re-granted - an unlisted column is INVISIBLE, so on a BYO store a read-only seat took a bare SQLSTATE `42501` on the two projections written specifically FOR it: `MonitoredServersSelectSql` (the Manage Servers list, which also drives the sidebar reconcile) and `NotificationSelectNoSecretSql` (the Settings prefill, where one such throw blanks every later section). Quiet in exactly the wrong direction - the columns exist, the script runs clean, `psql` as the owner sees everything, and only a read from the least-privilege role ever notices. Managed deployments were never affected. The three lists now match the C# source of truth exactly (nothing else was added - this closes existing drift, it does not widen the grant), and the real defect - a hand mirror with no guard - is closed by a new UNGATED `ProvisionRolesAclDriftTests`: it parses the SHIPPED .sql (copied beside the test binary, so there is no second copy to go stale and no repo path to hardcode), extracts every `GRANT SELECT (columns) ON schema.table TO role`, and asserts set-equality against the C# list in BOTH directions, plus the paired `REVOKE` and its ordering (without the REVOKE first the carve is a no-op and the secrets stay readable), that no secret column appears in any grant, and that the script still provisions admin+viewer only - never a half-provisioned `mcp` role, which BYO deliberately does not create. The guard is itself guarded: a meta-test mutates the real file three ways (drop a column, add one, delete a whole carve) and asserts the same comparison reports each, so a parser that silently matched nothing can never pass as "no drift". Verified by hand as well - removing either restored column reintroduces a failing test. -- **Lite + Darling: Linux hosts on SQL Server 2025 CU1+ get real other-process CPU again** ([#1630]) - the [#1048] guard NULLed `other_process_cpu_utilization` on EVERY Linux host, because the SCHEDULER_MONITOR ring buffer always reported `SystemIdle = 0` on Linux. SQL Server 2025 CU1 fixes the ring-buffer metrics (KB5078298), so the blanket guard was discarding a derivable value there forever. The guard now keys on the actual condition (`@is_linux = 1 AND system_idle = 0`) instead of the platform, so older Linux builds keep their honest NULL and 2025 CU1+ derives the real figure - version-proof by construction. Community fix by @argpna ([#1629]); applied to the shared collector (Lite + Darling) and the deprecated Dashboard's collection proc + NOC health query. -- **Lite + Darling: default trace collection works on SQL Server on Linux** ([#1633]) - the rollover-file path normalization assumed the trace file always carries a `_N` suffix (true on Windows); the initial file on Linux is plain `log.trc`, so the strip re-appended the extension onto the full path (`log.trc.trc`), `sys.fn_trace_gettable` raised Msg 19049 on the nonexistent file, and NO default-trace events were ever collected on Linux. The strip now falls back to the base path when there is no suffix to remove. Community fix by @argpna ([#1632]), live-verified against a 2025 Linux container; shared collector + the deprecated Dashboard's proc. -- **Full Dashboard (deprecated): the `report.trace_flag_changes` view no longer fails with Msg 245 on a trace-flag toggle** ([#1637]) - the view compared the bit-typed `previous_status`/`status` columns against `N'OFF'`/`N'ON'` literals; bit outranks nvarchar in type precedence, so SQL Server converted the literal to bit and the view failed the moment a toggle row flowed through it. It now compares `0`/`1`. Community fix by @argpna ([#1635]). -- **Default trace: the rollover strip no longer mangles paths whose DIRECTORY contains an underscore** ([#1636]) - follow-up to [#1633]: the strip keyed on the LAST underscore anywhere in the path, so a relocated trace in an underscore-named directory with an unsuffixed file (`/var/opt/my_sql/log/log.trc`) was cut at the directory underscore (`/var/opt/my.trc` - Msg 19049 again). An underscore now only counts as a rollover suffix when it sits after the last path separator (both separator families, since targets are Windows and Linux); truth-tabled live against SQL 2022 across nine path shapes. -- **Lite + Darling: Azure SQL DB collection works again when the client is firewalled out at the logical server but allowed at the DATABASE level** ([#1634]) - reported against 3.2.0 by a user monitoring an Elastic Pool database ([#1631]): every database-scoped collector failed with `40615 - Cannot open server ... Client with IP address ... is not allowed to access the server`, and the long-query XE session could not enumerate databases, even though the database itself was fully reachable from SSMS. **This is a regression of the [#857] single-database fallback, introduced by [#1506].** That fix correctly stopped reading 40615 ("client IP not allowed at the logical server") as a statement about a login's RIGHTS to read `master` - it is a reachability error, and misreading it had permanently wedged a different user whose public IP rotated daily. But it also dropped 40615 from the single-database FALLBACK path, on the stated premise that "falling back to a user database is futile because the same rule blocks that connection too." That premise is wrong: Azure SQL Database evaluates **database-level** IP firewall rules BEFORE server-level ones, and a client whose IP matches a database-level rule (`sp_set_database_firewall_rule`) is granted a connection to that database with **no** server-level rule permitting it - while `master`, where server-level rules live, still requires one. "Blocked at the server, allowed at the database" is therefore a real, supported configuration that Microsoft explicitly recommends ("use database-level IP firewall rules whenever possible"), and it is precisely the [#857] case. The two questions - *"does this login lack rights to master?"* and *"should database-scoped collection degrade to single-database?"* - are now separated in the shared `SqlErrorClassification`: the rights list is unchanged (40615 still correctly excluded, so every [#1506] invariant holds), and a new fallback list is that set plus 40615, which is what both SKUs' Azure database-enumeration catch sites actually ask. Acting on it is safe because [#1506]'s other fix survives untouched: the resulting verdict is a revocable throttle, not a latch - it expires after 15 minutes, is discarded when a server returns from an outage, and is skipped entirely when there is no target database to fall back to. A client genuinely firewalled out with no database-level rule therefore just fails its fallback attempt with the same 40615, logs an ordinary collector error, and recovers on the next successful connect. Pinned from both suites, independently: 40615 is still not a rights denial, AND it does trigger the fallback; the rights set remains a strict subset of the fallback set; and the disjointness invariant is now asserted against the broader fallback set (40613 stays transient-and-retryable, never a reason to degrade). Lite + Darling. -- **Darling: collectors no longer fail with `22021: invalid byte sequence for encoding "UTF8": 0x00`** ([#1614]) - SQL Server NVARCHAR allows embedded NUL characters and query text from `sys.dm_exec_sql_text` sometimes carries them, but Postgres `text` columns reject the byte, so one NUL-laden cached query failed the entire `query_stats` COPY batch every cycle (`DBCC FREEPROCCACHE` never helped because the app re-caches the same query). The Postgres row writer now strips NULs from every collected string - query text, plan XML, deadlock and blocked-process XML included - at the single COPY choke point. -- **Darling Web: a custom view no longer resets the picked time range / server / filters every 60 seconds** ([#1619]) - the background refresh that keeps the dashboard live rebuilt each view from its declared default, snapping any range/server/filter change back within a minute. A per-view scope memory now survives the re-render, so a picked scope sticks until you change it or hard-reload. -- **Darling: the Custom Views composer and analyze_*_plan no longer time out on large stores** ([#1620]) - the columns the composer filters/groups by (`procedure_stats.object_name`, `query_stats.query_hash`, `query_store_stats.query_hash`) and the single-row `analyze_*_plan` lookups (`sql_handle` / `query_id`+`plan_id`) had no supporting index, so on a large store they fell to a Seq Scan and hit the 15 s statement_timeout. The service now applies (idempotently, at startup, NOT a versioned migration) six indexes - three COVERING (the aggregate columns the composer SUM/AVGs are `INCLUDE`d, for an Index Only Scan) plus three lookup indexes - and a per-table `autovacuum_vacuum_insert_scale_factor = 0.02` override so the visibility map stays current on the pure-insert hypertable chunks (the default 0.2 left the day's hot chunk stale before the daily TimescaleDB rollover, degrading the Index Only Scan back to heap fetches). EXPLAIN-verified on a 137 GB field store: two panels that timed out at 15 s now run in 139 ms / 514 ms with 0 heap fetches. Results-invariant perf tuning, so it deliberately does not bump the schema version or gate the viewer. -- **Darling + Lite: the Long-Running Query alert's `sp_server_diagnostics` exclusion no longer fires a false alert when the health-check session is in a non-diagnostics wait** ([#1622]) - the exclusion keyed only off the wait type (`wait_type NOT LIKE '%SP_SERVER_DIAGNOSTICS%'`), but sp_server_diagnostics also does Extended Events work, so a field session (multi39) captured in `PREEMPTIVE_XE_GETTARGETSTATE` slipped the wait-type match and fired a false Long-Running Query alert (self-resolved 36 s later; nil real impact, but the filter missed its named target). The exclusion now also matches the query text (`query_text NOT ILIKE '%sp_server_diagnostics%'`, case-insensitive and NULL-safe), catching the session regardless of the wait it is captured in. Fixed in both apps for parity, plus a pin test. - -- `install-darling.ps1`: the viewer-shortcut step no longer throws when run with no loaded user profile (a Windows service, scheduled task, CI runner, or remote session) where `GetFolderPath('Desktop'/'StartMenu')` returns empty. It creates only the shortcuts whose folder resolves and skips cleanly otherwise. ([#1613]) - -### Documentation - -- **A second stale doc comment describing behaviour that no longer exists** ([#1744]) - `QuiesceTimescaleServerOptions` carried two stacked `` elements, the first left by an earlier edit. XML docs take the LAST summary, so tooling rendered the correct one and only a human reading the file was misled - invisible to the build, the tests and the analyzers, which is why it survived several re-reads and was found by pattern-matching the file instead. Same defect as [#1739]'s, in a second place. The surviving summary keeps both rationales that matter: the timescale/timescaledb#1593 `template0` deadlock behind quiescing the scheduler for the upgrade window, and the IPv4/IPv6 loopback trap behind restoring `listen_addresses=localhost` for it. -- Darling README: added a "verify it's actually reachable" recipe for the opt-in LAN MCP/web endpoints — confirm the listener is on the LAN address (not loopback), the scoped firewall rule covers the client, and the client connects to the box IP rather than `localhost` — plus what to re-run after a reinstall. Enabling an endpoint is not the same as reaching it. ([#1616]) - -## [3.2.0] - 2026-07-21 - -### Added - -- **Darling MCP: bulk add/remove monitored servers — an MCP client can now stand up FLEET monitoring conversationally** ([#1609]) - the Darling MCP server exposed read tools plus the [#1600] Custom Views and [#1608] alert-tuning writes, but no way for an MCP client (or Claude, over MCP) to add or remove the MONITORED SERVERS themselves - onboarding was WPF-Viewer-only (the Add / Manage Servers dialogs). Two new tools close that gap, the direct sibling of the alert-tuning slice: `add_servers` (BULK) takes a JSON ARRAY of server objects and, IN ORDER (sequential, mirroring [#1549]'s bulk-probe, to avoid a probe storm), validates each, connection-tests it IN the service, and saves the new+reachable ones - so "monitor these twenty servers with this login" stands up fleet monitoring in one call; `remove_server` removes one by name. **No divergent second implementation:** the probe is `DarlingServerConnector.ProbeAsync` run IN-PROCESS (the MCP host lives inside the service, which holds the network path + credentials - unlike the Viewer's dialogs, which enqueue a `test_connect` command for the service to run), the case-folded dedupe gate is the shared `ServerIdHelper.BuildStorageName` identity in a `HashSet(OrdinalIgnoreCase)` exactly as the [#1549] bulk dialog uses, the SQL password is DPAPI-encrypted through the SAME `DarlingSecrets.Protect` the service decrypts with at collection time (so it round-trips), and the INSERT mirrors `StoreConfigProvider.SeedMonitoredServersAsync`'s exact column set + `server_id` identity (so a tool-written row JOINs the collected data and the service's reconcile matches it, picking it up within one sweep - no restart). Per server, `add_servers` returns `status` `added` / `duplicate` (a case-variant or exact dup of an existing or earlier-in-batch server, skipped WITHOUT a probe) / `connection_failed` (recorded, the batch CONTINUES) / `invalid` (a bad field, or Entra/MFA/Service-Principal/Managed-Identity auth - interactive MFA is nonsensical headless, the same belt [#1549] applies), and the whole call returns `{added, skipped, failed, results:[...]}`; per Erik, the TLS options `encrypt_mode` (Optional/Mandatory/Strict) and `trust_server_certificate` are EXPOSED so a headless caller sets the connection posture explicitly. **Security (deliberate, scoped):** the tools connect as the least-privilege `mcp` role, now granted - via role PROVISIONING, not a migration - INSERT/UPDATE/DELETE on `config.config_monitored_servers` (mirroring how [#1600]/[#1608] granted their single tables), a single non-secret-KEY table: the `encrypted_password` column stays in the fail-closed secret carve, so `mcp` can WRITE a credential blob (onboarding) but can never READ one back, and it still cannot reach the `config_command` service-credential pivot or a schema-wide config write. **The #1608 beacon grant already covers this:** a `config_monitored_servers` write fires the existing `trg_bump_monitored_servers → config_bump_version` trigger (SECURITY INVOKER, UPDATEs `config_service.config_version` AS `mcp`), and [#1608] already granted `mcp` UPDATE on the two `config_service` beacon columns - verified, no new `config_service` grant is added. **Credential on the wire:** the SQL password travels to the MCP endpoint inside `add_servers`' request JSON (DPAPI-encrypted at rest, never returned by any read tool), which is one more reason a LAN deployment should front the endpoint with the documented TLS reverse proxy - the README's MCP blast-radius section now states this and recommends Windows/integrated auth for onboarded servers where possible. The MCP server instructions (now eighty-seven tools), the cross-app tool-inventory ratchet (Lite is a single-instance app with no central monitored-server store, so these are Darling-only), the `/api/read` write-exclusion set, and the Darling README are updated. Verified: the Darling service + `Darling.Tests` build clean (0 errors, no new warnings in the changed source), full `Darling.Tests` **2847 passed / 0 failed / 146 gated-live skipped** (with `DARLING_TEST_PG` cleared), and `Lite.Tests` `CrossAppMcpToolInventoryPinTests` **2 passed** - the ungated pins cover the two-tool surface, the Gemini-clean schema + required-params, validate-before-write (a malformed payload or a bad/MFA entry returns `invalid` WITHOUT probing or opening a connection), and the pure case-folded dedupe partition; one gated-live test (`DARLING_TEST_PG`, own-scoped + cleaned up, the SQL probe stubbed to success - no live SQL Server touched in CI) proves the store INSERT (the SQL secret is DPAPI-encrypted at rest and round-trips, the Windows-auth server is secret-free), the duplicate skip, the `config_version` self-bump, and `remove_server` (removed, then not_found). The REAL end-to-end probe is what the human dogfoods (remove sql2016, re-add via MCP). - -- **Darling MCP: alert-tuning write tools — an MCP client can now tune thresholds and manage mute rules conversationally** ([#1608]) - the Darling MCP server exposed alert READS (`get_alert_history` / `get_alert_settings` / `get_mute_rules`) but no way for an MCP client (or Claude, over MCP) to CHANGE the alerting - thresholds and mute rules could only be edited in the WPF Viewer's Settings window. Three new tools close that gap, the direct sibling of the [#1600] Custom Views MCP write tools: `update_alert_settings` (a PARTIAL update of the single global alert-settings row - the agent reads via `get_alert_settings`, changes fields, and sends only those back in the SAME nested shape, e.g. `{"cpu":{"threshold_percent":90},"cooldown_minutes":10}`), and `create_mute_rule` / `delete_mute_rule` (add/remove the mute rules the delivery paths honor). **No divergent second implementation:** `update_alert_settings` validates EVERY provided field against the SAME ranges/enums the Viewer's Settings window enforces (`SettingsWindow.BuildAlertRowFromControls` - thresholds in range, `cpu.mode` `sql`/`total`, `delivery.mode` `Summary`/`PerEvent`, counts within bounds) BEFORE any write - an out-of-range value or an unknown field (top-level or nested) returns `{status:"invalid"}` and writes nothing - then applies ONLY the provided columns via a targeted parameterized `UPDATE ... WHERE id = 1` and re-reads the merged state; `create_mute_rule` / `delete_mute_rule` reuse the SAME `PgMuteRuleStore` `get_mute_rules` reads through, with the same GUID id-generation the Viewer's mute-create path uses. SMTP/webhook delivery credentials are out of scope (the `mcp` role cannot read or write the secret columns). A `config_alert_settings` write self-bumps `config_version` via the existing config-table trigger, so the running service HOT-RELOADS the change within one collection sweep (the tool never writes `config_version` itself). **Security (deliberate, scoped):** the tools connect as the least-privilege `mcp` role, now granted - via role PROVISIONING, not a migration - INSERT/UPDATE/DELETE on `config.config_mute_rules` and UPDATE on the singleton `config.config_alert_settings` (mirroring how [#1600] granted `config.custom_views`), so a token-holder can tune alerting but still cannot reach the `config_command` service-credential pivot or the carved secret columns. **One non-obvious grant, called out for review:** the `config_alert_settings` bump trigger (`config_bump_version`) is SECURITY INVOKER and UPDATEs `config_service` AS the writing role, so the `mcp` role ALSO needs a COLUMN-level UPDATE on just the two `config_service` beacon columns (`config_version`, `updated_at`) or every `update_alert_settings` write would fail 42501 in production - and the superuser-run gated-live tests would never catch it. The column grant lets `mcp` bump the reload beacon but NOT flip `paused` / `capture_plans` / `mcp_enabled` / `mcp_port`; the live security test now proves this end-to-end as the real `mcp` role (a `config_alert_settings` UPDATE succeeds and fires the beacon; a `paused` UPDATE still 42501s). The MCP server instructions, the cross-app tool-inventory ratchet (Lite has no central alert store, so these are Darling-only), and the Darling README's MCP blast-radius section are updated. Verified: the Darling service + `Darling.Tests` build clean (0 errors, 0 warnings in the changed source), full `Darling.Tests` **2821 passed / 0 failed / 145 gated-live skipped** (with `DARLING_TEST_PG` cleared), and `Lite.Tests` `CrossAppMcpToolInventoryPinTests` **2 passed** - the ungated pins cover the six-tool surface, the Gemini-clean schema + required-params, and validate-before-write (a bad/unknown partial update returns `invalid` WITHOUT opening a connection); one gated-live test (`DARLING_TEST_PG`, own-scoped + restored) proves `update_alert_settings` flips a threshold AND self-bumps `config_version`, and `create_mute_rule`→`get_mute_rules`→`delete_mute_rule` round-trips, and it skips in the normal unit run like the others. - -- **Darling: headless `--enable-mcp` / `--disable-mcp` / `--enable-web` / `--disable-web` CLI verbs — bring an endpoint up (store + firewall) without the Viewer** ([#1601]) - a headless Darling box had no supported way to (a) turn the MCP or web-dashboard endpoint on/off or (b) open its firewall. Two structural reasons: `mcp.enabled`/`web.enabled` in `darling.json` are only a FIRST-RUN seed - after the first run the store (`config.config_service.mcp_enabled`/`web_enabled`) is authoritative and is normally toggled only by the WPF Viewer's Settings, which a headless deployment does not have; and the service runs as a virtual service account (`NT SERVICE\PerformanceMonitor Darling`) that CANNOT modify Windows Firewall, so its best-effort self-reconcile silently fails. Each verb closes both gaps in one elevated action. **(store)** a TARGETED `UPDATE config.config_service SET = ..., updated_by = 'cli' WHERE id = 1` flips ONLY that endpoint's flag; the existing BEFORE-UPDATE self-bump trigger increments `config_version`, so the worker HOT-RELOADS within one collection sweep - no restart - and the write deliberately never touches `config_version` itself, `paused`, or the other endpoint's flag (0 rows affected ⇒ the store isn't seeded yet, reported as such; the owner credential missing ⇒ the service has never initialized the store, reported as such). **(firewall)** only when the endpoint's `darling.json` network block opts into LAN exposure (a non-loopback `listen`, decided via the shared `DarlingNetwork.IsExposedListenAddress`): run ELEVATED, it opens/removes the SAME scoped, idempotent-by-DisplayName rule the host self-reconciles (the two `McpFirewallRuleName`/`WebFirewallRuleName` builders are now `internal` so the CLI and host act on the EXACT same rule, through the shared `BuildFirewallEnableCommand`/`BuildFirewallDisableCommand` builders); run NON-elevated, the store toggle still succeeds and the exact elevated command is printed as a HANDOFF (never a failure); a loopback-only endpoint gets a note pointing at `--configure-network` and takes no firewall action. A firewall failure is non-fatal. Managed-mode only (BYO governs its own `config_service` + exposure) and Windows-only (DPAPI credential decrypt + `WindowsPrincipal` + firewall), the same guard shape as `--print-viewer-connection`; wired into `IsKnownVerb`, `UsageText`, and the `Program` dispatch (allow-list and dispatch kept in sync per the existing anti-drift comments). The output states the store change, that the running service applies it live within one sweep, the firewall outcome/handoff, and a reminder that `darling.json`'s `enabled` is only the seed - the store is the live switch. No store schema change. Verified: `Darling.Tests` **2789 passed / 0 failed / 144 gated-live skipped** (with `DARLING_TEST_PG` cleared) and 0 new build warnings - the pure tests pin the four store-write SQL strings (right flag, `updated_by='cli'`, `WHERE id=1`, never `config_version`/`paused`/the other endpoint's flag), the verb recognition + classify wiring, the pure firewall-step classifier (exposed × elevated), and the shared rule names; one gated-live test (`DARLING_TEST_PG`, transaction-rolled-back) proves enable then disable flip the flag AND self-bump `config_version` against a throwaway Postgres, and it skips in the normal unit run exactly like the other `*_AgainstDevPostgres` tests (CI's `darling-pg` job runs it live). -- **Darling MCP: `describe_custom_view_catalog` — the compose vocabulary an LLM needs to author a view without guessing** ([#1602]) - [#1600] gave the Darling MCP server the seven Custom Views authoring tools, but an MCP client had no way to DISCOVER what a composed panel may contain. The tool descriptions give the panel SHAPE (a `source` + `measure`/`ratio` + `aggregate` + `unit` + `timeBucket`/`topN` + `groupBy`/`filters` + `viz`) and say "iterate until valid", but no tool enumerated the legal identifiers, and `validate_custom_view`'s errors name the first problem without listing valid values (`"unknown source 'X'"` never says what sources exist). So an assistant asked to "make me a top-waits view" had to GUESS `source`/`measure` names blind and flail on validation errors — the exact wall hit while dogfooding the [#1600] tools, cleared only by reading `MeasureCatalog.cs` source, which a customer's assistant cannot do. A human in the web composer gets dropdowns; the LLM got nothing. **Fix:** a new read-only `describe_custom_view_catalog` tool returns the SAME compose catalog the web composer's picker binds to (`DarlingWebEndpoints.BuildComposeCatalogNode`, already served at `/api/catalog`) — `{measures, dimensions, annotationSources, universalDimensions, unitFamilies, aggregates, timeBuckets, filterOps, viz}`, each measure carrying its `source`, `kind` (scalar/ratio), `validAggregates`, `allowedDimensions`, unit family + default/native unit, and per-server-type `appliesTo`. An LLM calls it ONCE and composes a valid panel on the first try instead of guessing. The MCP server instructions now direct clients to call it FIRST before authoring, and it joins the read-only `/api/read` exclusion set (like the other Custom Views tools it is served by its own richer endpoint, not the `/api/read/{tool}` 1:1 mirror). **No divergent second implementation:** the tool returns the one `BuildComposeCatalogNode` the web UI already serves — no parallel catalog. Static reference data: it reads no monitored server and no collected data, needs no store connection, and is not a write, so it does not widen the MCP token's blast radius. Verified: full `Darling.Tests` green (2768 passed / 0 failed / 143 gated-live skipped locally; the one local failure is the pre-existing timezone-sensitive viewer display test, green on UTC CI), including the updated tool-surface / Gemini-schema / required-param pins (now eight Custom Views tools), the `/api/read`-parity reflection test, and a new pin that the tool surfaces a known measure with its composable fields. - -- **Darling MCP: create and manage Custom Views (CV2) over MCP** ([#1600]) - the Darling MCP server previously exposed ~72 READ tools but no way for an MCP client (or Claude, over MCP) to build a Custom View programmatically - views could only be authored in the web viewer's editor. Seven new tools close that gap and add the server's ONE write surface: `list_custom_views` / `get_custom_view` (read the saved dashboards + notebooks), `validate_custom_view` (dry-run a definition against the measure catalog + composer rules WITHOUT persisting), `create_custom_view` / `update_custom_view` (validate THEN persist; optimistic-concurrency on update), `delete_custom_view`, and `run_custom_view_panel` (compile + run a single composed panel and return `{sql, rows, annotations}` - the composer's live preview, so a generated view can be checked end-to-end). Two payoffs: ask Claude to "make me a view of X" and it builds it, and Claude authoring views exercises the CV2 catalog / spec validation / render path end-to-end - exactly where CV2 gaps hide. **No divergent second implementation:** every tool routes through the EXISTING authority - persistence is the same `CustomViewStore`, validation is the same `DarlingWebEndpoints.ValidateDefinition` (through `ComposeSpec.TryParsePanel` and the hand-authored measure catalog), and the panel run is the SAME compile-and-run extracted from the web `/api/compose/run` into a shared `RunComposedPanelAsync` that now backs both surfaces; create/update ALWAYS validate before persisting, so a stored view can always compile and render. **Security (deliberate, scoped):** the tools connect as the least-privilege `mcp` role, now granted INSERT/UPDATE/DELETE on ONLY `config.custom_views` (mirroring the `viewer` role's single-table grant for the web composer) via role PROVISIONING, not a migration - a token-holder can author custom views but still cannot reach the `config_command` service-credential pivot or the carved secret columns, and a composed query names only `collect.*` collector tables so a stored view structurally cannot read a config control-plane table. The README's MCP blast-radius section is updated to state the widened write. Verified: full `Darling.Tests` green (see the PR body for the exact pass/fail counts), including the provisioning-grant pins, the tool-validation envelopes, and a gated-live round-trip through the tools against a real PostgreSQL. - - -- **Darling Web: Custom Views v2 - QS execution-weighted average + per-server measure availability greying** ([#1584]) - closes the two remaining #1563 deferrals so nothing's parked. **QS execution-weighted average:** a new `Weighted` ratio mode compiles `SUM(value * weight) / NULLIF(SUM(weight), 0)` - the correct execution-weighted mean across Query Store's pre-aggregated per-interval averages, NOT an avg-of-avgs - shipping `qs_avg_duration_us` and `qs_avg_cpu_us` (the QS avg column x `execution_count`), with both operand columns pinned to the collector's `PayloadColumns`. **Per-server measure auto-greying (D4):** `/api/fleet` cards now carry the reliable per-server platform (`engine_edition` + `is_azure_sql_db` / `is_azure_mi`, from the probed servers registry), and the composer GREYS (disables + a reason) measures a SINGLE selected `$server` can't collect (Azure SQL DB / MI / on-prem, matched to the measure's `appliesTo`); an already-chosen measure stays selectable (re-scoping never silently drops a panel's metric), and All / multi-select or an unprobed server keeps the Wave-1 badges. AWS RDS + msdb deliberately stay badge-only (no reliable per-server signal without a new store column). Wired into both the dashboard and notebook composers. All additive; full `Darling.Tests` green (2677 passed / 0 failed / 141 gated-live skipped). - -- **Darling Web: Custom Views v2 - notebook mode (investigation documents)** ([#1583]) - the final #1563 follow-up: an investigation / runbook document that interleaves prose with LIVE composed-panel cells, read top-to-bottom (vs the dashboard's grid for watching) - the "Datadog notebooks" half of the original ask. A notebook is a view `kind` in the SAME `config.custom_views` store: `{ kind:"notebook", cells:[ {type:"markdown",text} | {type:"panel", } ], variables, range }` - reusing the v2 panel spec, `/api/compose/run`, the render path, the store + CRUD, and the view-level `$server`/`$database`/time scope. Each panel cell is validated by the EXACT `TryParsePanel` authority (all the v2 safety); markdown cells cap at 32 KB; `ValidateDefinition` dispatches on `kind` (the dashboard `{panels}` path byte-for-byte unchanged); and the `/api/views` list summary now carries `kind` (a `definition->>'kind'` scalar, never the body) so the sidebar badges + routes notebooks with no extra fetch. **The one genuinely-new component is a minimal, air-gapped, XSS-safe Markdown -> DOM renderer** (`markdown.js`): ATX headings, paragraphs, bold/italic, inline + fenced code, ordered/unordered lists, and links - EVERY node built via `textContent` (NO innerHTML anywhere, no library), so raw markup in the source is inert BY CONSTRUCTION (a `