From bf80a7b6a58ab0289b41fba5a9e715bffd4aa28e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:33:47 -0400 Subject: [PATCH 1/2] A fresh store's fleet overview answers instead of failing with 'Out of range of DateTime' before collection_health_hourly materializes (#3973) CollectionHealthWatermarkSql guarded the aggregate's watermark with isfinite(), expecting "nothing materialized" to read -infinity. TimescaleDB (2.28.1, 2.30.1) reports the minimum finite timestamp, 4714-11-24 BC, both before the first refresh and after a refresh over an empty log. Npgsql cannot convert that to a DateTime, and the InvalidCastException escaped the guard's catch (PostgresException). So get_fleet_overview and /api/fleet failed on every fresh store until rows had landed and a refresh had covered them. - A watermark before MaterializedWatermarkFloor (2000-01-01) now reads as nothing materialized: NULL, the all-real-time path, which is exact. - The guard also treats a value the client cannot convert as a failed guard read (the exact raw scan), as its contract already said. - A TimescaleSupport doc that repeated the -infinity assumption is corrected. Live tests build a scratch store the product's way with a never-refreshed aggregate. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_018Z4T51J4E6L8fjriPHpsif --- .../Darling.Tests/FreshStoreWatermarkTests.cs | 200 ++++++++++++++++++ .../Mcp/DarlingFleetReader.cs | 26 ++- .../TimescaleSupport.cs | 5 +- 3 files changed, 223 insertions(+), 8 deletions(-) create mode 100644 Darling/Darling.Tests/FreshStoreWatermarkTests.cs diff --git a/Darling/Darling.Tests/FreshStoreWatermarkTests.cs b/Darling/Darling.Tests/FreshStoreWatermarkTests.cs new file mode 100644 index 0000000000..d007fe14fd --- /dev/null +++ b/Darling/Darling.Tests/FreshStoreWatermarkTests.cs @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// #3973: the fleet overview answers on a store whose collection_health_hourly aggregate has never +/// refreshed. That is every fresh TimescaleDB store until the aggregate's first policy run. +/// +/// The defect. guarded the watermark +/// with isfinite() alone, on the assumption that "nothing materialized" reads -infinity. A +/// never-refreshed aggregate reports the minimum FINITE timestamp instead, 4714-11-24 BC, on TimescaleDB 2.28.1 +/// and 2.30.1 alike. It passed the guard, Npgsql could not convert it to a , and +/// get_fleet_overview answered "Out of range of DateTime" while /api/fleet returned a bare 500. +/// +public sealed class FreshStoreWatermarkTests +{ + /// + /// The guard reads any watermark before the floor as nothing materialized, whichever sentinel the extension + /// uses. The floor is a date no store can have collected before, so no real refresh can put the watermark + /// below it, and it is well inside what a can hold. + /// + [Fact] + public void TheWatermarkGuard_ReadsAnythingBeforeTheFloor_AsNothingMaterialized() + { + var sql = DarlingFleetReader.CollectionHealthWatermarkSql; + + Assert.Contains("isfinite(w)", sql, StringComparison.Ordinal); + Assert.Contains( + $"w >= '{DarlingFleetReader.MaterializedWatermarkFloor:yyyy-MM-dd}'::timestamp", + sql, + StringComparison.Ordinal); + Assert.Equal(new DateTime(2000, 1, 1), DarlingFleetReader.MaterializedWatermarkFloor); + Assert.True(DarlingFleetReader.MaterializedWatermarkFloor > DateTime.MinValue); + } + + /// + /// And the guard's catch covers a value the client cannot convert, not only a server error: a guard read is + /// allowed to fail, and a failed guard means the exact raw scan, never a failed overview. Pinned on the text + /// because no guard read the SQL above lets through can produce one any more. + /// + [Fact] + public void AGuardReadTheClientCannotConvert_FallsBackToTheRawScan() + { + var reader = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "Mcp", "DarlingFleetReader.cs"); + + Assert.Contains("catch (Exception ex) when (ex is PostgresException or InvalidCastException)", reader, StringComparison.Ordinal); + } +} + +/// +/// Gated (DARLING_TEST_PG, TimescaleDB): the whole path on a store built the product's way. The aggregate is +/// created WITH NO DATA by , and its policy is +/// switched off so nothing refreshes behind the test (). +/// The premise is measured rather than assumed, then the overview is read before and after the first refresh. +/// +/* #1776 own-store, like CollectionHealthAggregateTests: every test here creates and drops its own scratch database + and never touches the shared one, so it is deliberately NOT in the "live-postgres" collection. */ +public sealed class FreshStoreWatermarkLiveTests +{ + [Fact] + public async Task TheOverview_AnswersBeforeTheFirstRefresh_AndAfterIt_AgainstDevPostgres() + { + var ct = TestContext.Current.CancellationToken; + var store = await CollectionHealthAggregateTests.OpenStoreAsync(ct); + Assert.SkipWhen(store is null, "Set DARLING_TEST_PG to a Postgres connection string with TimescaleDB to run the live #3973 test."); + var (scratch, connection, jobId) = store!.Value; + await using var _s = scratch; + await using var _c = connection; + + /* The premise: never refreshed, the extension's watermark is below the floor — and on the versions this + was measured on it is FINITE, which is the value isfinite() alone let through. Read as text, because + the point is that Npgsql cannot hold it as a DateTime. */ + var (finite, belowFloor, text) = await RawWatermarkAsync(connection, ct); + Assert.True(belowFloor, $"a never-refreshed watermark read {text}, which is not below the floor"); + TestContext.Current.SendDiagnosticMessage($"#3973 never-refreshed watermark: {text} (isfinite = {finite})"); + + /* Two servers the product would card, and three days of their collector runs. */ + foreach (var id in new[] { 1, 2 }) + { + await ExecAsync(connection, ct, + "INSERT INTO servers (server_id, server_name, display_name, is_enabled, created_date, modified_date) VALUES ($1, $2, $2, TRUE, $3, $3)", + id, "srv-" + id, DateTime.UtcNow.AddDays(-9)); + } + + await CollectionHealthAggregateTests.PlantAsync(connection, TimeSpan.FromDays(3), TimeSpan.FromMinutes(20), 20, 3_973_000, ct); + + /* The guard: nothing materialized reads as no watermark, and the composed read is usable, every bucket + served real-time from raw. This read is what threw before the fix. */ + Assert.Null(await ScalarAsync(connection, DarlingFleetReader.CollectionHealthWatermarkSql, ct)); + + var now = DateTime.UtcNow; + var headEnd = DarlingFleetReader.CeilingHour(DateTime.SpecifyKind(now.AddDays(-7), DateTimeKind.Unspecified)); + await using (var postgres = NpgsqlDataSource.Create(scratch.ConnectionString)) + { + Assert.True(await DarlingFleetReader.CollectionHealthRollupUsableAsync(postgres, headEnd, ct)); + + /* The surface #3973 was filed about: the overview answers, and both cards carry the collection-health + half — three collectors each, read through the unmaterialized aggregate. */ + var fresh = await DarlingFleetReader.GetFleetOverviewAsync(postgres, now.AddHours(-1), now, now, cancellationToken: ct); + AssertThreeCollectorsEach(fresh); + } + + /* After the first refresh the watermark is a real instant, and the same read keeps answering. A new data + source, so the collection-health memo is not what answers. */ + await CollectionHealthAggregateTests.RunPolicyAsync(connection, jobId, ct); + var mark = await ScalarAsync(connection, DarlingFleetReader.CollectionHealthWatermarkSql, ct); + Assert.IsType(mark); + Assert.True((DateTime)mark! > DarlingFleetReader.MaterializedWatermarkFloor); + + await using (var postgres = NpgsqlDataSource.Create(scratch.ConnectionString)) + { + var refreshed = await DarlingFleetReader.GetFleetOverviewAsync(postgres, now.AddHours(-1), now, now, cancellationToken: ct); + AssertThreeCollectorsEach(refreshed); + } + } + + /// + /// The case the issue left unmeasured: a store with nothing in collection_log yet, whose aggregate's + /// first policy run refreshes an empty window. Whatever that leaves the watermark at, the overview has to + /// answer, and the guard has to hand the reader either nothing or a real instant. + /// + [Fact] + public async Task ARefreshOverAnEmptyLog_StillLeavesTheOverviewAnswering_AgainstDevPostgres() + { + var ct = TestContext.Current.CancellationToken; + var store = await CollectionHealthAggregateTests.OpenStoreAsync(ct); + Assert.SkipWhen(store is null, "Set DARLING_TEST_PG to a Postgres connection string with TimescaleDB to run the live #3973 test."); + var (scratch, connection, jobId) = store!.Value; + await using var _s = scratch; + await using var _c = connection; + + await CollectionHealthAggregateTests.RunPolicyAsync(connection, jobId, ct); + + var (finite, belowFloor, text) = await RawWatermarkAsync(connection, ct); + TestContext.Current.SendDiagnosticMessage($"#3973 watermark after a refresh over an empty log: {text} (isfinite = {finite}, below floor = {belowFloor})"); + + var guarded = await ScalarAsync(connection, DarlingFleetReader.CollectionHealthWatermarkSql, ct); + Assert.True(guarded is null || (DateTime)guarded > DarlingFleetReader.MaterializedWatermarkFloor); + + await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); + var now = DateTime.UtcNow; + var result = await DarlingFleetReader.GetFleetOverviewAsync(postgres, now.AddHours(-1), now, now, cancellationToken: ct); + Assert.Equal(0, result.TotalServers); + } + + private static void AssertThreeCollectorsEach(FleetOverviewResult result) + { + var cards = result.Cards.Where(c => c.ServerId is 1 or 2).ToList(); + Assert.Equal(2, cards.Count); + Assert.All(cards, c => Assert.Equal(3, c.CollectorCount)); + } + + /// The aggregate's watermark as the extension reports it, unguarded: whether it is finite, whether it + /// is below the floor, and its text, which is the only way to hold a value a DateTime cannot. + private static async Task<(bool Finite, bool BelowFloor, string Text)> RawWatermarkAsync(NpgsqlConnection connection, CancellationToken ct) + { + await using var read = new NpgsqlCommand($@" +SELECT isfinite(w), w < '{DarlingFleetReader.MaterializedWatermarkFloor:yyyy-MM-dd}'::timestamp, w::text +FROM (SELECT _timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(mat_hypertable_id)) AS w + FROM _timescaledb_catalog.continuous_agg + WHERE user_view_schema = 'collect' + AND user_view_name = '{TimescaleSupport.CollectionHealthHourlyView}') x", connection); + await using var reader = await read.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct), "the aggregate is not in the catalog"); + return (reader.GetBoolean(0), reader.GetBoolean(1), reader.GetString(2)); + } + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection); + var value = await command.ExecuteScalarAsync(ct); + return value is DBNull ? null : value; + } + + private static async Task ExecAsync(NpgsqlConnection connection, CancellationToken ct, string sql, params object[] args) + { + await using var command = new NpgsqlCommand(sql, connection); + foreach (var arg in args) + { + command.Parameters.AddWithValue(arg is DateTime dt ? DateTime.SpecifyKind(dt, DateTimeKind.Unspecified) : arg); + } + + await command.ExecuteNonQueryAsync(ct); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index ea2a4b8bae..6bb59fe57f 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -674,12 +674,25 @@ AND server_id <> 0 internal const string CollectionHealthRollupProbeSql = "SELECT to_regclass('collect." + TimescaleSupport.CollectionHealthHourlyView + "') IS NOT NULL"; + /// The earliest watermark that can mean something was materialized (#3973): no Darling store holds + /// a collection from before 2000, so no real refresh can leave the watermark below it. Anything earlier is + /// the extension's "nothing materialized" sentinel, whichever one it uses. + internal static readonly DateTime MaterializedWatermarkFloor = new(2000, 1, 1); + /// The aggregate's watermark — the instant below which it serves ONLY materialized buckets. NULL - /// before the first refresh (-infinity: nothing materialized, the whole window is served real-time - /// from raw). Run only after said the aggregate exists (which - /// implies TimescaleDB, so the catalog is there). + /// while nothing is materialized, when the whole window is served real-time from raw. Run only after + /// said the aggregate exists (which implies TimescaleDB, so the + /// catalog is there). + /// + /// "Nothing materialized" is not -infinity (#3973). The extension reports the minimum + /// FINITE timestamp, 4714-11-24 BC, both before the first refresh and after a refresh over a window with no + /// rows (measured on TimescaleDB 2.28.1 and 2.30.1). isfinite() passes it, and Npgsql cannot hold it + /// as a , so the fleet overview failed with "Out of range of DateTime" on every fresh + /// store until rows had landed and a refresh had covered them. Anything before + /// now reads as nothing materialized, whichever sentinel produced + /// it. internal const string CollectionHealthWatermarkSql = @" -SELECT CASE WHEN isfinite(w) THEN w END +SELECT CASE WHEN isfinite(w) AND w >= '2000-01-01'::timestamp THEN w END FROM (SELECT _timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(mat_hypertable_id)) AS w FROM _timescaledb_catalog.continuous_agg WHERE user_view_schema = 'collect' @@ -759,7 +772,8 @@ internal static DateTime CeilingHour(DateTime windowStart) /// pass, else the exact raw scan. (a) ABSENT: no aggregate (plain PostgreSQL, or not yet created) → raw. /// (b) CONTINUITY: any whole hour in [head end, watermark) missing as a bucket → raw /// ( says why). A guard read that fails is treated as a failed - /// guard, availability-first: the raw scan is always exact, only slower. + /// guard, availability-first: the raw scan is always exact, only slower. That includes a value the client + /// cannot convert (#3973): a guard read that produced one used to fail the whole overview instead. /// internal static async Task CollectionHealthRollupUsableAsync( NpgsqlDataSource postgres, DateTime headEnd, CancellationToken cancellationToken) @@ -795,7 +809,7 @@ internal static async Task CollectionHealthRollupUsableAsync( var present = Convert.ToInt64(await count.ExecuteScalarAsync(cancellationToken)); return present >= expected; } - catch (PostgresException) + catch (Exception ex) when (ex is PostgresException or InvalidCastException) { return false; } diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index d41f6e150e..2ff2e85a02 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -939,8 +939,9 @@ public static string BaselineRelationExistsSql(string view) /// /// COVERAGE IS READ OFF THE MATERIALIZATION, NOT THROUGH THE VIEW (#3653). Measured on a /// TimescaleDB 2.28.1 rig while proving the interval-honest successors' rollout: a continuous aggregate - /// created WITH NO DATA has a watermark of -infinity, so its real-time branch serves EVERY - /// raw row and min(bucket) through the view is raw's own oldest bucket. The gate then reads + /// created WITH NO DATA has its watermark at the minimum timestamp (4714-11-24 BC — finite, not + /// -infinity, which is what #3973 tripped over), so its real-time branch serves EVERY raw row and + /// min(bucket) through the view is raw's own oldest bucket. The gate then reads /// "already covers" on the very start that created the aggregate and skips the backfill; one policy /// refresh later the watermark jumps to the trailing window's end and everything older than that window is /// served by neither branch — nineteen of twenty planted days went invisible on the rig — until the NEXT From c51f687aed658759889c1c1a2ff4aca20c8719a1 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Tue, 22 Sep 2026 22:34:50 -0400 Subject: [PATCH 2/2] Say which TimescaleDB version each watermark measurement was taken on (#3973) The empty-log refresh was measured on 2.28.1 only; the never-refreshed sentinel on 2.28.1 and 2.30.1. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_018Z4T51J4E6L8fjriPHpsif --- .../Mcp/DarlingFleetReader.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index 6bb59fe57f..3dfc25b81a 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -685,10 +685,10 @@ AND server_id <> 0 /// catalog is there). /// /// "Nothing materialized" is not -infinity (#3973). The extension reports the minimum - /// FINITE timestamp, 4714-11-24 BC, both before the first refresh and after a refresh over a window with no - /// rows (measured on TimescaleDB 2.28.1 and 2.30.1). isfinite() passes it, and Npgsql cannot hold it - /// as a , so the fleet overview failed with "Out of range of DateTime" on every fresh - /// store until rows had landed and a refresh had covered them. Anything before + /// FINITE timestamp, 4714-11-24 BC, before the first refresh (measured on TimescaleDB 2.28.1 and 2.30.1) + /// and still after a refresh over a window with no rows (measured on 2.28.1). isfinite() passes it, + /// and Npgsql cannot hold it as a , so the fleet overview failed with "Out of range of + /// DateTime" on every fresh store until rows had landed and a refresh had covered them. Anything before /// now reads as nothing materialized, whichever sentinel produced /// it. internal const string CollectionHealthWatermarkSql = @"