Query Store reads on the stored interval tables no longer miss an interval that began just before the table's floor (#5541)

Darling/Darling.Tests/ComposeQueryStoreWideExactnessLiveTests.cs

@@ -346,7 +346,7 @@ QueryStoreCollector.Row Row(
             };
 
         /* An anchor interval 45 days before windowStart (mirrors QueryStoreIntervalWideGridLiveTests'
-           SeedGridAsync): pushes this server's table floor comfortably past clause 3's IntervalSpanMargin
+           SeedGridAsync): pushes this server's table floor comfortably past clause 3's PurgeEdgeMargin
            check, so windowStart itself can sit exactly on the seeded span without the gate refusing on table
            floor alone. Outside the compared window, so it never appears in either side's rows. */
         var anchor = windowStart.AddDays(-45);

Darling/Darling.Tests/PlanRegressionDailyReadShapeLiveTests.cs

@@ -36,8 +36,8 @@ namespace Darling.Tests;
 /// server's; the test asserts what holds with that index (it does not scan the whole table, it keeps rows read under a
 /// bound set by the open days, and the day half reads <c>plan_regression_daily</c> through its unique index) and writes the
 /// rows read, the rows returned and the blocks touched to <c>DARLING_5448_SHAPE_OUT</c> when that is set, which is where the
-/// numbers in the PR come from. On a hypertable (the table is converted by hand: the product does not partition it yet,
-/// though <c>QueryStoreIntervalLatest.ChunkFloorsSql</c> already has the arm for it) the live half scans only the chunks
+/// numbers in the PR come from. On a hypertable (the table is converted by hand: the product does not partition it, and the read gate
+/// takes the table floor from the per-server minimum either way, #5541) the live half scans only the chunks
 /// at or above the live floor.</para>
 /// </summary>
 /* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Every test here reaches DARLING_TEST_PG only to

Darling/Darling.Tests/PlanRegressionIntervalTableEquivalenceTests.cs

@@ -151,8 +151,26 @@ public void UseTable_RefusesAWindowReachingBelowTheTablesFloor_WhenRawStillHolds
         /* A held store: raw keeps 30 days, the table's floor is 16 days back, and an anchored pass reaches 20. */
         Assert.False(QueryStoreIntervalLatest.UseTable(now.AddDays(-25), false, now.AddDays(-30), rawBound, now.AddDays(-16)));
 
-        /* The same window where raw holds nothing older than the table: the table. */
-        Assert.True(QueryStoreIntervalLatest.UseTable(now.AddDays(-25), false, now.AddDays(-15), rawBound, now.AddDays(-16)));
+        /* The same window where raw holds nothing older than the table, a full margin clear of its floor: the table. */
+        Assert.True(QueryStoreIntervalLatest.UseTable(now.AddDays(-25), false, now.AddDays(-14), rawBound, now.AddDays(-16)));
+    }
+
+    [Fact]
+    public void UseTable_RawFloorWithinTheMarginOfTheTableFloor_ReadsRaw()
+    {
+        /* #5541: raw drops a snapshot by collection_time, the table drops an interval by first_execution_time. A raw
+           floor at or just above the table floor can still hold a snapshot of an interval the table dropped, so the
+           table is only safe once raw's floor clears the table's by the purge-edge margin. */
+        var now = new DateTime(2026, 9, 23, 0, 0, 0, DateTimeKind.Unspecified);
+        var rawBound = now.AddDays(-20);
+        var h = now.AddDays(-16);
+        var margin = QueryStoreIntervalWide.PurgeEdgeMargin;
+        var minute = TimeSpan.FromMinutes(1);
+
+        Assert.False(QueryStoreIntervalLatest.UseTable(now.AddDays(-25), false, h, rawBound, h));
+        Assert.False(QueryStoreIntervalLatest.UseTable(now.AddDays(-25), false, h + margin - minute, rawBound, h));
+        Assert.True(QueryStoreIntervalLatest.UseTable(now.AddDays(-25), false, h + margin, rawBound, h));
+        Assert.True(QueryStoreIntervalLatest.UseTable(now.AddDays(-25), false, h + margin + minute, rawBound, h));
     }
 
     /* ---- the seed ---------------------------------------------------------------------------------------- */

Darling/Darling.Tests/QueryStoreIntervalWideGateTests.cs

@@ -55,18 +55,52 @@ public void Clause2_FilledSinceAfterMaxOfRawFloorAndWindowStart_ReadsRaw() =>
         Assert.False(Rule(filledSince: WindowEnd));
 
     [Fact]
-    public void Clause3_TableFloorShortOfTheOneDayMargin_ReadsRaw() =>
-        // table floor only reaches 23h before window start — short of the full one-day margin clause 3 needs —
-        // and raw has no floor (a plain store): the table cannot answer for the earliest part of the window.
+    public void Clause3_TableFloorShortOfThePurgeEdgeMargin_ReadsRaw()
+    {
+        /* #5541: the raw read counts a snapshot by collection_time alone, so a snapshot collected at or after the window
+           start can belong to an interval that began down to S - PurgeEdgeMargin (a 1440-minute interval collected up to
+           an hour plus the catch-up cap after it ended). A table floor between S - 26 h and S - 24 h used to pass the
+           old S - 1 day arm and then miss that interval. Raw has no floor here (a plain store), so only 3b can pass. */
+        Assert.False(Rule(tableFloor: WindowStart.AddHours(-24)));
+        Assert.False(Rule(tableFloor: WindowStart - QueryStoreIntervalWide.PurgeEdgeMargin + TimeSpan.FromMinutes(1)));
         Assert.False(Rule(tableFloor: WindowStart.AddHours(-23)));
+    }
 
     [Fact]
-    public void Clause3_TableFloorAtOrBeyondTheOneDayMargin_ReadsTheTable() =>
-        Assert.True(Rule(tableFloor: WindowStart.AddDays(-1)));
+    public void Clause3_ReviewFailingInput_AOneDayIntervalCollectedJustAfterTheWindowStart_ReadsRaw()
+    {
+        /* The round-1 review's pure form: default retention (raw keeps 30 days, so R is far below H and 3a fails),
+           H = S - 24 h, a 12 h minimum window. It returned true before the fix. */
+        var s = WindowStart;
+        Assert.False(QueryStoreIntervalWide.UseTable(
+            s.AddDays(-1), false, s.AddDays(-10), s, s.AddDays(1), null, s.AddDays(1), s.AddHours(-24), TimeSpan.FromHours(12)));
+    }
 
     [Fact]
-    public void Clause3_RawFloorAtOrAboveTableFloor_ReadsTheTable_EvenOutsideTheMargin() =>
-        Assert.True(Rule(rawFloor: WindowStart.AddDays(-3), tableFloor: WindowStart.AddDays(-3)));
+    public void Clause3_TableFloorAtOrBeyondThePurgeEdgeMargin_ReadsTheTable()
+    {
+        Assert.True(Rule(tableFloor: WindowStart - QueryStoreIntervalWide.PurgeEdgeMargin));
+        Assert.True(Rule(tableFloor: WindowStart - QueryStoreIntervalWide.PurgeEdgeMargin - TimeSpan.FromMinutes(1)));
+    }
+
+    [Fact]
+    public void Clause3_RawFloorAtOrAboveTableFloorPlusMargin_ReadsTheTable_EvenOutsideTheMargin()
+    {
+        /* The table floor sits 12h under the window start, short of the purge-edge margin, so only 3a can pass. */
+        var h = WindowStart.AddHours(-12);
+        Assert.True(Rule(rawFloor: h + QueryStoreIntervalWide.PurgeEdgeMargin, tableFloor: h));
+        Assert.True(Rule(rawFloor: h + QueryStoreIntervalWide.PurgeEdgeMargin + TimeSpan.FromMinutes(1), tableFloor: h));
+    }
+
+    [Fact]
+    public void Clause3_RawFloorWithinTheMarginOfTheTableFloor_ReadsRaw()
+    {
+        /* #5541: raw drops a snapshot by collection_time, the table drops an interval by first_execution_time, so
+           raw at or just above the table floor can still hold a snapshot of an interval the table dropped. */
+        var h = WindowStart.AddHours(-12);
+        Assert.False(Rule(rawFloor: h, tableFloor: h));
+        Assert.False(Rule(rawFloor: h + QueryStoreIntervalWide.PurgeEdgeMargin - TimeSpan.FromMinutes(1), tableFloor: h));
+    }
 
     [Fact]
     public void Clause4_OpenEnd_SkipsTheAppliedThroughComparison() =>

Darling/Darling.Tests/QueryStoreIntervalWideGridLiveTests.cs

@@ -124,7 +124,7 @@ await ExecAsync(connection,
     /// <summary>
     /// Review D4R H1: a window under <see cref="QueryStoreIntervalWide.GridWideMinWindow"/> must issue ZERO
     /// round trips against <c>collect.query_store_interval_wide</c> — no <c>ReadSourceInputsSql</c>, no
-    /// <c>ChunkFloorsSql</c>, and above all no <c>PlainTableFloorSql</c> (the unindexed
+    /// <c>RawChunkFloorSql</c>, and above all no <c>PlainTableFloorSql</c> (the unindexed
     /// <c>MIN(first_execution_time) WHERE server_id = $1</c> scan) — counted directly off
     /// <c>pg_stat_user_tables.seq_scan</c>/<c>idx_scan</c> before and after the call, since PostgreSQL's own
     /// catalog is the only seam that tells "no statement ran" apart from "a statement ran and returned

Darling/Darling.Tests/QueryStoreTopLiteralEndStraddleLiveTests.cs

@@ -200,10 +200,10 @@ private static async Task SeedStraddleAsync(DarlingCollectorRunner runner, Cance
 
         /* Clause 3's own floor control: UseTable's tableFloor is MIN(first_execution_time) across the WHOLE
            server, not just this identity, and clause 3 refuses unless raw's chunk floor or WindowStart minus
-           IntervalSpanMargin ("skewFloor") reaches at or past it. Left with only the 7001 identity below, the
-           table's floor sits at day0 (WindowStart + 1h) - AFTER skewFloor (WindowStart - 1 day) - so clause 3
+           PurgeEdgeMargin ("edgeFloor") reaches at or past it. Left with only the 7001 identity below, the
+           table's floor sits at day0 (WindowStart + 1h) - AFTER edgeFloor (WindowStart - 26 h) - so clause 3
            alone refuses even the open-end positive control, before clause 4 is ever reached. A second,
-           unrelated identity anchored at or before skewFloor pulls the server's table floor down there, so
+           unrelated identity anchored at or before edgeFloor pulls the server's table floor down there, so
            clause 3 passes and only clause 4 (LiteralEnd vs applied_through) can still refuse the straddle read. */
         var floorAnchorFirst = WindowStart.AddDays(-2);
         var floorAnchor = new QueryStoreCollector.Row

Darling/Darling.Tests/QueryStoreWideFleetEligibilityLiveTests.cs

@@ -157,7 +157,7 @@ public async Task FleetCheck_ReadsTheStoreWideFloorsOnce_AndEqualsTheSerialAnswe
 
         /* Scoped to this scratch database by name, so a class running in parallel against another database is
            invisible to the count. */
-        using var floorReads = new NpgsqlCommandCounter(scratch.DatabaseName, "table_is_hypertable", "raw_floor");
+        using var floorReads = new NpgsqlCommandCounter(scratch.DatabaseName, "raw_floor");
         using var controlReads = new NpgsqlCommandCounter(scratch.DatabaseName, "fleet_floor_count_control");
         await using (var control = postgres.CreateCommand("SELECT 1 AS fleet_floor_count_control"))
         {
@@ -191,7 +191,7 @@ public async Task FleetCheck_WhenTheFirstServerRefusesBeforeTheFloors_ReadsNoFlo
 
         /* A literal end before applied_through refuses at the step ahead of the floors read. */
         var literalEnd = S.AddHours(13);
-        using var floorReads = new NpgsqlCommandCounter(scratch.DatabaseName, "table_is_hypertable", "raw_floor");
+        using var floorReads = new NpgsqlCommandCounter(scratch.DatabaseName, "raw_floor");
         using var controlReads = new NpgsqlCommandCounter(scratch.DatabaseName, "fleet_floor_count_control");
         await using (var control = postgres.CreateCommand("SELECT 1 AS fleet_floor_count_control"))
         {

Darling/PerformanceMonitor.Darling.Service/DarlingRetention.cs

@@ -185,7 +185,7 @@ internal static int EffectivePurgeRetentionDays(string collectorName, int resolv
     /// different windows. The table keeps 9 days and deletes row by row on <c>first_execution_time</c>. A Query
     /// Store interval can span a day, so 8 days is exactly the edge for an interval that starts just past the
     /// horizon and closes inside a 7-day window; 9 days gives a day of margin. Reads use the table below raw's
-    /// chunk floor, down to L = max(window start, filled_since, table floor + 1 day), where the table floor is
+    /// chunk floor, down to L = max(window start, filled_since, table floor + 26 h (the purge-edge margin)), where the table floor is
     /// the oldest <c>first_execution_time</c> the purge has left. Like <see cref="QueryStoreIntervalLatestRetentionDays"/>
     /// this is a named constant, not a knob, and in no collector schedule.
     /// </summary>
@@ -803,8 +803,8 @@ would otherwise stay forever. Failure-isolated like every sibling. */
                (QueryStoreIntervalWideRetentionDays) on first_execution_time — a shorter horizon than V143's 15
                days. Same batched-DELETE shape, same pending-replay horizon reasoning, failure-isolated like every
                sibling. The purge deletes row by row on first_execution_time, so the table floor the read gate
-               checks (MIN(first_execution_time)) advances as it runs, and reads below raw's floor stop one day
-               above it (L = max(window start, filled_since, table floor + 1 day)). */
+               checks (MIN(first_execution_time)) advances as it runs, and reads below raw's floor stop 26 h
+               above it (L = max(window start, filled_since, table floor + 26 h, the purge-edge margin)). */
             var intervalWideDeleted = await PurgeOneAsync(
                 postgres, QueryStoreIntervalWide.TableName,
                 TimeSlicedDeleteSql("collect." + QueryStoreIntervalWide.TableName, "first_execution_time"),

Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalLatest.cs

@@ -327,37 +327,26 @@ LEFT JOIN collect.query_store_interval_latest_coverage AS c
   ON c.server_id = $1;";
 
     /// <summary>
-    /// The floors, from TimescaleDB's catalog (metadata, never a scan): raw's oldest chunk (NULL when raw is not a
-    /// hypertable, which the rule reads as "raw holds everything"), and the table's, when the table is one. A
-    /// chunk's <c>range_start</c> is at or below its oldest row, so both err toward raw. <c>AT TIME ZONE 'UTC'</c>
-    /// collapses the catalog's <c>timestamptz</c> to the store's naive-UTC discipline, as DarlingRetention does.
+    /// Raw's floor, from TimescaleDB's catalog (metadata, never a scan): its oldest chunk (NULL when raw is not a
+    /// hypertable, which the rule reads as "raw holds everything"). A chunk's <c>range_start</c> is at or below its
+    /// oldest row, so a low <c>R</c> errs toward raw. <c>AT TIME ZONE 'UTC'</c> collapses the catalog's
+    /// <c>timestamptz</c> to the store's naive-UTC discipline, as DarlingRetention does. The TABLE's floor is never
+    /// read from the catalog (#5541): neither interval table is a hypertable in production (retention purges both with
+    /// a sliced DELETE), and a chunk's <c>range_start</c> would sit far below the real floor anyway, because a DELETE
+    /// leaves emptied chunks listed, so a catalog <c>H</c> would move the rule toward the table, not toward raw.
     /// </summary>
-    public const string ChunkFloorsSql = @"
+    public const string RawChunkFloorSql = @"
 SELECT
-    (
-        SELECT
-            MIN(ch.range_start) AT TIME ZONE 'UTC'
-        FROM timescaledb_information.chunks AS ch
-        WHERE ch.hypertable_schema = 'collect'
-        AND   ch.hypertable_name = 'query_store_stats'
-    ) AS raw_floor,
-    EXISTS
-    (
-        SELECT
-            1
-        FROM timescaledb_information.hypertables AS h
-        WHERE h.hypertable_schema = 'collect'
-        AND   h.hypertable_name = 'query_store_interval_latest'
-    ) AS table_is_hypertable,
-    (
-        SELECT
-            MIN(ch.range_start) AT TIME ZONE 'UTC'
-        FROM timescaledb_information.chunks AS ch
-        WHERE ch.hypertable_schema = 'collect'
-        AND   ch.hypertable_name = 'query_store_interval_latest'
-    ) AS table_floor;";
+    MIN(ch.range_start) AT TIME ZONE 'UTC' AS raw_floor
+FROM timescaledb_information.chunks AS ch
+WHERE ch.hypertable_schema = 'collect'
+AND   ch.hypertable_name = 'query_store_stats';";
 
-    /// <summary>The table's floor for one server where the table is a plain heap: its oldest interval.</summary>
+    /// <summary>
+    /// The table's floor <c>H</c> for one server: its oldest interval. Always this read (#5541), whether or not the
+    /// table is ever converted to a hypertable, because it is exact: the purge deletes oldest first on
+    /// <c>first_execution_time</c>, so every deleted row is below the remaining minimum.
+    /// </summary>
     public const string PlainTableFloorSql = @"
 SELECT
     MIN(t.first_execution_time)
@@ -370,10 +359,19 @@ FROM collect.query_store_interval_latest AS t
     /// <item><c>F</c> exists: this build has claimed coverage for the server.</item>
     /// <item><c>F &lt;= max(R, B)</c>: the table holds every snapshot the raw read would read (it reads
     /// <c>collection_time &gt;= max(R, B)</c>), and may hold more, which is the ruled window extension.</item>
-    /// <item><c>R &gt;= H</c> or <c>B &gt;= H</c>: raw holds no history the table has dropped, or every interval the
-    /// window needs starts inside the table (<c>B</c> is the window minus a day, and an interval spans at most a
-    /// day). A NULL <c>H</c> means the table holds nothing for the server, and then clause 2 already says raw holds
-    /// nothing in the window either.</item>
+    /// <item><c>R &gt;= H + <see cref="QueryStoreIntervalWide.PurgeEdgeMargin"/></c> or <c>B &gt;= H</c>: raw holds no
+    /// snapshot of an interval the table has dropped, or every interval the window needs starts inside the table
+    /// (<c>B</c> is the window minus a day, and an interval spans at most a day). The margin is there because the two
+    /// purge on different columns (#5541): raw drops a snapshot by <c>collection_time</c>, the table drops an interval
+    /// by <c>first_execution_time</c>, and a snapshot can be collected up to one interval length plus the collector's
+    /// lag after its interval began. So with <c>R</c> only at or above <c>H</c>, raw can still hold a snapshot of an
+    /// interval that began below <c>H</c> and that the table dropped; <c>R</c> a full margin above <c>H</c> rules that
+    /// out (same bound as the Wide gate's below-floor edge). The <c>B &gt;= H</c> arm needs no margin here, unlike the
+    /// Wide gate's twin: this rule's raw read also filters <c>last_execution_time &gt;= $2</c>, which keeps every
+    /// interval it counts at <c>first_execution_time &gt;= B</c> (an interval spans at most a day), so an interval
+    /// the table purged (first execution below <c>H</c>) is below <c>B</c> and raw never counts it either. The Wide raw
+    /// read filters <c>collection_time</c> only. A NULL <c>H</c> means the table holds nothing for the
+    /// server, and then clause 2 already says raw holds nothing in the window either.</item>
     /// <item>No pending batch: a batch that stored raw but missed the table is not in the claim.</item>
     /// </list>
     /// A NULL <c>R</c> is raw with no chunk floor (a plain store, whose raw keeps 30 days): minus infinity, which can
@@ -397,7 +395,7 @@ public static bool UseTable(DateTime? filledSince, bool hasPending, DateTime? ra
             return true;
         }
 
-        return (rawFloor is DateTime floor && floor >= h) || rawBound >= h;
+        return (rawFloor is DateTime floor && floor >= h + QueryStoreIntervalWide.PurgeEdgeMargin) || rawBound >= h;
     }
 
     /// <summary>
@@ -434,21 +432,15 @@ public static async Task<bool> ReadsTableAsync(
             }
 
             DateTime? rawFloor = null;
-            DateTime? tableFloor = null;
-            var tableIsHypertable = false;
             if (hasTimescale)
             {
-                await using var floors = new NpgsqlCommand(ChunkFloorsSql, connection) { CommandTimeout = commandTimeoutSeconds };
-                await using var reader = await floors.ExecuteReaderAsync(cancellationToken);
-                await reader.ReadAsync(cancellationToken);
-                rawFloor = reader.IsDBNull(0) ? null : reader.GetDateTime(0);
-                tableIsHypertable = reader.GetBoolean(1);
-                tableFloor = reader.IsDBNull(2) ? null : reader.GetDateTime(2);
+                await using var floors = new NpgsqlCommand(RawChunkFloorSql, connection) { CommandTimeout = commandTimeoutSeconds };
+                rawFloor = await floors.ExecuteScalarAsync(cancellationToken) as DateTime?;
             }
 
-            if (!tableIsHypertable)
+            DateTime? tableFloor;
+            await using (var plain = new NpgsqlCommand(PlainTableFloorSql, connection) { CommandTimeout = commandTimeoutSeconds })
             {
-                await using var plain = new NpgsqlCommand(PlainTableFloorSql, connection) { CommandTimeout = commandTimeoutSeconds };
                 plain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = serverId });
                 tableFloor = await plain.ExecuteScalarAsync(cancellationToken) as DateTime?;
             }