From e970834bacf3f2ab2ace7580609d6214991e14e4 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:06:25 -0400 Subject: [PATCH 1/9] Legacy-interior scan window for MaterializationHoleScanWindows (#4301) Adds an optional third window kind to MaterializationHoleScanWindows, anchored inside a frozen legacy's own materialized span (below its last bucket), for a pre-freeze outage hole H2 identified as invisible to both the existing seam and ordinary windows. Purely additive: absent callers get the unchanged two-window shape. Pin A only (pure-function unit tests). The SQL probe (RetentionArmSafetySql), the RepairMaterializationHolesAsync branch, and the UNKNOWN-vs-provably-unrepairable classification are NOT in this commit -- see the handoff note. --- .../MaterializationHoleRepairTests.cs | 56 +++++++++++++++++++ .../TimescaleSupport.MaterializationHoles.cs | 27 ++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs index 9c27188434..ea112fed4c 100644 --- a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs +++ b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs @@ -341,6 +341,62 @@ public void ScanWindows_EdgeCases_OneBucketSeam_NoWindowsWhenNothingToScan_AndTh Assert.Throws(() => TimescaleSupport.MaterializationHoleScanWindows(Hour, Hour, Hour, Hour, TimeSpan.Zero)); } + /// + /// #4301 (H2): a legacy-interior gap — a hole BELOW the legacy's last bucket, inside its own frozen span, + /// from an outage that predates this store ever taking the freeze — gets a THIRD window, distinct from + /// both the seam window and the ordinary window, emitted FIRST (oldest of the three). RED on the pre-#4301 + /// signature: MaterializationHoleScanWindows had no parameter through which a legacy-interior span + /// could ever reach this method at all, so it returned at most the seam window and the ordinary window — + /// never anything anchored below l.mx — exactly the H2 regression this pins. + /// + [Fact] + public void ScanWindows_LegacyInterior_GivesAThirdWindow_EmittedFirst() + { + var width = TimescaleSupport.HourlyBucket; + var floor = Hour.AddHours(200); + var horizon = Hour.AddHours(190); + var seamFloor = Hour.AddHours(195); + var ceiling = Hour.AddHours(250); + var legacyInteriorFrom = Hour.AddHours(20); + var legacyInteriorTo = Hour.AddHours(180); + + var windows = TimescaleSupport.MaterializationHoleScanWindows( + floor, ceiling, horizon, seamFloor, width, legacyInteriorFrom, legacyInteriorTo); + + Assert.Equal(3, windows.Count); + Assert.Equal((legacyInteriorFrom, legacyInteriorTo), windows[0]); + Assert.Equal((seamFloor, floor.AddHours(-1)), windows[1]); + Assert.Equal((floor, ceiling), windows[2]); + + /* The legacy-interior window is unclamped by the horizon — same reasoning as the seam window: the + outage that opened it left the source with no rows there, not purged, so it can sit however far + below the horizon it needs to. */ + Assert.True(windows[0].From < horizon); + } + + /// + /// #4301: no legacy-interior span (both bounds null, the ordinary case once a store has run a while, or + /// any relation without a frozen legacy) yields exactly the same two windows #4186 already produced — + /// the new parameter is purely additive and does not disturb the seam/ordinary shape when it is absent. + /// An inverted or empty span (from after to) is also omitted, the same rule the seam window already uses. + /// + [Fact] + public void ScanWindows_NoLegacyInterior_LeavesTheExistingTwoWindowsUnchanged() + { + var width = TimescaleSupport.HourlyBucket; + var floor = Hour.AddHours(10); + var horizon = Hour.AddHours(2); + var seamFloor = Hour.AddHours(4); + var ceiling = Hour.AddHours(50); + + var windows = TimescaleSupport.MaterializationHoleScanWindows(floor, ceiling, horizon, seamFloor, width); + Assert.Equal(new[] { (seamFloor, floor.AddHours(-1)), (floor, ceiling) }, windows); + + var invertedInteriorWindows = TimescaleSupport.MaterializationHoleScanWindows( + floor, ceiling, horizon, seamFloor, width, legacyInteriorFrom: Hour.AddHours(5), legacyInteriorTo: Hour.AddHours(1)); + Assert.Equal(windows, invertedInteriorWindows); + } + /// /// The start path: launched (not awaited) right after the ensure, on its own connection, inside the /// TimescaleDB block, before the compression and retention ensures; drained at shutdown beside the baseline diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs index 116c40dd8d..935de692f2 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs @@ -404,9 +404,29 @@ public static (IReadOnlyList<(DateTime Start, DateTime End)> Repair, IReadOnlyLi /// below the horizon it reaches, while the ordinary window stays exactly [max(floor, horizon), ceiling] /// — the successor's own span below the horizon is the source retention's business, not this repair's, and /// widening it was never the fix. + /// + /// A third window, INSIDE the frozen legacy's own span (#4301, H2). The seam window only + /// reaches down to the legacy's last bucket — it repairs the outage AFTER the freeze. #4186 left the + /// legacy's OWN materialized span (below its last bucket) unchecked entirely: an outage BEFORE this store + /// ever took the freeze can leave a hole inside history the legacy claims to hold, and nothing ever + /// re-scans it because the six frozen views are excluded from + /// (repairing a frozen view is the one thing the freeze forbids — see that member's own note). This window, + /// through (typically + /// [time_bucket(source's oldest admitted row), legacy.max(bucket)]), is scanned against BOTH the + /// legacy's and the successor's own materializations — a bucket either one already holds is not a hole — + /// and its caller must never refresh the legacy for what it finds, only the successor + /// ('s legacy-interior branch). It is OPTIONAL (both bounds + /// null, or an empty/inverted span) for every relation without a frozen legacy, and for one whose raw + /// floor no longer reaches back into the legacy's span at all — the ordinary case once the store has run a + /// while. Emitted FIRST (oldest), ahead of the seam: repairing it can never move the successor's own floor + /// ('s stitch only reads s.mn, never a legacy-interior bucket), so + /// it is capped and walked oldest-first exactly like the ordinary window — see + /// for why it shares that pool rather than the seam's + /// newest-first one. /// public static IReadOnlyList<(DateTime From, DateTime To)> MaterializationHoleScanWindows( - DateTime floor, DateTime ceiling, DateTime horizon, DateTime seamFloor, TimeSpan bucketWidth) + DateTime floor, DateTime ceiling, DateTime horizon, DateTime seamFloor, TimeSpan bucketWidth, + DateTime? legacyInteriorFrom = null, DateTime? legacyInteriorTo = null) { if (bucketWidth <= TimeSpan.Zero) { @@ -415,6 +435,11 @@ public static (IReadOnlyList<(DateTime Start, DateTime End)> Repair, IReadOnlyLi var windows = new List<(DateTime From, DateTime To)>(); + if (legacyInteriorFrom is { } interiorFrom && legacyInteriorTo is { } interiorTo && interiorFrom <= interiorTo) + { + windows.Add((interiorFrom, interiorTo)); + } + if (seamFloor < floor) { var seamTo = floor - bucketWidth; From 3aa2d1dab80a23cc4295ea351ac7354398581bd7 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:31:19 -0400 Subject: [PATCH 2/9] Shared legacy/successor hole-definition SQL helper for #4301 (WIP) Adds LegacySuccessorHoleExistsSql beside MaterializationHoleScanSql: one OFFSET-0-fenced EXISTS expression a caller can drop into any query, true when raw admits a row in a bucket that neither the legacy nor the successor has materialized. Intended to be shared byte-identical by RetentionArmSafetySql (the gate) and the repair walk's interior/seam branch so the two can never disagree about what a hole is. WIP: nothing calls this yet. RetentionArmSafetySql is NOT yet rewired to be bucket-level over the legacy's interior and the seam (the brief's actual ask) and no pins exist. See lane-4301-1c.md for the handoff. --- .../TimescaleSupport.MaterializationHoles.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs index 935de692f2..707cd48c1a 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs @@ -286,6 +286,50 @@ SELECT 1 FROM collect.{target.Source} AS s ORDER BY c.bucket"; } + /// + /// ONE hole definition for a frozen-legacy/successor pair (#4301), shared TEXT-IDENTICAL by + /// (the gate) and — the next lane — the repair walk's + /// interior/seam branch: a bucket in [, ] is a + /// hole when raw admits at least one row in [bucket, bucket + width) AND neither the legacy nor the + /// successor has materialized that bucket. The gate and the walk must never disagree about what a hole is — + /// a bucket the gate calls Short that the walk never repairs holds the raw purge forever with no + /// self-release. / are SQL expressions (a literal, a + /// parameter placeholder, a correlated subquery) so each caller supplies its own bounds in its own idiom. + /// OFFSET 0 fenced for the same #3933 reason is: written bare, the + /// planner pulls the per-bucket EXISTS probes up into joins that scan the whole relation instead of + /// probing one bucket's worth. A bucket below raw's own current floor can hold no admitted row, so it can + /// never be a hole under this definition and never holds the purge — a gap left below the floor by an + /// earlier version's purge is invisible here by construction, not merely undetected (see this member's own + /// callers for what that means for the gate). + /// + public static string LegacySuccessorHoleExistsSql( + string relation, string sourceTimeColumn, string sourceFilter, string legacy, string successor, + string fromExpr, string toExpr, string bucketWidthLiteral) + { + ArgumentNullException.ThrowIfNull(relation); + ArgumentNullException.ThrowIfNull(sourceTimeColumn); + ArgumentNullException.ThrowIfNull(sourceFilter); + ArgumentNullException.ThrowIfNull(legacy); + ArgumentNullException.ThrowIfNull(successor); + ArgumentNullException.ThrowIfNull(fromExpr); + ArgumentNullException.ThrowIfNull(toExpr); + ArgumentNullException.ThrowIfNull(bucketWidthLiteral); + + var filterClause = sourceFilter.Length == 0 ? string.Empty : $"\n AND {sourceFilter}"; + + return $@"EXISTS ( + SELECT 1 + FROM generate_series({fromExpr}, {toExpr}, {bucketWidthLiteral}) AS hb(bucket) + WHERE NOT EXISTS (SELECT 1 FROM collect.{legacy} AS hl WHERE hl.bucket = hb.bucket OFFSET 0) + AND NOT EXISTS (SELECT 1 FROM collect.{successor} AS hs WHERE hs.bucket = hb.bucket OFFSET 0) + AND EXISTS ( + SELECT 1 FROM collect.{relation} AS hr + WHERE hr.{sourceTimeColumn} >= hb.bucket + AND hr.{sourceTimeColumn} < hb.bucket + {bucketWidthLiteral}{filterClause} + OFFSET 0) + OFFSET 0)"; + } + /// The materialized span of one aggregate — its oldest and newest bucket — read off the /// materialization hypertable, both index-endpoint lookups. NULLs for an aggregate that has never /// materialized, which is the backfill's case and not this pass's. From 82734c06e676bfe07736485907f60edf954f4036 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:47:10 -0400 Subject: [PATCH 3/9] Wire the shared legacy/successor hole gate into RetentionArmSafetySql (#4301) Replaces the row-level seam-only probe in RetentionArmSafetySql's legacy-stitch branch with the shared bucket-level hole definition (LegacySuccessorHoleExistsSql, added on this branch's prior commit) so the gate and the repair walk share one hole definition text- identical. The probe now also reaches the legacy's own interior, not only the seam above it, and is bounded on the successor's first bucket above the legacy's last (not min(bucket)), so an interior repair moving the successor's floor down cannot make the probe miss the seam. Updates the doc comments on RetentionArmSafetySql to describe the new bucket-level probe and its bound, and updates/adds unit pins in TimescaleContinuousAggregateTests.cs for the new SQL shape and for filter parity between the gate and the walk. Does not touch the repair walk's own interior/seam branch (next lane) or the non-legacy coverage path. --- .../TimescaleContinuousAggregateTests.cs | 97 +++++++++--- .../TimescaleSupport.cs | 149 ++++++++++-------- 2 files changed, 160 insertions(+), 86 deletions(-) diff --git a/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs b/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs index dfd15b7ff2..8d55aafb4c 100644 --- a/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs +++ b/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs @@ -1255,56 +1255,107 @@ public void RetentionArmSafetySql_NamesEveryCoverageRelation() } /// - /// WATCHED (mutation, #4186): a stitched slot (a coverage relation with a frozen legacy, found through - /// LegacyOf) must probe raw for a seam row before it falls back to the legacy's floor — an - /// UNCONDITIONAL stitch is the #4186 data-loss defect: a raw tail between the legacy's last bucket and - /// the successor's floor, never materialized by either side, read Covered and the purge dropped it. A - /// non-stitched slot (no LegacyOf match) must stay the plain min(bucket) form untouched. + /// WATCHED (mutation, #4186 then #4301): a stitched slot (a coverage relation with a frozen legacy, + /// found through LegacyOf) must probe raw BUCKET-LEVEL, over the legacy's own interior AND the + /// seam, before it falls back to the legacy's floor — an UNCONDITIONAL stitch is the #4186 data-loss + /// defect: a raw tail between the legacy's last bucket and the successor's floor, never materialized by + /// either side, read Covered and the purge dropped it. #4301 replaced the #4186 row-level seam-only probe + /// with the shared hole definition, bounded + /// on the successor's first bucket ABOVE the legacy's last (not s.mn, which an interior repair can + /// move below the seam). A non-stitched slot (no LegacyOf match) must stay the plain + /// min(bucket) form untouched. /// [Fact] - public void RetentionArmSafetySql_StitchedSlot_ProbesSeamBeforeFallingBackToLegacyFloor() + public void RetentionArmSafetySql_StitchedSlot_ProbesLegacySuccessorHoleBeforeFallingBackToLegacyFloor() { var sql = TimescaleSupport.RetentionArmSafetySql( "query_stats", "collection_time", new[] { TimescaleSupport.QueryStatsIntervalHourlyView }); - /* The seam probe: raw, bounded between the legacy's last bucket (+1h, so the legacy's own last - bucket is not re-counted) and the successor's floor (or infinity when it is empty). */ - Assert.Contains("EXISTS (", sql, StringComparison.Ordinal); - Assert.Contains("FROM collect.query_stats AS seam", sql, StringComparison.Ordinal); - Assert.Contains("l.mx + INTERVAL '1 hour'", sql, StringComparison.Ordinal); - Assert.Contains("COALESCE(s.mn, 'infinity'::timestamp)", sql, StringComparison.Ordinal); - - /* The filter must appear TWICE: once for source_oldest (every relation with a filter gets that - already) and once more inside the seam probe itself — a seam probe with no filter would let a - post-restart interval-0 row hold the gate open forever, exactly like source_oldest without it. */ + /* The shared hole definition: generate_series fenced with OFFSET 0, over raw/legacy/successor, + bounded on the successor's first bucket ABOVE l.mx — never s.mn, which an interior repair moves + down and would otherwise let the probe miss the seam once such a repair has run. */ + Assert.Contains("generate_series(", sql, StringComparison.Ordinal); + Assert.Contains("sa.bucket > l.mx", sql, StringComparison.Ordinal); + Assert.DoesNotContain("COALESCE(s.mn, 'infinity'", sql, StringComparison.Ordinal); + Assert.Contains($"FROM collect.{TimescaleSupport.QueryStatsHourlyView} AS hl", sql, StringComparison.Ordinal); + Assert.Contains($"FROM collect.{TimescaleSupport.QueryStatsIntervalHourlyView} AS hs", sql, StringComparison.Ordinal); + Assert.Contains("FROM collect.query_stats AS hr", sql, StringComparison.Ordinal); + + /* l.mx IS NULL -> today's plain LEAST, unconditionally, for a legacy that never materialized. */ + Assert.Contains("WHEN l.mx IS NULL THEN LEAST(l.mn, s.mn)", sql, StringComparison.Ordinal); + + /* The filter must appear at least TWICE: once for source_oldest and once more inside the hole + probe's raw-row EXISTS — a probe with no filter would let a post-restart interval-0 row read as + a hole that can never repair, holding the gate open forever. */ var filterOccurrences = sql.Split(new[] { TimescaleSupport.IntervalHonestSourceFilter }, StringSplitOptions.None).Length - 1; Assert.True(filterOccurrences >= 2, - $"expected the seam probe to carry its own {nameof(TimescaleSupport.IntervalHonestSourceFilter)} in addition to source_oldest's, found {filterOccurrences} occurrence(s) in: {sql}"); + $"expected the hole probe to carry its own {nameof(TimescaleSupport.IntervalHonestSourceFilter)} in addition to source_oldest's, found {filterOccurrences} occurrence(s) in: {sql}"); - /* Seam empty -> the stitched floor. PostgreSQL's LEAST already ignores NULLs, so the old + /* No hole found -> the stitched floor. PostgreSQL's LEAST already ignores NULLs, so the old COALESCE(LEAST(l.mn, s.mn), l.mn, s.mn) wrapper was redundant; it must not come back. */ Assert.Contains("LEAST(l.mn, s.mn)", sql, StringComparison.Ordinal); Assert.DoesNotContain("COALESCE(LEAST(", sql, StringComparison.Ordinal); - /* Seam NOT empty -> the successor's own floor alone, never the legacy's — the legacy cannot vouch - for raw history it never covered. */ - Assert.Contains("THEN s.mn", sql, StringComparison.Ordinal); + /* A detectable hole -> NULL, which MeasureRetentionCoverageAsync reads as Short — never s.mn, which + an interior repair can move below the seam and read back Covered. */ + Assert.Contains("THEN NULL", sql, StringComparison.Ordinal); + Assert.DoesNotContain("THEN s.mn", sql, StringComparison.Ordinal); Assert.Contains($"FROM collect.{TimescaleSupport.QueryStatsHourlyView}", sql, StringComparison.Ordinal); Assert.Contains($"FROM collect.{TimescaleSupport.QueryStatsIntervalHourlyView}", sql, StringComparison.Ordinal); /* A non-stitched slot (query_store_stats' two consumers have no LegacyOf match) stays the plain - form — none of the seam machinery leaks into a slot that never needed it. */ + form — none of the hole-probe machinery leaks into a slot that never needed it. */ var plainSql = TimescaleSupport.RetentionArmSafetySql( "query_store_stats", "collection_time", new[] { TimescaleSupport.QueryStoreStatsHourlyView, TimescaleSupport.QueryStoreStatsIntervalHourlyView }); + Assert.DoesNotContain("generate_series(", plainSql, StringComparison.Ordinal); Assert.DoesNotContain("EXISTS (", plainSql, StringComparison.Ordinal); - Assert.DoesNotContain("seam", plainSql, StringComparison.Ordinal); Assert.Contains($"(SELECT min(bucket) FROM collect.{TimescaleSupport.QueryStoreStatsHourlyView}) AS coverage_oldest_0", plainSql, StringComparison.Ordinal); Assert.Contains($"(SELECT min(bucket) FROM collect.{TimescaleSupport.QueryStoreStatsIntervalHourlyView}) AS coverage_oldest_1", plainSql, StringComparison.Ordinal); } + /// + /// PIN (#4301, RED on e970834ba and earlier): the generated coverage SQL for a stitched slot + /// carries the shared hole definition's generate_series AND its bound on the successor's first + /// bucket above the legacy's last (sa.bucket > l.mx), and no longer carries the old row-level + /// seam-only probe (COALESCE(s.mn, 'infinity'). The old text existed only through e970834ba; + /// on that commit this pin fails the generate_series/sa.bucket > l.mx assertions because + /// the old branch has neither — it reads s.mn unconditionally on any seam row instead. + /// + [Fact] + public void RetentionArmSafetySql_QueryStatsIntervalHourlyCoverage_UsesSharedHoleDefinition() + { + var sql = TimescaleSupport.RetentionArmSafetySql( + "query_stats", "collection_time", new[] { TimescaleSupport.QueryStatsIntervalHourlyView }); + + Assert.Contains("generate_series(", sql, StringComparison.Ordinal); + Assert.Contains("sa.bucket > l.mx", sql, StringComparison.Ordinal); + Assert.DoesNotContain("COALESCE(s.mn, 'infinity'", sql, StringComparison.Ordinal); + } + + /// + /// PIN (#4301, filter parity): the gate's hole probe (, + /// via ) and the repair walk + /// ( over the successor's own CREATE) + /// must read the SAME source filter for every + /// successor — the gate and the walk disagreeing about which raw rows count would let the gate call a + /// bucket Short (or Covered) that the walk judges by different rules, breaking the "never disagree about + /// what a hole is" invariant 's own doc states. + /// + [Fact] + public void LegacySuccessorHoleProbe_UsesSameFilterAsTheRepairWalk_ForEverySupersededSuccessor() + { + foreach (var (_, successor, _) in TimescaleSupport.SupersededHourlyRollups) + { + var successorCreateSql = TimescaleSupport.HourlyAggregates.Single(a => a.View == successor).CreateSql; + var walkFilter = TimescaleSupport.MaterializationHoleSourceFilterFor(successorCreateSql); + + Assert.Equal(TimescaleSupport.IntervalHonestSourceFilter, walkFilter); + } + } + /// /// The map both purge paths read (#1784) must name BOTH Query Store rollup families as raw's coverage. /// query_store_stats is the only raw table with two consumers; naming just one would let raw purge over diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index bc2b80e564..c600f54ee3 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -6119,44 +6119,57 @@ FROM timescaledb_information.jobs AS j /// the successor's first refresh leaves raw rows between the legacy's last bucket and the successor's /// floor that NEITHER side ever materializes (the outage shape MaterializationHoles.cs's class doc /// works through in full). An unconditional stitch reports that seam Covered right up to the moment the - /// raw purge destroys it. So the SQL probes raw itself, filtered the same way - /// filters everything else here, for a row at or after the legacy's last bucket and before the successor's - /// floor (or +infinity when the successor is empty): none found → LEAST(legacy.min, successor.min), - /// the stitched floor, honest because the seam really is empty; a row found → the successor's OWN - /// min(bucket) alone (NULL when empty, giving Short), because the legacy cannot vouch for - /// history it never covered. Once the successor's own floor reaches back past the legacy's last bucket, - /// there is no seam left to hold a row, the probe always comes back empty, and the two branches converge — - /// the same steady state the old unconditional stitch reached, just no longer assumed along the way. Both - /// empty → NULL → Short (correct: fresh install, no history yet). The seam is what - /// repairs, by giving a stitched successor a SEAM scan window - /// down to the legacy's last bucket, unclamped by the horizon that bounds its ordinary scan window — so an - /// outage longer than that horizon's own span (#4186 follow-up: the first cut folded the seam into the same - /// horizon clamp as the ordinary window, and a seam older than it was silently never scanned) still gets - /// repaired instead of clamped away — PROVIDED the raw purge was already held when the store stopped. - /// That proviso is real, not decoration: PostgreSQL runs its own overdue retention jobs at start, before - /// this service's start sweep can hold anything, so a purge left armed across a long enough stop drops the - /// chunk holding the seam before the walk ever gets a turn (#4299, pre-existing — 3.8.0 loses the same - /// chunk on the same schedule). Once the repair does run, the seam is empty and this gate's fallback - /// releases — automatically within the hour on a store that keeps running (#3812), but only from the - /// SECOND start after an outage that crosses the upgrade: the walk skips a successor with nothing - /// materialized yet, so the seam does not exist for it to find until the successor's own first refresh has - /// run, and nothing re-runs the walk between starts (#4300). Once it does run, release is bounded only by - /// the repair's own per-start cap — a seam wider than one start's cap takes more than one start to close in - /// full, but every start makes progress on it. - /// - /// The stitch also trusts the frozen legacy's OWN span unconditionally — accepted as 3.8.0 parity, - /// not fixed by this round. The seam probe above only checks AT OR AFTER the legacy's last bucket. - /// Below that bucket, LEAST(legacy.min, successor.min) runs with no check at all: a raw-row gap that - /// opened INSIDE the frozen legacy's own materialized span — from an outage before this store ever took the - /// freeze, that 3.8.0's own retention already lost before the upgrade — reads Covered forever, because the - /// legacy stopped refreshing at the freeze and the six frozen views never re-enter the walk to close it - /// ( excludes them; see its note). This is accepted, not a - /// regression: 3.8.0 has no walk at all, so it loses the exact same rows on the exact same schedule — its - /// raw purge drops them once they age past the horizon, upgrade or not. What this build adds beyond that - /// parity is the A6 backfill (--backfill-rollups, ), which — unlike the - /// automatic walk — fills the successor down to RAW's own oldest row, not merely the legacy's boundary, so - /// an operator who runs it closes a pre-upgrade hole the automatic gate will otherwise trust without ever - /// looking. + /// raw purge destroys it. So the SQL probes BUCKET-LEVEL, over the legacy's own interior AND the seam + /// together (#4301, sharing TEXT-IDENTICAL with the repair + /// walk — the gate and the walk must never disagree about what a hole is): from raw's own filtered floor + /// (below it raw admits no row, so no hole can exist there — a gap an EARLIER version's purge left below + /// that floor is invisible here BY CONSTRUCTION, not merely undetected) up to the successor's first + /// bucket strictly ABOVE the legacy's last bucket, minus one bucket width — bounded there and not by the + /// successor's min(bucket), because an INTERIOR repair (the next lane's walk branch) materializes + /// successor buckets AT OR BELOW the legacy's last bucket, which moves the successor's own floor down; a + /// probe bounded on that floor would then miss the seam once even one such repair had run, while bounding + /// on the successor's first bucket above the legacy's last keeps the seam fully probed regardless. Any + /// detectable hole in that range — a bucket where raw admits a row and neither the legacy nor the + /// successor has materialized it — makes the slot NULL, which MeasureRetentionCoverageAsync reads + /// as Short, rather than falling back to the successor's own floor: that floor could itself read + /// back as Covered by the same interior-repair shape the bound above exists to catch. No hole found → + /// LEAST(legacy.min, successor.min), the stitched floor, honest because the whole probed range + /// really is gap-free. Legacy never materialized (l.mx IS NULL) → today's plain + /// LEAST(legacy.min, successor.min), unchanged, because the comparison it would otherwise run is + /// against nothing. The seam and any interior hole are what + /// repairs, by giving a stitched successor a SEAM scan window down to the legacy's last bucket, unclamped + /// by the horizon that bounds its ordinary scan window — so an outage longer than that horizon's own span + /// (#4186 follow-up: the first cut folded the seam into the same horizon clamp as the ordinary window, and + /// a seam older than it was silently never scanned) still gets repaired instead of clamped away — + /// PROVIDED the raw purge was already held when the store stopped. That proviso is real, not decoration: + /// PostgreSQL runs its own overdue retention jobs at start, before this service's start sweep can hold + /// anything, so a purge left armed across a long enough stop drops the chunk holding the seam before the + /// walk ever gets a turn (#4299, pre-existing — 3.8.0 loses the same chunk on the same schedule). Once the + /// repair does run, every detectable hole is closed and this gate's fallback releases — automatically + /// within the hour on a store that keeps running (#3812), but only from the SECOND start after an outage + /// that crosses the upgrade: the walk skips a successor with nothing materialized yet, so the seam does + /// not exist for it to find until the successor's own first refresh has run, and nothing re-runs the walk + /// between starts (#4300). Once it does run, release is bounded only by the repair's own per-start cap — + /// a seam wider than one start's cap takes more than one start to close in full, but every start makes + /// progress on it. GAPS LEFT BY EARLIER VERSIONS' PURGES BELOW THE RAW FLOOR CANNOT BE DETECTED OR + /// REPAIRED; FROM THIS VERSION THE PURGE HOLDS UNTIL EVERY DETECTABLE HOLE IS REPAIRED. + /// + /// The probe now reaches INSIDE the frozen legacy's own materialized span too (#4301), not only + /// the seam above it — accepted below raw's own floor as 3.8.0 parity, not fixed by this round. The + /// probe's fromExpr starts at raw's own filtered floor, which sits at or below the legacy's last + /// bucket, so a raw-row gap that opened INSIDE the frozen legacy's own materialized span — an outage that + /// predates this store's freeze — is now caught the same way a seam gap is: the slot goes NULL (Short) + /// rather than the unconditional LEAST(legacy.min, successor.min) the six frozen views used to get + /// with no check at all. What stays accepted as 3.8.0 parity is strictly BELOW raw's own current floor: + /// a gap 3.8.0's own retention already purged before this store ever took the freeze is invisible to the + /// probe by construction (raw admits no row there to prove the hole), and the six frozen views never + /// re-enter the walk to close it ( excludes them; see its note). + /// This is accepted, not a regression: 3.8.0 has no walk at all, so it loses the exact same rows on the + /// exact same schedule — its raw purge drops them once they age past the horizon, upgrade or not. What + /// this build adds beyond that parity is the A6 backfill (--backfill-rollups, + /// ), which — unlike the automatic walk — fills the successor down to RAW's + /// own oldest row, not merely the legacy's boundary, so an operator who runs it closes a pre-upgrade hole + /// the automatic gate can only detect from raw's current floor, not resurrect from below it. /// /// Source filter for 0-interval rows. For query_stats and procedure_stats the /// source_oldest subquery adds WHERE to exclude @@ -6180,36 +6193,46 @@ filter to source_oldest. query_store_stats has no such filter in its hourly CREA /* Coverage SQL: for a coverage relation that is an interval-honest successor, stitch it with its frozen legacy so an empty successor on an upgrading store falls back to the legacy's - floor — BUT ONLY WHEN THE SEAM IS EMPTY (#4186). The legacy stopped refreshing at the - freeze; if the store was down past HourlyRefreshStartOffset before the successor's first - refresh, the raw rows collected between the legacy's last bucket and the successor's floor - were never materialized by either side (see MaterializationHoles.cs's class doc for the - shape). A stitch that ignores that seam reports Covered over history nobody holds. - So: probe raw for a filter-admitted row at or after the legacy's last bucket (+ one hour, so - the legacy's own last bucket is not re-counted as seam) and before the successor's floor (or - +infinity when the successor is empty). Any such row means the seam is NOT gap-free, so the - slot falls back to the successor's OWN min(bucket) — NULL when empty, which reports Short - honestly instead of the false Covered the unconditional stitch gave. LEGACY.mx is NULL for a - legacy that never materialized anything; the comparison against it is then UNKNOWN for every - row, EXISTS is false, and the stitch behaves exactly as it did before this seam probe existed. - LegacyOf() returns null for any relation that is not in SupersededHourlyRollups (dailies, - query_store_stats, baseline aggregates), keeping those as simple min(bucket). PostgreSQL's - LEAST ignores NULL arguments (returns NULL only when EVERY argument is NULL), so - LEAST(l.mn, s.mn) already is what COALESCE(LEAST(l.mn, s.mn), l.mn, s.mn) computed. */ + floor — BUT ONLY WHEN THE SEAM (AND THE LEGACY'S OWN INTERIOR) IS GAP-FREE (#4186, #4301). + The legacy stopped refreshing at the freeze; if the store was down past HourlyRefreshStartOffset + before the successor's first refresh, the raw rows collected between the legacy's last bucket + and the successor's floor were never materialized by either side (see MaterializationHoles.cs's + class doc for the shape). A stitch that ignores that seam reports Covered over history nobody + holds. This is now a BUCKET-LEVEL probe over the legacy's interior AND the seam, sharing its + hole definition TEXT-IDENTICAL with the repair walk via LegacySuccessorHoleExistsSql (#4301) — + the gate and the walk must never disagree about what a hole is, or a bucket the gate calls Short + that the walk never repairs holds the raw purge forever with no self-release. The scanned range + runs from raw's own filtered floor (below it raw admits no row, so no hole can exist there — a + gap left by an EARLIER version's purge below that floor is invisible here BY CONSTRUCTION, not + merely undetected) up to the successor's first bucket strictly ABOVE the legacy's last bucket + (or, when the successor holds nothing that high, the current hour) minus one bucket width. That + upper bound is deliberately NOT s.mn: an interior repair materializes successor buckets AT OR + BELOW l.mx, which moves s.mn itself down, and a probe bounded on s.mn would then miss the seam + entirely once even one such repair has run. Bounding instead on the successor's first bucket + ABOVE l.mx holds the seam '(l.mx, that bucket)' fully probed no matter how many interior repairs + ran; buckets from that point up are the successor's own span, which the walk's ordinary window + repairs, so they are left out of this gate exactly as before. WHEN THE PROBE FINDS ANY HOLE the + slot is NULL — MeasureRetentionCoverageAsync reads a NULL coverage column as Short — rather than + falling back to s.mn: s.mn could read back as Covered by the same interior-repair shape the + bound above exists to catch. LEGACY.mx is NULL for a legacy that never materialized anything; the + WHEN branch below only applies when l.mx IS NOT NULL, and in that case the stitch behaves exactly + as it did before this probe existed (today's plain LEAST). LegacyOf() returns null for any + relation that is not in SupersededHourlyRollups (dailies, query_store_stats, baseline aggregates), + keeping those as simple min(bucket). PostgreSQL's LEAST ignores NULL arguments (returns NULL only + when EVERY argument is NULL), so LEAST(l.mn, s.mn) already is what + COALESCE(LEAST(l.mn, s.mn), l.mn, s.mn) computed. */ var columns = coverageRelations.Select((c, i) => { var legacy = LegacyOf(c); var subquery = legacy is not null ? $"(SELECT CASE{Environment.NewLine}" - + $" WHEN EXISTS ({Environment.NewLine}" - + $" SELECT 1{Environment.NewLine}" - + $" FROM collect.{relation} AS seam{Environment.NewLine}" - + $" WHERE seam.{sourceTimeColumn} >= l.mx + INTERVAL '1 hour'{Environment.NewLine}" - + $" AND seam.{sourceTimeColumn} < COALESCE(s.mn, 'infinity'::timestamp){Environment.NewLine}" - + $" AND {IntervalHonestSourceFilter}{Environment.NewLine}" - + $" ORDER BY seam.{sourceTimeColumn}{Environment.NewLine}" - + $" LIMIT 1){Environment.NewLine}" - + $" THEN s.mn{Environment.NewLine}" + + $" WHEN l.mx IS NULL THEN LEAST(l.mn, s.mn){Environment.NewLine}" + + $" WHEN {LegacySuccessorHoleExistsSql( + relation, sourceTimeColumn, IntervalHonestSourceFilter, legacy, c, + fromExpr: $"time_bucket(INTERVAL '1 hour', (SELECT min(src.{sourceTimeColumn}) FROM collect.{relation} AS src WHERE {IntervalHonestSourceFilter}))", + toExpr: $"COALESCE((SELECT min(sa.bucket) FROM collect.{c} AS sa WHERE sa.bucket > l.mx), time_bucket(INTERVAL '1 hour', now()::timestamp)) - INTERVAL '1 hour'", + bucketWidthLiteral: "INTERVAL '1 hour'")}{Environment.NewLine}" + + $" THEN NULL{Environment.NewLine}" + $" ELSE LEAST(l.mn, s.mn){Environment.NewLine}" + $" END{Environment.NewLine}" + $" FROM (SELECT min(bucket) AS mn, max(bucket) AS mx FROM collect.{legacy}) l{Environment.NewLine}" From ffa3a6990ecff1cf40bb9b11aef00b3ad35cccaa Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 02:52:46 -0400 Subject: [PATCH 4/9] Live pins A-D for the legacy/successor hole gate (#4301) Four live tests in FrozenRollupLiveTests.cs against IsRawTierDropSafeAsync, modelled on the existing Outage_Seam* tests' scaffolding: - InteriorHole_BelowLegacysLastBucket_ReportsRawPurgeNotSafe (A): a gap inside the legacy's own span reports Short. RED on e970834ba (the old probe never looks inside the legacy's span). - InteriorRepairMovesSuccessorFloorBelowSeam_TrapDoesNotHideTheHole_ ReportsRawPurgeNotSafe (B): a successor bucket below the legacy's last bucket (as an interior repair leaves it) must not let a seam hole above it hide. RED on e970834ba (the old s.mn-bounded probe collapses to an empty range and misses the seam). - NoGapAnywhere_ReportsRawPurgeSafe (C): passes on both. - GapBelowRawsFilteredFloor_IsInvisibleToTheProbe_ReportsRawPurgeSafe (D): passes on both. All four GREEN on this branch; A and B verified RED on e970834ba by building and running the same file in a detached worktree of that commit. --- .../Darling.Tests/FrozenRollupLiveTests.cs | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) diff --git a/Darling/Darling.Tests/FrozenRollupLiveTests.cs b/Darling/Darling.Tests/FrozenRollupLiveTests.cs index 4c0471017d..27b527b562 100644 --- a/Darling/Darling.Tests/FrozenRollupLiveTests.cs +++ b/Darling/Darling.Tests/FrozenRollupLiveTests.cs @@ -980,6 +980,296 @@ await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async ( } } + /// + /// #4301, PIN A (interior hole -> false): the legacy's OWN interior has a gap (hour H2 refreshed by + /// neither side, despite raw admitting a row there) below its last bucket (H5), and the successor holds a + /// bucket ABOVE the legacy's last (H6, its own ordinary advance). + /// must read false: the gap is a real hole under 's + /// definition (raw admits a row, neither side materialized it), so the slot must go Short, not Covered. + /// RED on e970834ba: the old row-level seam-only probe never looks INSIDE the legacy's own span (it + /// only checks at/after l.mx), so it reports Covered here. + /// + [Fact] + public async Task InteriorHole_BelowLegacysLastBucket_ReportsRawPurgeNotSafe() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(20); + + /* Raw admits rows at H0, H2 (the interior hole) and H5 (legacy's last) and H6 (successor's + own floor above l.mx). H1, H3, H4 hold no raw rows at all, so their absence from either side + is never a hole. */ + await InsertProcedureStatsAsync(connection, b, "pinA_h0", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(2), "pinA_h2", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(5), "pinA_h5", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(6), "pinA_h6", 900, 9, 3600, ct); + + /* Legacy materializes H0 and H5 only -- H2 is skipped, the interior hole. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b, b.AddHours(1), ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b.AddHours(5), b.AddHours(6), ct); + + /* Successor materializes H6 only -- its own ordinary advance above l.mx, no interior repair. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b.AddHours(6), b.AddHours(7), ct); + + Assert.False(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + + /// + /// #4301, PIN B (the trap -> false): the successor holds ONE bucket BELOW the legacy's last (H3, as an + /// INTERIOR repair leaves it), which moves s.mn below l.mx, AND a seam hour above the + /// legacy's last (H6) has a raw row and no bucket anywhere. A probe bounded on s.mn (the old shape) + /// would see the seam range collapse to (l.mx, s.mn) = (H5, H3), an EMPTY/inverted range, and + /// miss H6 entirely -- exactly the trap this lane's bound (the successor's first bucket ABOVE l.mx, + /// H7) exists to avoid. must read false. + /// RED on e970834ba: the old code's seam probe is bounded on COALESCE(s.mn, 'infinity'), + /// which here is H3, before l.mx + 1h = H6 -- the probe range is empty, no seam row is found, and + /// the old code falls through to LEAST(l.mn, s.mn), reporting Covered. + /// + [Fact] + public async Task InteriorRepairMovesSuccessorFloorBelowSeam_TrapDoesNotHideTheHole_ReportsRawPurgeNotSafe() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(30); + + /* Raw admits rows at H0 (raw's floor), H3 (successor's interior repair), H5 (legacy's last), + H6 (the seam hole -- no bucket anywhere) and H7 (successor's own ordinary advance above + l.mx, needed so the probe's upper bound resolves to H6 and not the current hour). */ + await InsertProcedureStatsAsync(connection, b, "pinB_h0", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(3), "pinB_h3", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(5), "pinB_h5", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(6), "pinB_h6", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(7), "pinB_h7", 900, 9, 3600, ct); + + /* Legacy materializes H0 and H5 only -- l.mx ends at H5. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b, b.AddHours(1), ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b.AddHours(5), b.AddHours(6), ct); + + /* Successor materializes H3 (an interior repair BELOW l.mx) and H7 (its own ordinary advance + ABOVE l.mx) -- s.mn is H3, below l.mx, the exact shape that moves the successor's floor + under the seam. H6 (the seam hour) is left uncovered by both sides. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b.AddHours(3), b.AddHours(4), ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b.AddHours(7), b.AddHours(8), ct); + + Assert.False(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + + /// + /// #4301, PIN C (clean -> true): the legacy covers every raw-admitted bucket up to its own last bucket + /// (H5), and the successor's only bucket (H6) sits immediately above it, with no gap anywhere in between. + /// must read true. Passes on both this branch + /// and e970834ba (both find no hole here); included to show the new probe does not fire on a + /// gap-free stitch. + /// + [Fact] + public async Task NoGapAnywhere_ReportsRawPurgeSafe() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(40); + + await InsertProcedureStatsAsync(connection, b, "pinC_h0", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(5), "pinC_h5", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(6), "pinC_h6", 900, 9, 3600, ct); + + /* Legacy materializes the whole H0-H5 span, no gap. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b, b.AddHours(6), ct); + + /* Successor materializes H6, its own ordinary advance immediately above l.mx. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b.AddHours(6), b.AddHours(7), ct); + + Assert.True(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + + /// + /// #4301, PIN D (below the floor -> true): the legacy holds no bucket for H0, OLDER than raw's own + /// filtered floor (H0 carries a first-pass, sample_interval_seconds = 0 row that + /// excludes from admission, so it never sets + /// fromExpr). Everything from raw's real admitted floor (H2) up to the legacy's last bucket (H5) is + /// covered. must read true: a gap below the + /// probed floor is invisible by construction, exactly as this lane's doc comment states. + /// + [Fact] + public async Task GapBelowRawsFilteredFloor_IsInvisibleToTheProbe_ReportsRawPurgeSafe() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(50); + + /* H0: a first-pass restart row (interval 0), excluded from admission -- never sets fromExpr, + and the legacy is never refreshed for it (a pre-freeze purge could have taken this bucket + and it would look identical). H2: raw's real admitted floor. H5: legacy's last bucket. + H6: successor's own ordinary advance, bounding the probe's upper edge. */ + await InsertProcedureStatsAsync(connection, b, "pinD_h0_restart", 0, 0, 0, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(2), "pinD_h2", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(5), "pinD_h5", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(6), "pinD_h6", 900, 9, 3600, ct); + + /* Legacy materializes H2-H5 only -- H0 is never touched, and never needs to be. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b.AddHours(2), b.AddHours(6), ct); + + /* Successor materializes H6, its own ordinary advance immediately above l.mx. */ + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b.AddHours(6), b.AddHours(7), ct); + + Assert.True(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + private static async Task InsertQueryStatsAsync( NpgsqlConnection connection, DateTime at, string hash, long workerUs, long executions, int intervalSeconds, CancellationToken ct) { From 380741ebd6a1aa835b7c07d4f04c930be1e9dd32 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:06:46 -0400 Subject: [PATCH 5/9] #4301: gate's hole probe uses each stitched successor's own filter, not the bare constant RetentionArmSafetySql's LegacySuccessorHoleExistsSql call now passes MaterializationHoleSourceFilterFor(successor's own CREATE) instead of the bare IntervalHonestSourceFilter constant -- the same filter the repair walk reads off the same successor's CREATE. query_stats_db_interval_hourly's own filter (delta_worker_time IS NOT NULL AND sample_interval_seconds IS DISTINCT FROM 0) is strictly wider than the bare constant, so the two disagreed about which raw rows count for that successor. Also applies the per-successor filter to that slot's fromExpr floor (the successor's own filtered raw floor), matching the walk's own floor computation. source_oldest's own sourceWhere is left as the pre-existing, conservative bare-constant shape -- untouched per brief. Rewrote LegacySuccessorHoleProbe_UsesSameFilterAsTheRepairWalk_ForEverySupersededSuccessor to exercise the actual generated gate SQL (RetentionArmSafetySql) rather than comparing two constants -- the prior form could never fail regardless of what the gate's code did, since it never called the gate. Pin now passes. --- .../TimescaleContinuousAggregateTests.cs | 28 +++++++++++++------ .../TimescaleSupport.cs | 8 ++++-- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs b/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs index 8d55aafb4c..9bcee8223b 100644 --- a/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs +++ b/Darling/Darling.Tests/TimescaleContinuousAggregateTests.cs @@ -1336,13 +1336,21 @@ public void RetentionArmSafetySql_QueryStatsIntervalHourlyCoverage_UsesSharedHol } /// - /// PIN (#4301, filter parity): the gate's hole probe (, - /// via ) and the repair walk - /// ( over the successor's own CREATE) - /// must read the SAME source filter for every - /// successor — the gate and the walk disagreeing about which raw rows count would let the gate call a - /// bucket Short (or Covered) that the walk judges by different rules, breaking the "never disagree about - /// what a hole is" invariant 's own doc states. + /// PIN (#4301, filter parity — RED before the fix, because the gate passed the bare + /// constant to every stitched slot's hole probe + /// regardless of the successor's own CREATE). The gate's hole probe + /// (, via + /// ) must carry the SAME source filter the repair + /// walk reads off the successor's own CREATE () + /// for every successor — the gate and the walk + /// disagreeing about which raw rows count would let the gate call a bucket Short (or Covered) that the walk + /// judges by different rules, breaking the "never disagree about what a hole is" invariant + /// 's own doc states. Exercises the ACTUAL + /// generated SQL rather than comparing two constants, so a regression that restores the bare constant fails + /// this pin even when itself is untouched — the + /// case that matters for , whose own filter + /// (delta_worker_time IS NOT NULL AND sample_interval_seconds IS DISTINCT FROM 0) is strictly wider + /// than the bare constant. /// [Fact] public void LegacySuccessorHoleProbe_UsesSameFilterAsTheRepairWalk_ForEverySupersededSuccessor() @@ -1351,8 +1359,12 @@ public void LegacySuccessorHoleProbe_UsesSameFilterAsTheRepairWalk_ForEverySuper { var successorCreateSql = TimescaleSupport.HourlyAggregates.Single(a => a.View == successor).CreateSql; var walkFilter = TimescaleSupport.MaterializationHoleSourceFilterFor(successorCreateSql); + Assert.False(string.IsNullOrEmpty(walkFilter), $"{successor}'s CREATE has no WHERE for the walk to read a filter from."); - Assert.Equal(TimescaleSupport.IntervalHonestSourceFilter, walkFilter); + var coverageEntry = TimescaleSupport.RawTierCoverage.Single(t => t.Coverage.Contains(successor)); + var sql = TimescaleSupport.RetentionArmSafetySql(coverageEntry.Relation, coverageEntry.TimeColumn, new[] { successor }); + + Assert.Contains(walkFilter, sql, StringComparison.Ordinal); } } diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index c600f54ee3..2073a2a771 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -6224,12 +6224,16 @@ keeping those as simple min(bucket). PostgreSQL's LEAST ignores NULL arguments ( var columns = coverageRelations.Select((c, i) => { var legacy = LegacyOf(c); + var successorFilter = legacy is not null + ? MaterializationHoleSourceFilterFor(HourlyAggregates.Single(a => a.View == c).CreateSql) + : string.Empty; + var successorFloorWhere = successorFilter.Length == 0 ? string.Empty : $" WHERE {successorFilter}"; var subquery = legacy is not null ? $"(SELECT CASE{Environment.NewLine}" + $" WHEN l.mx IS NULL THEN LEAST(l.mn, s.mn){Environment.NewLine}" + $" WHEN {LegacySuccessorHoleExistsSql( - relation, sourceTimeColumn, IntervalHonestSourceFilter, legacy, c, - fromExpr: $"time_bucket(INTERVAL '1 hour', (SELECT min(src.{sourceTimeColumn}) FROM collect.{relation} AS src WHERE {IntervalHonestSourceFilter}))", + relation, sourceTimeColumn, successorFilter, legacy, c, + fromExpr: $"time_bucket(INTERVAL '1 hour', (SELECT min(src.{sourceTimeColumn}) FROM collect.{relation} AS src{successorFloorWhere}))", toExpr: $"COALESCE((SELECT min(sa.bucket) FROM collect.{c} AS sa WHERE sa.bucket > l.mx), time_bucket(INTERVAL '1 hour', now()::timestamp)) - INTERVAL '1 hour'", bucketWidthLiteral: "INTERVAL '1 hour'")}{Environment.NewLine}" + $" THEN NULL{Environment.NewLine}" From abc8ce3c4e62c305106e4998cc263dde101fd560 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:09:33 -0400 Subject: [PATCH 6/9] #4301 H2: one shared hole-definition body, two wrappers (EXISTS and LIST) Refactored LegacySuccessorHoleExistsSql into a thin EXISTS(...) wrapper around a new private LegacySuccessorHoleBodySql, and added LegacySuccessorHoleScanSql -- the LIST form the repair walk needs (SELECT hb.bucket ... ORDER BY hb.bucket), wrapping the SAME body. TEXT-IDENTICAL body, so the gate's EXISTS probe and the walk's bucket list can never disagree about what a hole is. Added a unit pin (LegacySuccessorHoleExistsSql_AndLegacySuccessorHoleScanSql_ShareTheIdenticalBody) asserting both wrappers' generated SQL contains the identical buckets clause verbatim. --- .../MaterializationHoleRepairTests.cs | 42 +++++++++++++++++++ .../TimescaleSupport.MaterializationHoles.cs | 33 +++++++++++++-- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs index ea112fed4c..45c9de5b40 100644 --- a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs +++ b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs @@ -397,6 +397,48 @@ public void ScanWindows_NoLegacyInterior_LeavesTheExistingTwoWindowsUnchanged() Assert.Equal(windows, invertedInteriorWindows); } + /// + /// #4301 (H2, filter parity companion): (the + /// gate's EXISTS wrapper) and (the walk's + /// bucket LIST) must share the IDENTICAL body text — one private builder, two wrappers — so the gate and + /// the walk can never drift into disagreeing about what a hole is. Asserts the shared + /// generate_series(...) ... OFFSET 0 buckets clause appears verbatim inside both generated strings. + /// + [Fact] + public void LegacySuccessorHoleExistsSql_AndLegacySuccessorHoleScanSql_ShareTheIdenticalBody() + { + const string relation = "query_stats"; + const string sourceTimeColumn = "collection_time"; + const string sourceFilter = "sample_interval_seconds IS DISTINCT FROM 0"; + const string legacy = "query_stats_hourly"; + const string successor = "query_stats_interval_hourly"; + const string fromExpr = "$1::timestamp"; + const string toExpr = "$2::timestamp"; + const string bucketWidthLiteral = "$3::interval"; + + var existsSql = TimescaleSupport.LegacySuccessorHoleExistsSql( + relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral); + var scanSql = TimescaleSupport.LegacySuccessorHoleScanSql( + relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral); + + Assert.StartsWith("EXISTS (", existsSql, StringComparison.Ordinal); + Assert.StartsWith("SELECT hb.bucket", scanSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY hb.bucket", scanSql, StringComparison.Ordinal); + + /* Strip each wrapper down to the shared buckets clause and compare verbatim — the whole point of the + refactor is that this body is ONE piece of text, not two that happen to agree today. Located by + IndexOf rather than a hardcoded literal length, so the file's own line-ending convention (CRLF) + does not throw the split off by one. */ + var existsBody = existsSql.Substring("EXISTS (".Length, existsSql.Length - "EXISTS (".Length - 1); + var scanFromIndex = scanSql.IndexOf("FROM (", StringComparison.Ordinal) + "FROM (".Length; + var scanCloseIndex = scanSql.LastIndexOf(") AS hb(bucket)", StringComparison.Ordinal); + var scanBody = scanSql.Substring(scanFromIndex, scanCloseIndex - scanFromIndex); + + Assert.Equal(existsBody, scanBody); + Assert.Contains("generate_series(", existsBody, StringComparison.Ordinal); + Assert.Contains("OFFSET 0", existsBody, StringComparison.Ordinal); + } + /// /// The start path: launched (not awaited) right after the ensure, on its own connection, inside the /// TimescaleDB block, before the compression and retention ensures; drained at shutdown beside the baseline diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs index 707cd48c1a..7a84428fc9 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs @@ -305,6 +305,33 @@ SELECT 1 FROM collect.{target.Source} AS s public static string LegacySuccessorHoleExistsSql( string relation, string sourceTimeColumn, string sourceFilter, string legacy, string successor, string fromExpr, string toExpr, string bucketWidthLiteral) + => $"EXISTS ({LegacySuccessorHoleBodySql(relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral)})"; + + /// + /// The LIST form of the same hole definition (#4301, H2): every hole bucket in + /// [, ], oldest first — the repair walk's own + /// shape ('s ORDER BY c.bucket), sharing the identical + /// body wraps in + /// EXISTS(...) for the gate. TEXT-IDENTICAL body, so the gate's EXISTS and the walk's + /// bucket list can never disagree about what a hole is. + /// + public static string LegacySuccessorHoleScanSql( + string relation, string sourceTimeColumn, string sourceFilter, string legacy, string successor, + string fromExpr, string toExpr, string bucketWidthLiteral) + => $@"SELECT hb.bucket +FROM ({LegacySuccessorHoleBodySql(relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral)}) AS hb(bucket) +ORDER BY hb.bucket"; + + /// + /// The shared body wraps in EXISTS(...) and + /// wraps in a SELECT ... ORDER BY — kept as ONE private + /// builder (#4301, H2) so the gate's EXISTS probe and the walk's bucket list can never drift into + /// disagreeing about what a hole is; a change to the definition edits exactly one place. TEXT of the + /// buckets clause is otherwise identical to 's own prior body. + /// + private static string LegacySuccessorHoleBodySql( + string relation, string sourceTimeColumn, string sourceFilter, string legacy, string successor, + string fromExpr, string toExpr, string bucketWidthLiteral) { ArgumentNullException.ThrowIfNull(relation); ArgumentNullException.ThrowIfNull(sourceTimeColumn); @@ -317,8 +344,8 @@ public static string LegacySuccessorHoleExistsSql( var filterClause = sourceFilter.Length == 0 ? string.Empty : $"\n AND {sourceFilter}"; - return $@"EXISTS ( - SELECT 1 + return $@" + SELECT hb.bucket FROM generate_series({fromExpr}, {toExpr}, {bucketWidthLiteral}) AS hb(bucket) WHERE NOT EXISTS (SELECT 1 FROM collect.{legacy} AS hl WHERE hl.bucket = hb.bucket OFFSET 0) AND NOT EXISTS (SELECT 1 FROM collect.{successor} AS hs WHERE hs.bucket = hb.bucket OFFSET 0) @@ -327,7 +354,7 @@ SELECT 1 FROM collect.{relation} AS hr WHERE hr.{sourceTimeColumn} >= hb.bucket AND hr.{sourceTimeColumn} < hb.bucket + {bucketWidthLiteral}{filterClause} OFFSET 0) - OFFSET 0)"; + OFFSET 0"; } /// The materialized span of one aggregate — its oldest and newest bucket — read off the From 1d7b1be8fcbae4af2aa6cd3d2682d256dddefb36 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:29:19 -0400 Subject: [PATCH 7/9] #4301: fill the successor's seam contiguously down to raw's filtered floor The walk's #4186 seam-fix block lowered seamFloor to the frozen legacy's last bucket + one bucket width. Per the ruling (#4301 comment 5844202704), the walk now lowers it to raw's own filtered floor (AlignDown'd), the same bound RetentionArmSafetySql's gate probes from, whenever that reaches further back than the successor's own floor. Everything downstream (the successor-only scan, the newest-first cap and walk, the shared cap with the ordinary window) is unchanged and is itself the contiguous downward fill: RollupCoverage.StitchedRelationSql splits its read at the successor's floor, so every row below it must land as a successor bucket for the stitch to read each row exactly once. Removed the now-dead legacyInteriorFrom/legacyInteriorTo third-window parameters from MaterializationHoleScanWindows and the two pins that exercised them (ScanWindows_LegacyInterior_*, ScanWindows_NoLegacyInterior_*): under this ruling a legacy-interior hole is repaired by the same newest-first seam descent as everything else below the successor's floor, so the separate oldest-first interior branch never gets built. Removed LegacySuccessorHoleScanSql (the walk's list-form twin of the gate's LegacySuccessorHoleExistsSql) since the walk no longer shares that hole definition; RetentionArmSafetySql keeps LegacySuccessorHoleExistsSql for its own probe. --- .../MaterializationHoleRepairTests.cs | 88 ++-------- .../TimescaleSupport.MaterializationHoles.cs | 155 ++++++++---------- 2 files changed, 84 insertions(+), 159 deletions(-) diff --git a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs index 45c9de5b40..5b55b59bbc 100644 --- a/Darling/Darling.Tests/MaterializationHoleRepairTests.cs +++ b/Darling/Darling.Tests/MaterializationHoleRepairTests.cs @@ -344,68 +344,22 @@ public void ScanWindows_EdgeCases_OneBucketSeam_NoWindowsWhenNothingToScan_AndTh /// /// #4301 (H2): a legacy-interior gap — a hole BELOW the legacy's last bucket, inside its own frozen span, /// from an outage that predates this store ever taking the freeze — gets a THIRD window, distinct from - /// both the seam window and the ordinary window, emitted FIRST (oldest of the three). RED on the pre-#4301 - /// signature: MaterializationHoleScanWindows had no parameter through which a legacy-interior span - /// could ever reach this method at all, so it returned at most the seam window and the ordinary window — - /// never anything anchored below l.mx — exactly the H2 regression this pins. + /// both the seam window and the ordinary window, emitted FIRST (oldest of the three). Superseded by the + /// #4301 ruling ("fill the successor CONTIGUOUSLY DOWNWARD"): the walk no longer takes a separate + /// legacy-interior window at all — the seam window's own lower bound moved to raw's filtered floor, so a + /// hole below the legacy's last bucket is repaired by the SAME newest-first descent as everything else + /// below the successor's floor. MaterializationHoleScanWindows dropped the + /// legacyInteriorFrom/legacyInteriorTo parameters this test exercised; removed with them. /// - [Fact] - public void ScanWindows_LegacyInterior_GivesAThirdWindow_EmittedFirst() - { - var width = TimescaleSupport.HourlyBucket; - var floor = Hour.AddHours(200); - var horizon = Hour.AddHours(190); - var seamFloor = Hour.AddHours(195); - var ceiling = Hour.AddHours(250); - var legacyInteriorFrom = Hour.AddHours(20); - var legacyInteriorTo = Hour.AddHours(180); - - var windows = TimescaleSupport.MaterializationHoleScanWindows( - floor, ceiling, horizon, seamFloor, width, legacyInteriorFrom, legacyInteriorTo); - - Assert.Equal(3, windows.Count); - Assert.Equal((legacyInteriorFrom, legacyInteriorTo), windows[0]); - Assert.Equal((seamFloor, floor.AddHours(-1)), windows[1]); - Assert.Equal((floor, ceiling), windows[2]); - - /* The legacy-interior window is unclamped by the horizon — same reasoning as the seam window: the - outage that opened it left the source with no rows there, not purged, so it can sit however far - below the horizon it needs to. */ - Assert.True(windows[0].From < horizon); - } /// - /// #4301: no legacy-interior span (both bounds null, the ordinary case once a store has run a while, or - /// any relation without a frozen legacy) yields exactly the same two windows #4186 already produced — - /// the new parameter is purely additive and does not disturb the seam/ordinary shape when it is absent. - /// An inverted or empty span (from after to) is also omitted, the same rule the seam window already uses. + /// #4301 (H2, filter parity): (the gate's + /// EXISTS wrapper) is the shared hole definition + /// uses. This pins its shape: a well-formed EXISTS(...) wrapping the generate_series(...) + /// buckets clause, fenced with OFFSET 0 for the #3933 reason its own doc states. /// [Fact] - public void ScanWindows_NoLegacyInterior_LeavesTheExistingTwoWindowsUnchanged() - { - var width = TimescaleSupport.HourlyBucket; - var floor = Hour.AddHours(10); - var horizon = Hour.AddHours(2); - var seamFloor = Hour.AddHours(4); - var ceiling = Hour.AddHours(50); - - var windows = TimescaleSupport.MaterializationHoleScanWindows(floor, ceiling, horizon, seamFloor, width); - Assert.Equal(new[] { (seamFloor, floor.AddHours(-1)), (floor, ceiling) }, windows); - - var invertedInteriorWindows = TimescaleSupport.MaterializationHoleScanWindows( - floor, ceiling, horizon, seamFloor, width, legacyInteriorFrom: Hour.AddHours(5), legacyInteriorTo: Hour.AddHours(1)); - Assert.Equal(windows, invertedInteriorWindows); - } - - /// - /// #4301 (H2, filter parity companion): (the - /// gate's EXISTS wrapper) and (the walk's - /// bucket LIST) must share the IDENTICAL body text — one private builder, two wrappers — so the gate and - /// the walk can never drift into disagreeing about what a hole is. Asserts the shared - /// generate_series(...) ... OFFSET 0 buckets clause appears verbatim inside both generated strings. - /// - [Fact] - public void LegacySuccessorHoleExistsSql_AndLegacySuccessorHoleScanSql_ShareTheIdenticalBody() + public void LegacySuccessorHoleExistsSql_WrapsTheBucketsClauseInExists() { const string relation = "query_stats"; const string sourceTimeColumn = "collection_time"; @@ -418,25 +372,11 @@ public void LegacySuccessorHoleExistsSql_AndLegacySuccessorHoleScanSql_ShareTheI var existsSql = TimescaleSupport.LegacySuccessorHoleExistsSql( relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral); - var scanSql = TimescaleSupport.LegacySuccessorHoleScanSql( - relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral); Assert.StartsWith("EXISTS (", existsSql, StringComparison.Ordinal); - Assert.StartsWith("SELECT hb.bucket", scanSql, StringComparison.Ordinal); - Assert.Contains("ORDER BY hb.bucket", scanSql, StringComparison.Ordinal); - - /* Strip each wrapper down to the shared buckets clause and compare verbatim — the whole point of the - refactor is that this body is ONE piece of text, not two that happen to agree today. Located by - IndexOf rather than a hardcoded literal length, so the file's own line-ending convention (CRLF) - does not throw the split off by one. */ - var existsBody = existsSql.Substring("EXISTS (".Length, existsSql.Length - "EXISTS (".Length - 1); - var scanFromIndex = scanSql.IndexOf("FROM (", StringComparison.Ordinal) + "FROM (".Length; - var scanCloseIndex = scanSql.LastIndexOf(") AS hb(bucket)", StringComparison.Ordinal); - var scanBody = scanSql.Substring(scanFromIndex, scanCloseIndex - scanFromIndex); - - Assert.Equal(existsBody, scanBody); - Assert.Contains("generate_series(", existsBody, StringComparison.Ordinal); - Assert.Contains("OFFSET 0", existsBody, StringComparison.Ordinal); + Assert.EndsWith(")", existsSql, StringComparison.Ordinal); + Assert.Contains("generate_series(", existsSql, StringComparison.Ordinal); + Assert.Contains("OFFSET 0", existsSql, StringComparison.Ordinal); } /// diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs index 7a84428fc9..830e9dae49 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.MaterializationHoles.cs @@ -55,26 +55,31 @@ namespace PerformanceMonitor.Darling.Storage; /// business and are never touched here; buckets past the last materialized one are the live edge the policy /// owns. /// -/// The seam case (#4186): a successor with a frozen legacy gets a SECOND scan window down to the -/// LEGACY's last bucket, scanned however far below the horizon it reaches. The three interval-honest -/// successors (SupersededHourlyRollups) can each have an un-materialized TAIL below their own floor: an -/// outage that outlasts before the successor's first refresh leaves raw -/// rows between the frozen legacy's last bucket and the successor's floor that neither side ever materialized -/// (the outage shape worked through above). Those rows sit BELOW the successor's floor, so the ordinary "floor -/// up to ceiling" scan never reaches them — a hole, by this pass's own definition, has to be inside the -/// materialized span. For a successor found through LegacyOf, -/// adds a seam window down to the legacy's last bucket (plus one bucket width, so the legacy's own last bucket is -/// not re-scanned as if it were the successor's) whenever that reaches further back than the successor's floor +/// The seam case (#4186), lowered to a CONTIGUOUS DOWNWARD FILL by #4301's ruling. The three +/// interval-honest successors (SupersededHourlyRollups) can each have an un-materialized span BELOW +/// their own floor: an outage that outlasts before the successor's +/// first refresh leaves raw rows between the frozen legacy's last bucket and the successor's floor that +/// neither side ever materialized (the outage shape worked through above) — and, separately, an outage that +/// predates this store's freeze can leave a hole INSIDE the legacy's own already-materialized span that +/// nothing ever re-scans, because the six frozen views are excluded from . +/// Both sit BELOW the successor's floor, so the ordinary "floor up to ceiling" scan never reaches them — a +/// hole, by this pass's own definition, has to be inside the materialized span. For a successor found through +/// LegacyOf, adds a seam window down to RAW's own filtered +/// floor — not merely the legacy's last bucket — whenever that reaches further back than the successor's floor /// already does — UNCLAMPED by the horizon that still bounds the ordinary window, because an outage longer than /// the horizon's own span is exactly the shape that needs repairing, not a shape to skip (an earlier cut folded /// the seam into that same horizon clamp, and a seam older than the horizon was silently never scanned — see -/// 's own doc for that history). The seam tail then reads as an -/// ordinary hole and the existing machinery repairs it: bounded per start, oldest first, filter-aware — a seam -/// wider than one start's cap () takes more than one start to -/// close in full, but every start makes progress on it. Once repaired, the successor's floor covers the seam on -/// its own and 's seam probe — which exists because this stitch is NOT -/// gap-free by construction — finds nothing there and releases the raw purge gate automatically, with no manual -/// step, bounded only by that same per-start repair cap. +/// 's own doc for that history). Filling all the way to raw's floor +/// rather than stopping at the legacy's last bucket is the point of the #4301 ruling: RollupCoverage.StitchedRelationSql +/// splits its read at the successor's own floor, so that floor has to be contiguous with everything raw still +/// admits for the stitch to read every row exactly once. The seam window then reads as an ordinary hole and +/// the existing machinery repairs it: bounded per start, NEWEST FIRST (the successor's own invariant — see the +/// H1 note below), filter-aware — a span wider than one start's cap () +/// takes more than one start to close in full, but every start makes progress on it, walking downward from the +/// successor's floor toward raw's. Once repaired, the successor's floor covers the seam on its own and +/// 's seam probe — which exists because this stitch is NOT gap-free by +/// construction — finds nothing there and releases the raw purge gate automatically, with no manual step, +/// bounded only by that same per-start repair cap. /// /// Bounded, and the bound is stated. Per aggregate per start, at most one refresh policy window's /// worth of buckets (: 24 hourly, 3 daily) is refreshed, @@ -287,13 +292,11 @@ SELECT 1 FROM collect.{target.Source} AS s } /// - /// ONE hole definition for a frozen-legacy/successor pair (#4301), shared TEXT-IDENTICAL by - /// (the gate) and — the next lane — the repair walk's - /// interior/seam branch: a bucket in [, ] is a - /// hole when raw admits at least one row in [bucket, bucket + width) AND neither the legacy nor the - /// successor has materialized that bucket. The gate and the walk must never disagree about what a hole is — - /// a bucket the gate calls Short that the walk never repairs holds the raw purge forever with no - /// self-release. / are SQL expressions (a literal, a + /// ONE hole definition for a frozen-legacy/successor pair (#4301), used by + /// (the gate): a bucket in + /// [, ] is a hole when raw admits at least one + /// row in [bucket, bucket + width) AND neither the legacy nor the successor has materialized that + /// bucket. / are SQL expressions (a literal, a /// parameter placeholder, a correlated subquery) so each caller supplies its own bounds in its own idiom. /// OFFSET 0 fenced for the same #3933 reason is: written bare, the /// planner pulls the per-bucket EXISTS probes up into joins that scan the whole relation instead of @@ -301,6 +304,15 @@ SELECT 1 FROM collect.{target.Source} AS s /// never be a hole under this definition and never holds the purge — a gap left below the floor by an /// earlier version's purge is invisible here by construction, not merely undetected (see this member's own /// callers for what that means for the gate). + /// + /// The repair walk (#4301, the ruling lane) does NOT share this definition. An earlier cut of + /// #4301 planned a walk branch that probed the legacy-or-successor union the same way the gate does; the + /// ruling replaced it with a plain successor-only fill down to raw's own filtered floor + /// ('s seam-floor block), so the walk's own hole definition + /// stays (successor-only) throughout — every non-empty hour below + /// the successor's floor simply becomes a successor bucket. The LIST-form twin this method used to have + /// (LegacySuccessorHoleScanSql) had no other caller once that ruling landed and was removed with it. + /// /// public static string LegacySuccessorHoleExistsSql( string relation, string sourceTimeColumn, string sourceFilter, string legacy, string successor, @@ -308,26 +320,9 @@ public static string LegacySuccessorHoleExistsSql( => $"EXISTS ({LegacySuccessorHoleBodySql(relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral)})"; /// - /// The LIST form of the same hole definition (#4301, H2): every hole bucket in - /// [, ], oldest first — the repair walk's own - /// shape ('s ORDER BY c.bucket), sharing the identical - /// body wraps in - /// EXISTS(...) for the gate. TEXT-IDENTICAL body, so the gate's EXISTS and the walk's - /// bucket list can never disagree about what a hole is. - /// - public static string LegacySuccessorHoleScanSql( - string relation, string sourceTimeColumn, string sourceFilter, string legacy, string successor, - string fromExpr, string toExpr, string bucketWidthLiteral) - => $@"SELECT hb.bucket -FROM ({LegacySuccessorHoleBodySql(relation, sourceTimeColumn, sourceFilter, legacy, successor, fromExpr, toExpr, bucketWidthLiteral)}) AS hb(bucket) -ORDER BY hb.bucket"; - - /// - /// The shared body wraps in EXISTS(...) and - /// wraps in a SELECT ... ORDER BY — kept as ONE private - /// builder (#4301, H2) so the gate's EXISTS probe and the walk's bucket list can never drift into - /// disagreeing about what a hole is; a change to the definition edits exactly one place. TEXT of the - /// buckets clause is otherwise identical to 's own prior body. + /// The body wraps in EXISTS(...) — kept as its own + /// method (#4301, H2) so a future second caller can share it without duplicating the buckets clause; + /// today is its only caller. /// private static string LegacySuccessorHoleBodySql( string relation, string sourceTimeColumn, string sourceFilter, string legacy, string successor, @@ -458,8 +453,8 @@ public static (IReadOnlyList<(DateTime Start, DateTime End)> Repair, IReadOnlyLi /// The scan window(s) for one aggregate this start, given its own and /// , the horizon computes for its /// source, and the seam bound (, equal to when the - /// aggregate has no frozen legacy or the legacy's own last bucket does not reach back past the floor). - /// Pure, so the tests can walk it. + /// aggregate has no frozen legacy or raw's own filtered floor does not reach back past the successor's + /// floor). Pure, so the tests can walk it. /// /// Two windows, not one (#4186 follow-up). The seam fix's first cut folded the seam into the /// SAME max(_, horizon) the ordinary scan already clamps to — from = max(seamFloor, horizon) @@ -476,28 +471,16 @@ public static (IReadOnlyList<(DateTime Start, DateTime End)> Repair, IReadOnlyLi /// — the successor's own span below the horizon is the source retention's business, not this repair's, and /// widening it was never the fix. /// - /// A third window, INSIDE the frozen legacy's own span (#4301, H2). The seam window only - /// reaches down to the legacy's last bucket — it repairs the outage AFTER the freeze. #4186 left the - /// legacy's OWN materialized span (below its last bucket) unchecked entirely: an outage BEFORE this store - /// ever took the freeze can leave a hole inside history the legacy claims to hold, and nothing ever - /// re-scans it because the six frozen views are excluded from - /// (repairing a frozen view is the one thing the freeze forbids — see that member's own note). This window, - /// through (typically - /// [time_bucket(source's oldest admitted row), legacy.max(bucket)]), is scanned against BOTH the - /// legacy's and the successor's own materializations — a bucket either one already holds is not a hole — - /// and its caller must never refresh the legacy for what it finds, only the successor - /// ('s legacy-interior branch). It is OPTIONAL (both bounds - /// null, or an empty/inverted span) for every relation without a frozen legacy, and for one whose raw - /// floor no longer reaches back into the legacy's span at all — the ordinary case once the store has run a - /// while. Emitted FIRST (oldest), ahead of the seam: repairing it can never move the successor's own floor - /// ('s stitch only reads s.mn, never a legacy-interior bucket), so - /// it is capped and walked oldest-first exactly like the ordinary window — see - /// for why it shares that pool rather than the seam's - /// newest-first one. + /// The seam now reaches raw's own filtered floor, not merely the legacy's last bucket (#4301, + /// per the ruling "fill the successor CONTIGUOUSLY DOWNWARD"). An earlier cut of this method took a + /// separate, oldest-first-walked third window for a hole strictly INSIDE the frozen legacy's own span — + /// dead code once the ruling landed: the walk fills every hole below the successor's floor down to raw's + /// floor in ONE newest-first descent (the seam window itself, now with its lower bound moved), because + /// contiguity from the successor's floor upward is the property RollupCoverage.StitchedRelationSql + /// needs, and a bucket does not care which side of the legacy's last bucket it happened to sit on. /// public static IReadOnlyList<(DateTime From, DateTime To)> MaterializationHoleScanWindows( - DateTime floor, DateTime ceiling, DateTime horizon, DateTime seamFloor, TimeSpan bucketWidth, - DateTime? legacyInteriorFrom = null, DateTime? legacyInteriorTo = null) + DateTime floor, DateTime ceiling, DateTime horizon, DateTime seamFloor, TimeSpan bucketWidth) { if (bucketWidth <= TimeSpan.Zero) { @@ -506,11 +489,6 @@ public static (IReadOnlyList<(DateTime Start, DateTime End)> Repair, IReadOnlyLi var windows = new List<(DateTime From, DateTime To)>(); - if (legacyInteriorFrom is { } interiorFrom && legacyInteriorTo is { } interiorTo && interiorFrom <= interiorTo) - { - windows.Add((interiorFrom, interiorTo)); - } - if (seamFloor < floor) { var seamTo = floor - bucketWidth; @@ -625,25 +603,32 @@ public static async Task RepairMaterialization continue; } - /* #4186 seam fix: a successor whose legacy is frozen (LegacyOf, non-null only for the three - SupersededHourlyRollups successors) can hold an un-materialized tail BELOW its own floor — - the seam an outage opens between the legacy's last bucket and the successor's first refresh - (see this class's doc, and RetentionArmSafetySql's, for the full shape). A hole is defined as - a gap INSIDE the materialized span, so scanning from the successor's own floor never reaches - that tail. Extend the lower bound down to the legacy's last bucket (+ one bucket width, so - the legacy's own already-materialized last bucket is not rescanned) whenever that reaches - further back than the successor's own floor; min() is a no-op once the successor's floor - overtakes the legacy's boundary on its own, so this converges to plain floor scanning as the - successor accumulates history. A legacy with nothing materialized (max(bucket) is NULL, a - frozen-but-empty legacy) leaves the floor untouched. */ + /* #4186 seam fix, lowered by #4301's ruling ("fill the successor CONTIGUOUSLY DOWNWARD"): + a successor whose legacy is frozen (LegacyOf, non-null only for the three + SupersededHourlyRollups successors) can hold an un-materialized span BELOW its own floor — + not only the seam an outage opens between the legacy's last bucket and the successor's + first refresh, but any hole INSIDE the legacy's own frozen span from an outage that predates + this store's freeze (see this class's doc, and RetentionArmSafetySql's, for the full shape). + A hole is defined as a gap INSIDE the materialized span, so scanning from the successor's own + floor never reaches either one. Extend the lower bound down to raw's own filtered floor — + the successor's admitted source floor, the SAME bound RetentionArmSafetySql's stitch probes + from — whenever that reaches further back than the successor's own floor; below it raw + admits no row, so no hole can exist there and the walk has nothing left to fill (this is the + contiguous-downward fill: RollupCoverage.StitchedRelationSql splits its read at the + successor's floor, so every row below it must already be a successor bucket once this + converges). min() is a no-op once the successor's floor overtakes raw's floor on its own, so + this converges to plain floor scanning as the successor accumulates history. A raw table + with nothing admitted (min is NULL) leaves the floor untouched. */ var seamFloor = floor.Value; var legacy = LegacyOf(target.View); if (legacy is not null) { - using var legacyCeiling = new NpgsqlCommand($"SELECT max(bucket) FROM collect.{legacy}", connection) { CommandTimeout = SetupTimeoutSeconds }; - if (await legacyCeiling.ExecuteScalarAsync(cancellationToken) is DateTime legacyMaxBucket) + var sourceFilter = MaterializationHoleSourceFilterFor(target.CreateSql); + var sourceWhere = sourceFilter.Length == 0 ? string.Empty : $" WHERE {sourceFilter}"; + using var rawFilteredFloor = new NpgsqlCommand($"SELECT min({target.SourceTimeColumn}) FROM collect.{target.Source}{sourceWhere}", connection) { CommandTimeout = SetupTimeoutSeconds }; + if (await rawFilteredFloor.ExecuteScalarAsync(cancellationToken) is DateTime rawFloor) { - var seamBound = legacyMaxBucket + target.BucketWidth; + var seamBound = AlignDown(rawFloor, target.BucketWidth); if (seamBound < seamFloor) { seamFloor = seamBound; From e856edcc9348f79bac7545a3a5c473b60f80fcdf Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:41:24 -0400 Subject: [PATCH 8/9] Update the seam pins for the raw-floor fill (#4301) Three pre-existing FrozenRollupLiveTests seam tests expected the old seam-tail-only fill (2-bucket seam above the legacy boundary, or 11 of 35 seam buckets on the wide-seam pass). The #4301 fix lowers the walk's seam floor to raw's own filtered floor, so the walk now fills contiguously down to raw's floor: 7 buckets (not 2) on the two 6-hour tail tests, and a 36-bucket seam (not 35) split 24/12 (not 24/11) on the wide-seam test. Updated the expected counts, floors, and comments to match; no assertions were loosened. --- .../Darling.Tests/FrozenRollupLiveTests.cs | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/Darling/Darling.Tests/FrozenRollupLiveTests.cs b/Darling/Darling.Tests/FrozenRollupLiveTests.cs index 27b527b562..c539378d35 100644 --- a/Darling/Darling.Tests/FrozenRollupLiveTests.cs +++ b/Darling/Darling.Tests/FrozenRollupLiveTests.cs @@ -512,14 +512,16 @@ public async Task Outage_SeamBetweenFrozenLegacyAndSuccessor_HoleWalkRepairsItAn /* The seam holds raw rows the stitch cannot see through unconditionally — Short, not Covered. */ Assert.False(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); - /* The product's own start-path entry point (DarlingWorker calls this exact method). */ + /* The product's own start-path entry point (DarlingWorker calls this exact method). #4301: the + walk now fills CONTIGUOUSLY down to raw's own filtered floor (s-6h), not just the seam tail + above the legacy's boundary — 7 buckets (s-6h..s), not the old 2-bucket seam tail. */ var summary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); - Assert.True(summary.BucketsRepaired >= 2, $"expected the 2-bucket seam tail to be repaired, got {summary.BucketsRepaired}"); + Assert.Equal(7, summary.BucketsRepaired); await using (var span = new NpgsqlCommand($"SELECT min(bucket) FROM collect.{TimescaleSupport.ProcedureStatsIntervalHourlyView}", connection)) { var newFloor = (DateTime)(await span.ExecuteScalarAsync(ct))!; - Assert.Equal(s.AddHours(-1), newFloor); + Assert.Equal(s.AddHours(-6), newFloor); } /* The seam is now empty (the successor's own floor reaches the legacy's boundary) — Covered. */ @@ -607,14 +609,16 @@ computes from U. */ /* The product's own start-path entry point (DarlingWorker calls this exact method). Before the #4186 follow-up fix, the seam's lower bound was clamped to U minus the 4-day span — comfortably ABOVE the seam, since the outage is 6 days — so this repaired 0 buckets and the seam stood - forever without a manual --backfill-rollups. */ + forever without a manual --backfill-rollups. #4301: the walk now fills CONTIGUOUSLY down to + raw's own filtered floor (s-6h), not just the seam tail above the legacy's boundary — 7 + buckets (s-6h..s), not the old 2-bucket seam tail. */ var summary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); - Assert.True(summary.BucketsRepaired >= 2, $"expected the 2-bucket seam tail to be repaired, got {summary.BucketsRepaired}"); + Assert.Equal(7, summary.BucketsRepaired); await using (var span = new NpgsqlCommand($"SELECT min(bucket) FROM collect.{TimescaleSupport.ProcedureStatsIntervalHourlyView}", connection)) { var newFloor = (DateTime)(await span.ExecuteScalarAsync(ct))!; - Assert.Equal(s.AddHours(-1), newFloor); + Assert.Equal(s.AddHours(-6), newFloor); } /* The seam is now empty (the successor's own floor reaches the legacy's boundary) — Covered. */ @@ -640,8 +644,8 @@ await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async ( /// early. Oldest-first repaired the buckets FARTHEST from the successor's floor first, which still moved /// the floor (a bare min(bucket)) all the way down to them — stranding the un-repaired NEWER seam /// buckets above the new floor and outside 's probe. - /// Newest-first must NOT do that: repairing the top 24 of a 35-bucket seam should leave the floor exactly - /// adjacent to the still-open 11-bucket remainder, so the probe keeps finding it and the gate stays Short + /// Newest-first must NOT do that: repairing the top 24 of a 36-bucket seam should leave the floor exactly + /// adjacent to the still-open 12-bucket remainder, so the probe keeps finding it and the gate stays Short /// until a second walk closes the rest. /// [Fact] @@ -674,9 +678,11 @@ public async Task Outage_SeamWiderThanTheCap_NewestFirstRepairsTheTopAndKeepsThe var bodySucceeded = false; try { - /* S is the stop. The legacy materializes only ONE bucket, 35 hours back, so l.mx = S-35h and the - seam floor is S-34h. Raw carries an unbroken run of 35 hourly buckets from S-34h through S — - wider than the 24-bucket cap, so ONE walk cannot close it in one pass. */ + /* S is the stop. The legacy materializes only ONE bucket, 35 hours back, so l.mx = S-35h. Raw + carries an unbroken run of 36 hourly buckets from S-35h through S — #4301: the walk's seam + floor is raw's own filtered floor (S-35h), not the legacy's max+width (S-34h), so the oldest + raw bucket is IN the seam too — wider than the 24-bucket cap, so ONE walk cannot close it in + one pass. */ var s = D0.AddDays(3); for (var hour = 0; hour <= 35; hour++) @@ -692,9 +698,11 @@ seam floor is S-34h. Raw carries an unbroken run of 35 hourly buckets from S-34h Assert.False(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); - /* Walk 1: the cap takes the NEWEST 24 of the 35 seam buckets (S-23h..S), leaving the OLDER 11 - (S-34h..S-24h) as a hole immediately below the new floor. The product's own start-path entry - point (DarlingWorker calls this exact method). */ + /* Walk 1: the cap takes the NEWEST 24 of the 36 seam buckets (S-23h..S), leaving the OLDER 12 + (S-35h..S-24h) as a hole immediately below the new floor. The product's own start-path entry + point (DarlingWorker calls this exact method). #4301: the seam floor is raw's own filtered + floor (S-35h, the oldest raw row), not the legacy's max+width (S-34h) — one bucket lower, so + the seam is 36 wide, not 35. */ var summary1 = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); Assert.Equal(24, summary1.BucketsRepaired); @@ -704,10 +712,10 @@ seam floor is S-34h. Raw carries an unbroken run of 35 hourly buckets from S-34h walk. */ Assert.False(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); - /* Walk 2: the remaining 11-bucket range is now the whole seam (S-34h..S-24h, under the cap), and + /* Walk 2: the remaining 12-bucket range is now the whole seam (S-35h..S-24h, under the cap), and closes it completely. */ var summary2 = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); - Assert.Equal(11, summary2.BucketsRepaired); + Assert.Equal(12, summary2.BucketsRepaired); Assert.True(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); From ab7b0db4edb0ed5f516698fd837bf0881fb26146 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 26 Sep 2026 03:51:21 -0400 Subject: [PATCH 9/9] Live pins 1-5 for the raw-floor contiguous fill (#4301) Five new live tests in FrozenRollupLiveTests.cs against RepairMaterializationHolesAsync / IsRawTierDropSafeAsync / the StitchedRelationSql read path, modelled on lane 1d's pins A-D: - ARepairedShape_SuccessorFloorAtOrBelowRawsFloor_RepairsNothing (1): an A6-backfilled shape where the walk has nothing left to do. - InteriorHole_RepairedByTheWalk_ThenReportsCovered (2): an interior hole reads Short, converges to Covered after enough walk passes. - StitchedReadThroughTheWalk_KeepsEveryRow_AndSuccessorFloorReachesRawsFloor (3): a stitched sum read (RollupCoverage.StitchedRelationSql) over [raw floor, now) is identical before and after the walk, and the successor's floor reaches raw's floor afterward. RED on abc8ce3c4, verified in a detached worktree: the old walk leaves the floor at the seam tail (b+6h) instead of raw's floor (b). - CapAcrossPasses_LeavesNoGapBetweenPasses (4): a 31-bucket range wider than the 24-bucket cap fills 24 then 6 with no gap between passes. - ProbeFailure_UnknownSourceState_ReportsRawPurgeNotSafe (5): renaming the successor CAGG mid-run makes the gate's probe fail; the Unknown verdict answers false, same as a measured Short. All 19 tests in the class (14 existing + 5 new) green in-process on a local TimescaleDB 2.30.1-pg18 rig. Also re-ran TimescaleContinuousAggregateTests, MaterializationHoleRepairTests, MaterializationHoleRepairLiveTests, RollupBackfillLiveTests, RetentionReevaluationLiveTests: 73 passed, 3 skipped, 0 failed -- unaffected by the #4301 change. --- .../Darling.Tests/FrozenRollupLiveTests.cs | 418 ++++++++++++++++++ 1 file changed, 418 insertions(+) diff --git a/Darling/Darling.Tests/FrozenRollupLiveTests.cs b/Darling/Darling.Tests/FrozenRollupLiveTests.cs index c539378d35..aa10c36f5c 100644 --- a/Darling/Darling.Tests/FrozenRollupLiveTests.cs +++ b/Darling/Darling.Tests/FrozenRollupLiveTests.cs @@ -1278,6 +1278,424 @@ await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async ( } } + /// + /// #4301, PIN 1 (no-op on an A6-backfilled shape): the successor's floor already sits AT OR BELOW raw's + /// own filtered floor, exactly the shape a store leaves once #4301's walk (or a manual --backfill-rollups) + /// has already caught the successor up. There is nothing below the successor's floor for the walk to find, + /// so must repair NOTHING. + /// + [Fact] + public async Task ARepairedShape_SuccessorFloorAtOrBelowRawsFloor_RepairsNothing() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(30); + + /* Raw admits rows at H0..H3; the successor already refreshed the WHOLE span, so its floor equals + raw's own filtered floor -- the A6-backfilled shape. No legacy row anywhere. */ + for (var hour = 0; hour <= 3; hour++) + { + await InsertProcedureStatsAsync(connection, b.AddHours(hour), $"pin1_h{hour}", 900, 9, 3600, ct); + } + + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b, b.AddHours(4), ct); + + Assert.True(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + var summary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, b.AddHours(4), ct); + Assert.Equal(0, summary.BucketsRepaired); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + + /// + /// #4301, PIN 2 (interior hole: Short, then the walk, then Covered): an interior hole below the legacy's + /// last bucket (H2 skipped, as PIN A), and the successor floor already sits above l.mx (H6). Before the + /// walk, reads false. Running + /// repeatedly until it converges must + /// eventually leave the successor covering the whole interior span, after which the same probe reads + /// true. + /// + [Fact] + public async Task InteriorHole_RepairedByTheWalk_ThenReportsCovered() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(40); + + /* Same shape as PIN A: raw at H0, H2 (interior hole), H5 (legacy's last), H6 (successor floor). */ + await InsertProcedureStatsAsync(connection, b, "pin2_h0", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(2), "pin2_h2", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(5), "pin2_h5", 900, 9, 3600, ct); + await InsertProcedureStatsAsync(connection, b.AddHours(6), "pin2_h6", 900, 9, 3600, ct); + + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b, b.AddHours(1), ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b.AddHours(5), b.AddHours(6), ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b.AddHours(6), b.AddHours(7), ct); + + Assert.False(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + var u = b.AddHours(7); + for (var pass = 0; pass < 10 && !await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct); pass++) + { + var summary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); + if (summary.BucketsRepaired == 0) + { + break; + } + } + + Assert.True(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + + /// + /// #4301, PIN 3 (stitched reads keep every row -- the trap): legacy buckets cover [raw floor, l.mx] with + /// NO zero-interval rows, so legacy and successor agree on totals; the successor holds only its own + /// buckets from l.mx + 1h up. A sum read through over + /// [raw floor, now) BEFORE the walk must equal the SAME read AFTER the walk -- the walk only fills + /// successor buckets the legacy already agreed with, so the stitch (which reads whichever side a bucket's + /// time falls on) must not double count or drop anything. After the walk, the successor's own floor must + /// reach down to raw's floor. + /// + [Fact] + public async Task StitchedReadThroughTheWalk_KeepsEveryRow_AndSuccessorFloorReachesRawsFloor() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(50); + + /* Raw admits an unbroken run of hourly rows b..b+5h (no zero-interval rows), legacy materializes + ALL of them (b..b+5h), and the successor only has its own advance from b+6h up. The seam is + [b, b+5h] -- below the successor's floor, above raw's floor -- exactly the walk's fill range. */ + for (var hour = 0; hour <= 5; hour++) + { + await InsertProcedureStatsAsync(connection, b.AddHours(hour), $"pin3_h{hour}", 900, 9, 3600, ct); + } + + await InsertProcedureStatsAsync(connection, b.AddHours(6), "pin3_h6", 900, 9, 3600, ct); + + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsHourlyView, b, b.AddHours(6), ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b.AddHours(6), b.AddHours(7), ct); + + var u = b.AddHours(7); + + await using var dataSource = NpgsqlDataSource.Create(scratch.ConnectionString); + + async Task ReadStitchedSumAsync() + { + var rollups = await TimescaleSupport.DetectRollupsAsync(dataSource, ct); + var coverage = await TimescaleSupport.DetectRollupCoverageAsync(dataSource, rollups, ct); + var fromClause = coverage.StitchedRelationSql( + TimescaleSupport.ProcedureStatsHourlyView, "p", b.AddYears(-1), RollupCoverage.StitchTier.Hourly); + await using var command = new NpgsqlCommand( + $"SELECT coalesce(sum(worker_time_sum), 0) FROM {fromClause}", connection); + return Convert.ToDecimal(await command.ExecuteScalarAsync(ct)); + } + + var before = await ReadStitchedSumAsync(); + Assert.True(before > 0, "expected the pre-walk stitched read to see the legacy's rows"); + + for (var pass = 0; pass < 10; pass++) + { + var summary = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); + if (summary.BucketsRepaired == 0) + { + break; + } + } + + var after = await ReadStitchedSumAsync(); + Assert.Equal(before, after); + + await using (var span = new NpgsqlCommand($"SELECT min(bucket) FROM collect.{TimescaleSupport.ProcedureStatsIntervalHourlyView}", connection)) + { + var newFloor = (DateTime)(await span.ExecuteScalarAsync(ct))!; + Assert.Equal(b, newFloor); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + + /// + /// #4301, PIN 4 (the cap across passes): a range wider than + /// below raw's floor. Pass 1 fills the newest cap-worth from the top; pass 2 continues from the new floor + /// down. After enough passes every non-empty hour in [raw floor, original floor) has a successor + /// bucket, with no gap between the two passes' ranges. + /// + [Fact] + public async Task CapAcrossPasses_LeavesNoGapBetweenPasses() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + /* Raw admits an unbroken run of 30 hourly rows -- wider than the 24-bucket cap -- with NO legacy + materialization at all, so the seam floor is raw's own filtered floor. The successor holds + only its own advance above the whole span. */ + var b = D0.AddDays(60); + + for (var hour = 0; hour <= 30; hour++) + { + await InsertProcedureStatsAsync(connection, b.AddHours(hour), $"pin4_h{hour}", 900, 9, 3600, ct); + } + + var u = b.AddHours(31); + await InsertProcedureStatsAsync(connection, u.AddHours(-1), "pin4_successor_floor", 900, 9, 3600, ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, u.AddHours(-1), u, ct); + + var summary1 = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); + Assert.Equal(24, summary1.BucketsRepaired); + + DateTime floorAfterPass1; + await using (var span = new NpgsqlCommand($"SELECT min(bucket) FROM collect.{TimescaleSupport.ProcedureStatsIntervalHourlyView}", connection)) + { + floorAfterPass1 = (DateTime)(await span.ExecuteScalarAsync(ct))!; + } + + var summary2 = await TimescaleSupport.RepairMaterializationHolesAsync(connection, null, u, ct); + Assert.Equal(6, summary2.BucketsRepaired); + + DateTime floorAfterPass2; + await using (var span = new NpgsqlCommand($"SELECT min(bucket) FROM collect.{TimescaleSupport.ProcedureStatsIntervalHourlyView}", connection)) + { + floorAfterPass2 = (DateTime)(await span.ExecuteScalarAsync(ct))!; + } + + /* No gap between the two passes' ranges: pass 2's floor is exactly one bucket below pass 1's. */ + Assert.Equal(floorAfterPass1.AddHours(-6), floorAfterPass2); + Assert.Equal(b, floorAfterPass2); + + Assert.True(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + + /// + /// #4301, PIN 5 (an unknown source state holds): the gate's probe is made to fail by renaming the + /// successor's underlying continuous aggregate mid-run, so + /// names a relation that no longer exists. must fail + /// closed: an Unknown probe answers false, same as a measured Short, never true. + /// + [Fact] + public async Task ProbeFailure_UnknownSourceState_ReportsRawPurgeNotSafe() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), + "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live A6 freeze test."); + + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + var timescaleEnabled = await TimescaleSupport.TryEnableAsync(connection, null, ct); + Assert.SkipWhen(!timescaleEnabled, "The live A6 freeze test needs TimescaleDB."); + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + Assert.True(await TimescaleSupport.EnsureCollectionLogHypertableAsync(connection, null, ct)); + + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await DarlingMcpTestData.RegisterServerAsync(connection, ServerId, ServerName, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + var bodySucceeded = false; + try + { + var b = D0.AddDays(70); + + await InsertProcedureStatsAsync(connection, b, "pin5_h0", 900, 9, 3600, ct); + await RefreshAsync(connection, TimescaleSupport.ProcedureStatsIntervalHourlyView, b, b.AddHours(1), ct); + + /* Baseline: with the successor's own view present, the gate reads Covered. */ + Assert.True(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + /* Rename the successor CAGG's view underneath the gate -- RetentionArmSafetySql still names the + old view, so its probe query fails with an undefined-relation error, caught and turned into + the Unknown verdict. */ + await using (var rename = new NpgsqlCommand( + $"ALTER MATERIALIZED VIEW collect.{TimescaleSupport.ProcedureStatsIntervalHourlyView} RENAME TO pin5_renamed_away", connection)) + { + await rename.ExecuteNonQueryAsync(ct); + } + + Assert.False(await TimescaleSupport.IsRawTierDropSafeAsync(connection, "procedure_stats", ct)); + + /* Restore the name so LiveStoreCleanup's own teardown (DROP DATABASE) does not trip over an + unexpected shape while the scratch database is torn down. */ + await using (var restore = new NpgsqlCommand( + $"ALTER MATERIALIZED VIEW collect.pin5_renamed_away RENAME TO {TimescaleSupport.ProcedureStatsIntervalHourlyView}", connection)) + { + await restore.ExecuteNonQueryAsync(ct); + } + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(scratch.ConnectionString, bodySucceeded, async (cleanup, cleanupCt) => + { + await using var probe = new NpgsqlCommand( + "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE datname = pg_catalog.current_database() " + + "AND backend_type LIKE 'TimescaleDB Background Worker Scheduler%'", cleanup); + var schedulers = Convert.ToInt64(await probe.ExecuteScalarAsync(cleanupCt)); + Assert.Equal(0L, schedulers); + }); + } + } + private static async Task InsertQueryStatsAsync( NpgsqlConnection connection, DateTime at, string hash, long workerUs, long executions, int intervalSeconds, CancellationToken ct) {