diff --git a/Darling/Darling.Tests/DarlingRetentionTests.cs b/Darling/Darling.Tests/DarlingRetentionTests.cs index 86e92b96a2..048b593d64 100644 --- a/Darling/Darling.Tests/DarlingRetentionTests.cs +++ b/Darling/Darling.Tests/DarlingRetentionTests.cs @@ -85,7 +85,9 @@ referenced by all three positions. */ "DELETE FROM collection_log WHERE collection_time < $1 AND collection_time >= (SELECT min(collection_time) FROM collection_log WHERE collection_time < $1) AND collection_time < (SELECT min(collection_time) FROM collection_log WHERE collection_time < $1) + INTERVAL '1 days'", DarlingRetention.TimeSlicedDeleteSql("collection_log", "collection_time")); - /* The slice width IS the hypertable chunk width — the fallback's unit of work stays one chunk. */ + /* The slice width matches ChunkIntervalDays, the ladder's CEILING since #4211 (RawChunkIntervalPlanner + can narrow a raw table's actual chunk_time_interval below a day) — the fallback's unit of work stays + bounded to at most one day's worth even where it is no longer exactly one chunk. */ Assert.Equal(1, TimescaleSupport.ChunkIntervalDays); } diff --git a/Darling/Darling.Tests/RawChunkIntervalPlannerTests.cs b/Darling/Darling.Tests/RawChunkIntervalPlannerTests.cs new file mode 100644 index 0000000000..ce315e03c5 --- /dev/null +++ b/Darling/Darling.Tests/RawChunkIntervalPlannerTests.cs @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// Unit tests for (#4211) — pure inputs, no database, no +/// rig. +public sealed class RawChunkIntervalPlannerTests +{ + private static readonly DateTime AsOf = new(2026, 9, 25, 0, 0, 0, DateTimeKind.Utc); + + private const double GiB = 1024.0 * 1024.0 * 1024.0; + private const double GB = 1_000_000_000.0; + + [Fact] + public void Ladder_IsTwentyFourTwelveSix_ReadFromChunkIntervalDays() + { + /* The top rung is read from TimescaleSupport.ChunkIntervalDays (the #4211 ruling's "ladder's + ceiling"), not a second literal 24, so the two can never silently disagree. */ + Assert.Equal(new[] { 24, 12, 6 }, RawChunkIntervalPlanner.RungHours); + Assert.Equal(TimescaleSupport.ChunkIntervalDays * 24, RawChunkIntervalPlanner.RungHours[0]); + Assert.Equal(6, RawChunkIntervalPlanner.FloorHours); + } + + [Fact] + public void SmallStore_NothingMoves() + { + /* A handful of monitored servers: every table's 24-hour open chunk fits comfortably under an 8 GB + budget, so nothing narrows and nothing widens (everything is already at the ceiling). */ + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("wait_stats", 5 * GB / 24, 24, null), + new RawChunkIntervalPlanner.TableInput("query_stats", 3 * GB / 24, 24, null), + new RawChunkIntervalPlanner.TableInput("deadlocks", 0.01 * GB, 24, null), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 8 * GiB, currentTotalChunkCount: 30, AsOf); + + Assert.All(decisions, d => + { + Assert.Equal(24, d.TargetIntervalHours); + Assert.False(d.Changes); + Assert.Contains("fits within budget", d.Reason, StringComparison.Ordinal); + }); + } + + /// Store A's rates from the #4211 design (issuecomment-5827299104): total ingest 2.2 GB/h, of + /// which 1.62 GB/h sits in the three query tables, modeled here as query_stats (0.9), other_raw — standing + /// in for every other raw hypertable in the store, since H3 requires the budget to count them too — (0.58), + /// query_snapshots (0.5) and wait_stats (0.22). Both Facts start from "day 2": every table already + /// narrowed once to 12 h (the most one day's reconcile could do from a 24-hour start against either + /// budget below) and is eligible to move again, so the two host sizes can diverge on THIS call instead of + /// both spending it on the same first rung. + private static RawChunkIntervalPlanner.TableInput[] StoreADay2Tables() => + new[] + { + new RawChunkIntervalPlanner.TableInput("query_stats", 0.9 * GB, 12, AsOf.AddDays(-2)), + new RawChunkIntervalPlanner.TableInput("other_raw", 0.58 * GB, 12, AsOf.AddDays(-2)), + new RawChunkIntervalPlanner.TableInput("query_snapshots", 0.5 * GB, 12, AsOf.AddDays(-2)), + new RawChunkIntervalPlanner.TableInput("wait_stats", 0.22 * GB, 12, AsOf.AddDays(-2)), + }; + + [Fact] + public void StoreA_SixtyThreeGiBHost_NarrowsTheThreeHeaviestAndHoldsTheLightestAtTwelve() + { + var budget = RawChunkIntervalPlanner.ManagedBudgetBytes((long)(63 * GiB)); + var decisions = RawChunkIntervalPlanner.Plan(StoreADay2Tables(), budget, currentTotalChunkCount: 100, AsOf) + .ToDictionary(d => d.TableName, StringComparer.Ordinal); + + /* Heaviest-rate-first narrowing closes the gap after three of the four tables move — the budget is + satisfied before wait_stats, the lightest, is reached. */ + Assert.Equal(6, decisions["query_stats"].TargetIntervalHours); + Assert.Equal(6, decisions["other_raw"].TargetIntervalHours); + Assert.Equal(6, decisions["query_snapshots"].TargetIntervalHours); + Assert.Equal(12, decisions["wait_stats"].TargetIntervalHours); + Assert.False(decisions["wait_stats"].Changes); + + var finalTotal = 6 * 0.9 * GB + 6 * 0.58 * GB + 6 * 0.5 * GB + 12 * 0.22 * GB; + Assert.True(finalTotal <= budget, $"final open-chunk bytes {finalTotal:N0} must fit under budget {budget:N0}"); + } + + [Fact] + public void StoreA_ThirtyOnePointFiveGiBHost_EveryTableReachesTheFloorAndStillDoesNotFit() + { + var budget = RawChunkIntervalPlanner.ManagedBudgetBytes((long)(31.5 * GiB)); + var decisions = RawChunkIntervalPlanner.Plan(StoreADay2Tables(), budget, currentTotalChunkCount: 100, AsOf) + .ToDictionary(d => d.TableName, StringComparer.Ordinal); + + /* The smaller host's budget is too small for this rate even at the 6-hour floor — every table narrows + all the way down, and the planner stops there (there is no rung left to try), leaving the store over + budget. That is the honest outcome, not a bug: review finding H3 on #4211 made exactly this point + about a host this size. */ + Assert.All(decisions.Values, d => Assert.Equal(6, d.TargetIntervalHours)); + + var finalTotal = 6 * (0.9 + 0.58 + 0.5 + 0.22) * GB; + Assert.True(finalTotal > budget, $"expected the floor to still exceed budget {budget:N0}, got {finalTotal:N0}"); + } + + [Fact] + public void OneRungPerDay_RecentlyMovedTableIsHeldEvenWhenOverBudget() + { + var tables = new[] + { + /* Way over any reasonable budget, but changed under an hour ago. */ + new RawChunkIntervalPlanner.TableInput("query_stats", 5 * GB, 24, AsOf.AddHours(-1)), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 1 * GB, currentTotalChunkCount: 10, AsOf); + + var decision = Assert.Single(decisions); + Assert.False(decision.Changes); + Assert.Contains("moved within the last", decision.Reason, StringComparison.Ordinal); + } + + [Fact] + public void OneRungPerDay_TableLastChangedExactlyOneDayAgoIsEligible() + { + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("query_stats", 5 * GB, 24, AsOf.AddDays(-1)), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 1 * GB, currentTotalChunkCount: 10, AsOf); + + var decision = Assert.Single(decisions); + Assert.True(decision.Changes); + Assert.Equal(12, decision.TargetIntervalHours); + } + + [Fact] + public void Hysteresis_WidensWhenStoreTotalStaysAtOrUnderHalfBudget() + { + /* One table, already narrowed to 6 h. Budget is set so the store-wide total after widening lands + EXACTLY on the B/2 line (#4211 ruling, issuecomment-5836734285: "at or under half of B") — the + boundary is inclusive, so this must widen, not hold. */ + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("query_stats", 1 * GB, 6, AsOf.AddDays(-3)), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 24 * GB, currentTotalChunkCount: 10, AsOf) + .ToDictionary(d => d.TableName, StringComparer.Ordinal); + + Assert.Equal(12, decisions["query_stats"].TargetIntervalHours); + Assert.True(decisions["query_stats"].Changes); + Assert.Contains("moved up", decisions["query_stats"].Reason, StringComparison.Ordinal); + } + + [Fact] + public void Hysteresis_HeldWhenStoreTotalFallsInTheBandAboveHalfBudget() + { + /* heavy_table sits at the ceiling and never moves — it is only there to give the store a total large + enough that widening narrowing_table lands in the BAND above B/2 but still under B (246 GB now, + 252 GB if narrowing_table widens; half-budget is 150 GB, budget is 300 GB). Under the ruling this + must hold, not widen. Under the old equal-share rule it would have widened: with two tables, + shareBytes = 150 GB, halfShare = 75 GB, and narrowing_table's OWN widened bytes (12 GB) sit far + under that — the old rule never looked at the store-wide total against half the budget, only this + table's own bytes against its slice, so it missed that the move would leave the store deep in the + band. Confirmed once by hand against the pre-#4211-ruling pass 2: with these inputs it set + narrowing_table's TargetIntervalHours to 12 (Changes = true), where the ruling requires 6 (held). */ + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("heavy_table", 10 * GB, 24, null), + new RawChunkIntervalPlanner.TableInput("narrowing_table", 1 * GB, 6, AsOf.AddDays(-3)), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 300 * GB, currentTotalChunkCount: 10, AsOf) + .ToDictionary(d => d.TableName, StringComparer.Ordinal); + + Assert.Equal(6, decisions["narrowing_table"].TargetIntervalHours); + Assert.False(decisions["narrowing_table"].Changes); + Assert.Contains("over half", decisions["narrowing_table"].Reason, StringComparison.Ordinal); + } + + [Fact] + public void Hysteresis_LighterNarrowedTableWidensBeforeHeavierWhenOnlyOneFitsUnderHalfBudget() + { + /* Both tables are already at 6 h. Only one widen fits under the 50 GB half-budget line, and the + lightest-rate table gets the chance first (#4211 ruling): light_table's move (36 -> 42 GB) fits, + so it goes first and takes the room; by the time heavy_table is considered the store is already at + 42 GB, and its own move would push the total to 72 GB, over the line, so it is held. */ + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("light_table", 1 * GB, 6, AsOf.AddDays(-3)), + new RawChunkIntervalPlanner.TableInput("heavy_table", 5 * GB, 6, AsOf.AddDays(-3)), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 100 * GB, currentTotalChunkCount: 10, AsOf) + .ToDictionary(d => d.TableName, StringComparer.Ordinal); + + Assert.Equal(12, decisions["light_table"].TargetIntervalHours); + Assert.True(decisions["light_table"].Changes); + Assert.Contains("moved up", decisions["light_table"].Reason, StringComparison.Ordinal); + + Assert.Equal(6, decisions["heavy_table"].TargetIntervalHours); + Assert.False(decisions["heavy_table"].Changes); + Assert.Contains("over half", decisions["heavy_table"].Reason, StringComparison.Ordinal); + } + + [Fact] + public void Hysteresis_WidenedStoreIsNotNarrowedByASecondPlanCall() + { + /* Day 1: the table widens from 6 h to 12 h, landing the store total exactly on the B/2 line (as in + Hysteresis_WidensWhenStoreTotalStaysAtOrUnderHalfBudget). Day 2, one day later (eligible again): + feed that 12-hour result back in and confirm it does not flap back down to 6 h. Pass 1 never + considers it (12 GB already fits comfortably under the 24 GB budget), and pass 2 holds it at 12 h + rather than widening again to 24 h (24 GB would be over the 12 GB half-budget line). */ + var dayOne = new[] + { + new RawChunkIntervalPlanner.TableInput("query_stats", 1 * GB, 6, AsOf.AddDays(-3)), + }; + var dayOneDecision = Assert.Single(RawChunkIntervalPlanner.Plan(dayOne, budgetBytes: 24 * GB, currentTotalChunkCount: 10, AsOf)); + Assert.Equal(12, dayOneDecision.TargetIntervalHours); + + var dayTwo = new[] + { + new RawChunkIntervalPlanner.TableInput("query_stats", 1 * GB, dayOneDecision.TargetIntervalHours, AsOf), + }; + var dayTwoDecision = Assert.Single(RawChunkIntervalPlanner.Plan(dayTwo, budgetBytes: 24 * GB, currentTotalChunkCount: 10, AsOf.AddDays(1))); + + Assert.Equal(12, dayTwoDecision.TargetIntervalHours); + Assert.False(dayTwoDecision.Changes); + } + + [Fact] + public void Hysteresis_NeverWidensPastTheCeiling() + { + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("idle_table", 0.001 * GB, 24, AsOf.AddDays(-10)), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 100 * GB, currentTotalChunkCount: 10, AsOf); + + var decision = Assert.Single(decisions); + Assert.Equal(24, decision.TargetIntervalHours); + Assert.False(decision.Changes); + } + + [Fact] + public void ChunkCountCap_BlocksNarrowingEvenWhenOverBudget() + { + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("query_stats", 5 * GB, 24, null), + }; + + var atCap = RawChunkIntervalPlanner.Plan( + tables, budgetBytes: 1 * GB, currentTotalChunkCount: RawChunkIntervalPlanner.ChunkCountCapThreshold, AsOf); + var capped = Assert.Single(atCap); + Assert.False(capped.Changes); + Assert.Contains("cap", capped.Reason, StringComparison.Ordinal); + + /* Same inputs, one chunk under the cap: the move goes through. */ + var underCap = RawChunkIntervalPlanner.Plan( + tables, budgetBytes: 1 * GB, currentTotalChunkCount: RawChunkIntervalPlanner.ChunkCountCapThreshold - 1, AsOf); + var allowed = Assert.Single(underCap); + Assert.True(allowed.Changes); + Assert.Equal(12, allowed.TargetIntervalHours); + } + + [Fact] + public void ChunkCountCap_NeverBlocksWidening() + { + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("idle_table", 0.001 * GB, 6, AsOf.AddDays(-5)), + }; + + var decisions = RawChunkIntervalPlanner.Plan( + tables, budgetBytes: 100 * GB, currentTotalChunkCount: RawChunkIntervalPlanner.ChunkCountCapThreshold + 500, AsOf); + + var decision = Assert.Single(decisions); + Assert.Equal(12, decision.TargetIntervalHours); + Assert.True(decision.Changes); + } + + [Fact] + public void CurrentIntervalOutsideTheLadder_IsLeftUnchanged() + { + /* An adopted bring-your-own store carrying a hand-set 7-day (168-hour) interval from before #4211 — + Plan must not guess a rung for it in either direction. */ + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("legacy_table", 50 * GB, 168, null), + }; + + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 1 * GB, currentTotalChunkCount: 10, AsOf); + + var decision = Assert.Single(decisions); + Assert.Equal(168, decision.TargetIntervalHours); + Assert.False(decision.Changes); + Assert.Contains("outside the", decision.Reason, StringComparison.Ordinal); + } + + [Fact] + public void Plan_ThrowsOnNullTables() + { + Assert.Throws(() => + RawChunkIntervalPlanner.Plan(null!, budgetBytes: 1, currentTotalChunkCount: 0, AsOf)); + } + + [Fact] + public void ManagedBudgetBytes_IsExactlyTwentyFivePercentOfTheRawRamFigure() + { + /* NOT DeriveMemorySettings' SharedBuffersMb output, which is capped at 1 GB (#1559) — this reads the + same raw byte count that method takes as input, per the #4211 ruling. */ + Assert.Equal(4L * GiB, RawChunkIntervalPlanner.ManagedBudgetBytes((long)(16 * GiB))); + Assert.Equal(16 * GiB, RawChunkIntervalPlanner.ManagedBudgetBytes((long)(64 * GiB))); + } + + [Fact] + public void BringYourOwnBudgetBytes_TakesTheLargerOfSharedBuffersAndAThirdOfEffectiveCacheSize() + { + /* Untuned store: stock 128 MB shared_buffers, stock 4 GB effective_cache_size — the 1.33 GB floor + wins, matching review finding H3's fix (not the 128 MB shared_buffers alone, which would call + almost everything "heavy"). */ + var untuned = RawChunkIntervalPlanner.BringYourOwnBudgetBytes( + sharedBuffersBytes: 128 * 1024.0 * 1024.0, effectiveCacheSizeBytes: 4 * GiB); + Assert.Equal((4 * GiB) / 3.0, untuned); + + /* Tuned store: shared_buffers itself raised well past a third of effective_cache_size. */ + var tuned = RawChunkIntervalPlanner.BringYourOwnBudgetBytes( + sharedBuffersBytes: 20 * GiB, effectiveCacheSizeBytes: 48 * GiB); + Assert.Equal(20 * GiB, tuned); + } + + [Fact] + public void Plan_CoversEveryTablesOpenChunkNotOnlyTheHeaviestOnes() + { + /* Finding H3: a light table's 24-hour bytes still count against the budget, so a store where only the + SUM exceeds budget still narrows the heaviest table even though no single table looks "heavy" on + its own. */ + var tables = new[] + { + new RawChunkIntervalPlanner.TableInput("a", 0.3 * GB, 24, null), + new RawChunkIntervalPlanner.TableInput("b", 0.3 * GB, 24, null), + new RawChunkIntervalPlanner.TableInput("c", 0.3 * GB, 24, null), + }; + + /* 24h * 0.9 GB/h combined = 21.6 GB, over a 10 GB budget, even though each table alone is only + 7.2 GB. */ + var decisions = RawChunkIntervalPlanner.Plan(tables, budgetBytes: 10 * GB, currentTotalChunkCount: 10, AsOf) + .ToDictionary(d => d.TableName, StringComparer.Ordinal); + + Assert.True(decisions.Values.Any(d => d.Changes), "expected at least one table to narrow once the combined total exceeded budget"); + } +} diff --git a/Darling/Darling.Tests/TimescaleSupportTests.cs b/Darling/Darling.Tests/TimescaleSupportTests.cs index 0ba1a8db62..69ca6485ad 100644 --- a/Darling/Darling.Tests/TimescaleSupportTests.cs +++ b/Darling/Darling.Tests/TimescaleSupportTests.cs @@ -125,8 +125,10 @@ so a moved grid is loud there rather than silently agreed with here. */ + waitStatsPhase.ToString(CultureInfo.InvariantCulture) + " minutes')", TimescaleSupport.AddCompressionPolicySql(byName["wait_stats"])); - /* 1 day matches the 1-day chunk interval so chunks become compressible quickly, keeping the - managed store compact (#1458). */ + /* 1 day, fixed on every table since #4211 (decision 1: A no longer tracks a narrowed raw table's + chunk_time_interval — RawChunkIntervalPlanner only ever moves I). Originally chosen to match the + 1-day chunk interval so chunks become compressible quickly, keeping the managed store compact + (#1458). */ Assert.Equal(1, TimescaleSupport.CompressAfterDays); /* #1778: schedule_interval is passed EXPLICITLY on every table. Omitting it does not mean "some @@ -156,6 +158,22 @@ managed store compact (#1458). */ TimescaleSupport.AddCompressionPolicySql(TimescaleSupport.CollectionLogTable)); } + /* ---------------- raw compress_after floor (#4211) ---------------- */ + + [Fact] + public void CompressAfterDays_IsAtLeastTheHourlyRefreshStartOffset() + { + /* #4211 ruling decision 1: a raw table's compress_after (A) stays fixed regardless of how far + RawChunkIntervalPlanner narrows that table's own chunk_time_interval (I). The reason is this + invariant — A below HourlyRefreshStartOffset would let a chunk compress before the hourly + continuous-aggregate refresh has finished reading it uncompressed, removing the gap the product's + lock safety relies on. Pinned as a span comparison, not a re-parse of the "1 day" strings, so it + breaks loudly if either side ever moves without the other. */ + Assert.True( + TimeSpan.FromDays(TimescaleSupport.CompressAfterDays) >= TimescaleSupport.HourlyRefreshStartSpan, + $"CompressAfterDays ({TimescaleSupport.CompressAfterDays} days) must stay at least HourlyRefreshStartOffset ({TimescaleSupport.HourlyRefreshStartSpan})."); + } + /* ---------------- compression-job self-heal (#1581) — pure predicate ---------------- */ private static readonly DateTime s_now = new(2026, 7, 19, 12, 0, 0, DateTimeKind.Utc); diff --git a/Darling/PerformanceMonitor.Darling.Storage/EventWindowFloor.cs b/Darling/PerformanceMonitor.Darling.Storage/EventWindowFloor.cs index bad186968e..c42c95c7f5 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/EventWindowFloor.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/EventWindowFloor.cs @@ -25,9 +25,9 @@ namespace PerformanceMonitor.Darling.Storage; /// clock, which stamps the event, and the collector host's, which stamps the collection. Measured on the same /// store: blocked-process reports land 4.8 s to 62 s after their event_time, a DMV blocking snapshot's /// event_time IS its collection_time, and the worst deadlock sat 27 ms AFTER its collection. -/// is one whole chunk — a day — so a monitored clock would have to run a day -/// ahead of the collector's before a row fell out, and the floor costs at most one chunk more than the window -/// itself while still excluding every chunk older than that. +/// is one day — a day — so a monitored clock would have to run a day ahead of the +/// collector's before a row fell out, and the floor costs at most one day more than the window itself while +/// still excluding every chunk older than that. /// /// Deliberately no upper bound on collection_time. An event collected late — the catch-up /// after an outage — is still an event inside the window, and it is exactly the row an upper bound would @@ -35,10 +35,15 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class EventWindowFloor { - /// How far before an event's own timestamp its collection_time may fall and still be read: - /// the store's chunk width, derived from it so "at most one chunk beyond the window" stays true if the - /// width ever moves. - public static readonly TimeSpan SkewAllowance = TimeSpan.FromDays(TimescaleSupport.ChunkIntervalDays); + /// How far before an event's own timestamp its collection_time may fall and still be + /// read. Its OWN 1-day constant (#4211) — a clock-skew tolerance, not the store's chunk width: since + /// #4211, a raw hypertable's chunk_time_interval can be narrower than a day for the heaviest + /// tables (), while + /// stays only the ladder's CEILING. blocked_process_reports, dmv_blocking_snapshots and + /// deadlocks — the three tables this floor exists for — are not among the heavy query tables the + /// planner narrows, so they stay at the 24-hour ceiling regardless, and a day of allowance still costs at + /// most one of their chunks. + public static readonly TimeSpan SkewAllowance = TimeSpan.FromDays(1); /// The collection_time floor for an event window starting at /// , as naive UTC — the store's timestamp convention, so the bind diff --git a/Darling/PerformanceMonitor.Darling.Storage/RawChunkIntervalPlanner.cs b/Darling/PerformanceMonitor.Darling.Storage/RawChunkIntervalPlanner.cs new file mode 100644 index 0000000000..67cc83501d --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/RawChunkIntervalPlanner.cs @@ -0,0 +1,312 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// Derives each RAW hypertable's chunk_time_interval from its own ingest rate and a RAM budget +/// (#4211) — PURE, so it is unit-testable with no database: every input is a plain value, and the only +/// output is a target interval and a one-line reason per table. +/// is NOT one of its outputs: the #4211 ruling (issuecomment-5836205190, decision 1) keeps every table's +/// compress_after fixed at 1 day regardless of its interval — a shorter delay below the hourly refresh +/// window removes the gap the product's lock safety relies on, and that needs its own design. Only +/// chunk_time_interval (I) moves here. Reading live rates and catalog state, calling +/// set_chunk_time_interval, and storing the rung history are a separate change (#4211's second half); +/// this class only decides WHAT to call it with. +/// +/// The ladder. Three rungs only: 24 h ( — the +/// ceiling every new hypertable still starts at), 12 h, and 6 h (the floor). Rungs under 6 h stay out until +/// the reads that plan every chunk of a table because they carry no time bound — the Query Store backfill's +/// candidate read among them — are bounded (#4211 review, finding M2). +/// +/// The invariant tries to hold. The sum, over EVERY raw hypertable passed +/// in — not only the ones that move — of its ingest rate times its OWN interval must stay at or under the +/// budget B. That is what one open chunk of that table costs while it is still being written, indexes +/// included (#4211 review, finding H3: a budget that counts only the "heavy" tables misses the open chunks +/// every OTHER table is also holding at the same time). B itself comes from +/// or — this class never computes it from a live RAM or setting +/// read. +/// +/// Which tables move. When the sum exceeds B, tables narrow one rung at a time, heaviest RATE +/// first — never a fixed "B/20 is heavy" threshold (#4211 ruling decision 3) — until the sum fits, every +/// remaining table has reached the floor, or refuses the next move. A +/// rate tie breaks on table name, ordinally, so the order is deterministic. A table changed within the last +/// day is left exactly as it is — neither narrowed nor widened — +/// because is meant to run once a day and this is what keeps two calls (a restart loop +/// among them) from moving the same table twice. +/// +/// Moving back up. Considered only for a table the narrowing pass above did not touch today, +/// and only when narrower than the ceiling. A table moves back up one rung only when the store-wide open-chunk +/// total, with that move made, stays at or under half of B (#4211 ruling, issuecomment-5836734285). Narrowing +/// starts above B and widening needs the total under B/2, so a store sitting near the line never flaps. The +/// lightest-rate table is considered first, so the moves that add the fewest bytes go first; a rate tie breaks +/// on the table name, as in the narrowing pass. +/// +/// The chunk-count cap. TimescaleDB's own guidance treats maintaining over ~1,000 chunks on one +/// hypertable as an anti-pattern. This class has no per-table retention input to forecast an exact per-table +/// count (a database read, out of scope here), so it takes the store's CURRENT total chunk count, across +/// every hypertable, as a single input and refuses every narrowing move once that total is at or past +/// — a coarse, store-wide brake, not a per-table forecast, deliberately +/// on the safe side of "about 1,000" (#4211 ruling decision 3). Widening is never blocked by the cap: it can +/// only reduce a table's future chunk count. +/// +/// A current interval outside the ladder. An adopted bring-your-own store can carry a hypertable +/// whose chunk_time_interval was set by hand to something other than 24, 12 or 6 hours before this +/// rule existed. does not guess a safe step for it in either direction — it reports the +/// table unchanged, with a reason saying so, rather than silently snapping it onto the nearest rung. +/// +public static class RawChunkIntervalPlanner +{ + /// The rungs, widest first. The top rung is read from + /// — decision 9 of the #4211 ruling makes that constant + /// the ladder's ceiling — rather than restated as a second literal 24. + public static readonly IReadOnlyList RungHours = new[] { TimescaleSupport.ChunkIntervalDays * 24, 12, 6 }; + + /// The narrowest interval a raw hypertable may reach under this rule (#4211 ruling decision 4; + /// review finding M2 — the unbounded reads that would plan every chunk of a table narrower than this are + /// not yet fixed). + public const int FloorHours = 6; + + /// A table may move at most one rung per this many days, so a restart loop cannot move it twice + /// and a manual override survives at least this long before revisits it (#4211 ruling + /// decision 4). + public const int MinimumDaysBetweenMoves = 1; + + /// TimescaleDB's own docs call maintaining over 1,000 chunks per hypertable an anti-pattern. + /// stops narrowing at this store-wide total — on the safe side of that line, since it + /// has no per-table retention input to forecast an exact per-table count (#4211 ruling decision 3). + public const long ChunkCountCapThreshold = 1_000; + + /// One raw hypertable's planning inputs, all plain values so takes no + /// database dependency. + /// The hypertable's name — used only for the reason text and as the deterministic + /// tie-break when two tables share a rate. + /// Bytes written per hour, heap AND indexes together — what + /// chunk_compression_stats() reports for a closed chunk, not a heap-only figure (#4211 review + /// finding H3). + /// The table's chunk_time_interval today. Normally one of + /// ; see the class remarks for a value outside the ladder. + /// When the interval last moved a rung under this rule, or + /// null if it never has — treated as eligible to move today. + public readonly record struct TableInput( + string TableName, + double IngestBytesPerHour, + int CurrentIntervalHours, + DateTime? IntervalLastChangedUtc); + + /// One table's decision: stay, narrow, or widen, with the reason a log line can carry + /// verbatim. + public readonly record struct Decision( + string TableName, + int CurrentIntervalHours, + int TargetIntervalHours, + string Reason) + { + /// true when differs from + /// — the caller's cue to actually call + /// set_chunk_time_interval and record the move. + public bool Changes => TargetIntervalHours != CurrentIntervalHours; + } + + /// + /// 25% of — the managed-store budget B (#4211 ruling + /// decision 2), taken from the SAME raw RAM figure DarlingManagedPostgres.DeriveMemorySettings + /// receives (that type lives in the Service project, which this one does not reference, hence no + /// cref here), not that method's SharedBuffersMb OUTPUT: that output is capped at 1 GB for + /// the co-located-store / Windows 487 mitigation (#1559), which is a real shared_buffers setting, + /// not a ceiling on how much RAM chunk sizing may plan against. No fallback for a non-positive input: the + /// caller already applies whatever fallback DeriveMemorySettings itself uses before this figure + /// reaches here — an input error should not silently produce a plausible-looking budget from this + /// class. + /// + public static double ManagedBudgetBytes(long totalPhysicalMemoryBytes) => + totalPhysicalMemoryBytes * 0.25; + + /// + /// max(shared_buffers, effective_cache_size / 3) — the bring-your-own-store budget B (#4211 ruling + /// decision 2, replacing the design's plain shared_buffers after review finding H3: PostgreSQL's + /// default shared_buffers is 128 MB, which would call almost every table "heavy"). A tuned store + /// sets effective_cache_size near 75% of RAM, so B lands near 25% of RAM, matching + /// ; an untuned store's 4 GB default effective_cache_size still + /// gives a 1.3 GB floor instead of 128 MB. + /// + public static double BringYourOwnBudgetBytes(double sharedBuffersBytes, double effectiveCacheSizeBytes) => + Math.Max(sharedBuffersBytes, effectiveCacheSizeBytes / 3.0); + + /// + /// Plans at most one rung move for every table in . Deterministic: the same + /// inputs always produce the same decisions, returned in the same order was + /// given. + /// + /// Every RAW hypertable the store is deriving an interval for. Must include tables + /// that will not move — their current bytes still count against (#4211 + /// review finding H3). + /// B, from or + /// . + /// The store's current chunk count across every hypertable, checked + /// against before any narrowing move. + /// "Now", for the check — passed in rather + /// than read from the clock so a test can hold it fixed. + public static IReadOnlyList Plan( + IReadOnlyList tables, + double budgetBytes, + long currentTotalChunkCount, + DateTime asOfUtc) + { + if (tables is null) + { + throw new ArgumentNullException(nameof(tables)); + } + + var current = new Dictionary(StringComparer.Ordinal); + foreach (var table in tables) + { + current[table.TableName] = table.CurrentIntervalHours; + } + + double TotalBytes() + { + var sum = 0.0; + foreach (var table in tables) + { + sum += table.IngestBytesPerHour * current[table.TableName]; + } + + return sum; + } + + static int RungIndex(int hours) + { + for (var i = 0; i < RungHours.Count; i++) + { + if (RungHours[i] == hours) + { + return i; + } + } + + return -1; + } + + bool EligibleToday(TableInput table) => + table.IntervalLastChangedUtc is not DateTime last + || (asOfUtc - last).TotalDays >= MinimumDaysBetweenMoves; + + var decisions = new Dictionary(StringComparer.Ordinal); + + /* Tables outside the ladder, or already moved within the day limit, are decided up front and taken + out of both passes below — see the class remarks for why neither pass may touch them. */ + foreach (var table in tables) + { + if (RungIndex(table.CurrentIntervalHours) < 0) + { + decisions[table.TableName] = new Decision( + table.TableName, table.CurrentIntervalHours, table.CurrentIntervalHours, + $"held at {table.CurrentIntervalHours} h: outside the {string.Join("/", RungHours)} h ladder, no safe step known"); + } + else if (!EligibleToday(table)) + { + decisions[table.TableName] = new Decision( + table.TableName, table.CurrentIntervalHours, table.CurrentIntervalHours, + $"held at {table.CurrentIntervalHours} h: moved within the last {MinimumDaysBetweenMoves} day(s)"); + } + } + + var chunkCapReached = currentTotalChunkCount >= ChunkCountCapThreshold; + + /* Pass 1 — narrow, heaviest rate first, one rung per table, only while the store-wide total still + exceeds budget at that table's turn. A table visited while the total already fits is left for + pass 2 to consider for widening instead. */ + var byRateDescending = tables + .Where(t => !decisions.ContainsKey(t.TableName)) + .OrderByDescending(t => t.IngestBytesPerHour) + .ThenBy(t => t.TableName, StringComparer.Ordinal); + + foreach (var table in byRateDescending) + { + if (TotalBytes() <= budgetBytes) + { + continue; + } + + var rungIndex = RungIndex(current[table.TableName]); + if (rungIndex == RungHours.Count - 1) + { + continue; /* already at the floor — pass 2 may still consider widening it */ + } + + if (chunkCapReached) + { + decisions[table.TableName] = new Decision( + table.TableName, table.CurrentIntervalHours, table.CurrentIntervalHours, + $"held at {table.CurrentIntervalHours} h: store chunk count {currentTotalChunkCount:N0} is at or past the {ChunkCountCapThreshold:N0} cap"); + continue; + } + + var narrower = RungHours[rungIndex + 1]; + current[table.TableName] = narrower; + decisions[table.TableName] = new Decision( + table.TableName, table.CurrentIntervalHours, narrower, + $"moved to {narrower} h: store-wide open-chunk bytes exceeded the {budgetBytes:N0} B budget (rate-ordered, {table.IngestBytesPerHour:N0} B/h)"); + } + + /* Pass 2 — widen, hysteresis-guarded (#4211 ruling, issuecomment-5836734285): a table moves back up one + rung only when the store-wide open-chunk total, with that move made, stays at or under half of B. + Narrowing starts above B and widening needs the total under B/2, so a store near the line never flaps. + Lightest rate first, so the moves that add the fewest bytes go first; a rate tie breaks on the table + name, as in pass 1. */ + var halfBudget = budgetBytes / 2.0; + foreach (var table in tables + .OrderBy(t => t.IngestBytesPerHour) + .ThenBy(t => t.TableName, StringComparer.Ordinal)) + { + if (decisions.ContainsKey(table.TableName)) + { + continue; + } + + var rungIndex = RungIndex(current[table.TableName]); + if (rungIndex == 0) + { + continue; /* already at the ceiling; falls through to the default reason below */ + } + + var wider = RungHours[rungIndex - 1]; + var widenedTotal = TotalBytes() - (table.IngestBytesPerHour * current[table.TableName]) + + (table.IngestBytesPerHour * wider); + if (widenedTotal > halfBudget) + { + decisions[table.TableName] = new Decision( + table.TableName, table.CurrentIntervalHours, table.CurrentIntervalHours, + $"held at {table.CurrentIntervalHours} h: at {wider} h the store would hold {widenedTotal:N0} B, over half the {budgetBytes:N0} B budget"); + continue; + } + + current[table.TableName] = wider; + decisions[table.TableName] = new Decision( + table.TableName, table.CurrentIntervalHours, wider, + $"moved up to {wider} h: the store holds {widenedTotal:N0} B, under half the {budgetBytes:N0} B budget"); + } + + var results = new List(tables.Count); + foreach (var table in tables) + { + results.Add(decisions.TryGetValue(table.TableName, out var decision) + ? decision + : new Decision( + table.TableName, table.CurrentIntervalHours, table.CurrentIntervalHours, + $"unchanged at {table.CurrentIntervalHours} h: fits within budget")); + } + + return results; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/RollupBackfill.cs b/Darling/PerformanceMonitor.Darling.Storage/RollupBackfill.cs index ef1dc42192..5b79dc36bb 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/RollupBackfill.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/RollupBackfill.cs @@ -55,12 +55,13 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class RollupBackfill { - /// How much history one refresh call covers. Deliberately ONE source chunk - /// (): the slice's read set is then a single chunk, which - /// is what keeps the lock window short enough not to sit across a compression job (#1778). Slices are - /// midnight-aligned, so they always land on bucket boundaries for both the hourly and daily grains — an - /// unaligned window would be silently widened to bucket boundaries by TimescaleDB anyway, making progress - /// arithmetic lie. + /// How much history one refresh call covers. Deliberately + /// wide — the ladder's CEILING since #4211, not every source table's actual chunk width: on a table + /// has narrowed, one slice now spans at most a few of that table's + /// chunks rather than exactly one, but it is still bounded to at most a day, which is what keeps the lock + /// window short enough not to sit across a compression job (#1778). Slices are midnight-aligned, so they + /// always land on bucket boundaries for both the hourly and daily grains — an unaligned window would be + /// silently widened to bucket boundaries by TimescaleDB anyway, making progress arithmetic lie. public static readonly TimeSpan SliceWidth = TimeSpan.FromDays(TimescaleSupport.ChunkIntervalDays); /// Measured compression throughput on the field host class this verb exists for (~16 MB/s). Used diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index 84c1cfda58..ab83967384 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -60,12 +60,20 @@ public static partial class TimescaleSupport /// Compress chunks older than this many days — hardcoded (defaults over speculative config). /// Compressed chunks remain fully queryable, just columnar and ~10-20x smaller: this IS /// Darling's archival tier, the centralized-store answer to Lite's parquet archive, keeping the - /// full retention horizon cheap instead of splitting hot/cold stores. Kept short (1 day) to - /// match : at the collectors' 1-minute cadence a longer lag left - /// the whole store uncompressed (a chunk cannot compress until it closes AND then ages past - /// this), so even a near-idle fleet grew ~1 GB in a couple of days of hot data. Collectors only - /// ever append current-time rows, so a day-old chunk never takes another write — safe to - /// compress. Measured on this data: perfmon ~16.7x, plan-XML-heavy query_stats ~6.4x. + /// full retention horizon cheap instead of splitting hot/cold stores. Kept short (1 day): + /// at the collectors' 1-minute cadence a longer lag left the whole store uncompressed (a chunk + /// cannot compress until it closes AND then ages past this), so even a near-idle fleet grew + /// ~1 GB in a couple of days of hot data. Collectors only ever append current-time rows, so a + /// day-old chunk never takes another write — safe to compress. Measured on this data: perfmon + /// ~16.7x, plan-XML-heavy query_stats ~6.4x. + /// + /// Fixed at 1 day on EVERY table, even where narrows a raw + /// table's chunk_time_interval below a day (#4211). This is the ladder's CEILING now, not "the" + /// delay tied one-for-one to chunk width: the #4211 ruling (issuecomment-5836205190, decision 1) keeps + /// compress_after here regardless of I, because a shorter delay below + /// removes the gap the product's lock safety relies on, and + /// lowering it needs its own design built from field reads. Pinned at least + /// by TimescaleSupportTests for exactly that reason. /// public const int CompressAfterDays = 1; @@ -108,11 +116,22 @@ public static partial class TimescaleSupport public static readonly TimeSpan CompressScheduleSpan = TimeSpan.FromHours(1); /// - /// Hypertable chunk width in days. TimescaleDB's 7-day default is far too coarse for - /// 1-minute-cadence monitoring data: a chunk stays open (and uncompressible) for its whole - /// span, so 7-day chunks meant nothing compressed for ~2 weeks. 1-day chunks close daily and - /// become compressible within , keeping the store compact. - /// Applies at hypertable creation (fresh stores); existing chunks keep their original width. + /// Hypertable chunk width in days at CREATION (fresh stores; existing chunks keep their original width). + /// TimescaleDB's 7-day default is far too coarse for 1-minute-cadence monitoring data: a chunk stays open + /// (and uncompressible) for its whole span, so 7-day chunks meant nothing compressed for ~2 weeks. 1-day + /// chunks close daily and become compressible within , keeping the store + /// compact. + /// + /// Since #4211, this is the ladder's CEILING, not every raw table's actual width. + /// derives a narrower chunk_time_interval — 12 or 6 hours — + /// for whichever raw hypertables' own ingest rate needs it to keep the store's open-chunk bytes under a + /// RAM budget, using this constant only as the widest rung. Every OTHER dependent still reads this as a + /// safe upper bound / margin rather than an exact width, and stays correct as chunks narrow: + /// DarlingDimensionGcBound, QueryStorePlanMap.MarginOrderingHolds, + /// RollupBackfill.SliceWidth and the aggregate compress margin. + /// does NOT follow the derived width down — it stays fixed at 1 day on every table (#4211 ruling decision + /// 1). has its own 1-day constant rather than reading this + /// one, since it is a clock-skew tolerance that only happens to equal this value today. /// public const int ChunkIntervalDays = 1;