diff --git a/Darling/Darling.Tests/QueryStoreIntervalBrinLiveTests.cs b/Darling/Darling.Tests/QueryStoreIntervalBrinLiveTests.cs new file mode 100644 index 0000000000..dcb5dd5be3 --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreIntervalBrinLiveTests.cs @@ -0,0 +1,655 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Npgsql; +using PerformanceMonitor.Darling.Storage; +using Xunit; +using static Darling.Tests.QueryStoreIntervalRetentionTestKit; + +namespace Darling.Tests; + +/// +/// #5594: BRIN autosummarize on the Query Store interval tables must not cancel autovacuum. A BRIN summarize work item waits for +/// the table's SHARE UPDATE EXCLUSIVE lock, and after deadlock_timeout PostgreSQL cancels the VACUUM that holds it. Every BRIN +/// leaf is off (an index an earlier build made with on included), and the hourly summarize that replaces the work items +/// never waits as long as deadlock_timeout for a lock. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Every test reaches DARLING_TEST_PG only to + CREATE and DROP its own database through ScratchPostgres and works entirely inside it, so it cannot race + live collection. */ +public sealed class QueryStoreIntervalBrinLiveTests +{ + private const string Brin = "ix_query_store_interval_wide_collection_time_brin"; + + /* BRIN leaf indexes (relkind i; the partitioned parent index is relkind I and cannot be altered) of the wide table's family. */ + private const string LeafBrinFrom = + "FROM pg_index x JOIN pg_class i ON i.oid = x.indexrelid JOIN pg_am am ON am.oid = i.relam " + + "WHERE am.amname = 'brin' AND i.relkind = 'i' " + + "AND (x.indrelid = 'collect.query_store_interval_wide'::regclass " + + "OR x.indrelid IN (SELECT relid FROM pg_partition_tree('collect.query_store_interval_wide'::regclass)))"; + + private const string LeafBrinCountSql = "SELECT count(*) " + LeafBrinFrom; + + private const string LeafBrinOnCountSql = "SELECT count(*) " + LeafBrinFrom + " AND coalesce(i.reloptions @> ARRAY['autosummarize=on'], false)"; + + /* Enough rows for several COMPLETE 128-page ranges after the index exists: a handful of rows fills less than one range, and + the last, partial range is summarized too (brin_summarize_range passes include_partial), so the count would say little. */ + private static Task InsertManyAsync(NpgsqlConnection connection, CancellationToken ct) => + ExecAsync( + connection, + "INSERT INTO collect.query_store_interval_wide (collection_time, server_id, database_name, query_id, plan_id, execution_type_desc, " + + "first_execution_time, last_execution_time, query_text, execution_count, avg_duration_us, runtime_stats_interval_id, interval_start_time_utc) " + + "SELECT t, 1, 'db', g, g, 'Regular', t, t, repeat('select 1 ', 20), 1, 1000, 1, t " + + "FROM (SELECT g, timestamp '2026-10-08 12:00:00' + g * interval '1 second' AS t FROM generate_series(1, 30000) AS g) AS s", + ct); + + [Theory] + [InlineData(1000, 500)] + [InlineData(200, 100)] + [InlineData(199, 0)] + [InlineData(1, 0)] + [InlineData(5000, 2000)] + [InlineData(60000, 2000)] + public void TheSummarizeLockWait_IsAlwaysStrictlyBelowDeadlockTimeout_OrTheStepDoesNotRun(int deadlockMs, int expectedMs) + { + var lockMs = QueryStoreIntervalBrin.SummarizeLockTimeoutMs(deadlockMs); + Assert.Equal(expectedMs, lockMs); + Assert.True(lockMs == 0 || lockMs < deadlockMs); + } + + [Fact] + public async Task AnUpgradedStoreWithAutosummarizeOn_EndsUpOff_OnTheLegacyTable_EveryDayLeafAndDefault() + { + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + + /* The state a pre-#5594 store is in after V171: #4862's legacy index with autosummarize on, attached to a parent index that + was made with it on. */ + await ExecAsync(connection, $"CREATE INDEX {Brin}_legacy ON collect.query_store_interval_wide_legacy USING brin (collection_time) WITH (autosummarize = on)", ct); + await ExecAsync(connection, $"CREATE INDEX {Brin} ON ONLY collect.query_store_interval_wide USING brin (collection_time) WITH (autosummarize = on)", ct); + await ExecAsync(connection, $"ALTER INDEX collect.{Brin} ATTACH PARTITION collect.{Brin}_legacy", ct); + Assert.Equal(1L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + + /* The hourly step alone (the start path runs it before the collectors): the legacy leaf goes off. */ + var turnedOff = await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, Wide, logger, ct); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.Done, turnedOff.Outcome); + Assert.Equal(1, turnedOff.Count); + Assert.Equal(0L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.NothingToDo, (await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, Wide, logger, ct)).Outcome); + + /* The partitioned parent index cannot be altered, so it keeps cloning "on" into every leaf the next steps make. */ + Assert.Equal(1L, await CountAsync(connection, $"SELECT count(*) FROM pg_class WHERE oid = 'collect.{Brin}'::regclass AND relkind = 'I' AND reloptions @> ARRAY['autosummarize=on']", ct)); + + /* Promotion makes the day partitions and DEFAULT by PARTITION OF, each cloning the parent's option: all off. */ + await PromoteAsync(connection, Wide, Now, logger, ct); + var leaves = await CountAsync(connection, LeafBrinCountSql, ct); + Assert.True(leaves >= 4, $"expected the legacy table, DEFAULT and the day partitions to have a BRIN each; found {leaves}"); + Assert.Equal(0L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + + /* Create-ahead makes the next days by CREATE TABLE + ATTACH, which also clones: off, in the same transaction. */ + var ahead = await QueryStoreIntervalPartitions.CreateAheadAsync(connection, Wide, Now.AddDays(3), logger, ct); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.Done, ahead.Outcome); + Assert.True(await CountAsync(connection, LeafBrinCountSql, ct) > leaves); + Assert.Equal(0L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + + /* An index somebody turned back on (or a leaf some other path made) is turned off by the next pass, and the pass counts it. */ + await ExecAsync(connection, $"ALTER INDEX collect.{Brin}_legacy SET (autosummarize = on)", ct); + Assert.Equal(1L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + var pass = await QueryStoreIntervalPartitions.RunMaintenancePassAsync(connection, Now.AddDays(3), logger, ct); + Assert.Equal(0, pass.Failed); + Assert.True(pass.Changed >= 1); + Assert.Equal(0L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + } + + [Fact] + public async Task AFreshStoresIndexBuild_AndItsLeaves_AreOffFromTheStart() + { + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + + await QueryStoreIntervalWideBrinIndex.EnsureAsync(connection, logger, ct); + await PromoteAsync(connection, Wide, Now, logger, ct); + await QueryStoreIntervalPartitions.CreateAheadAsync(connection, Wide, Now.AddDays(3), logger, ct); + + Assert.True(await CountAsync(connection, LeafBrinCountSql, ct) >= 4); + Assert.Equal(0L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + Assert.Equal(0L, await CountAsync(connection, $"SELECT count(*) FROM pg_class WHERE oid = 'collect.{Brin}'::regclass AND reloptions @> ARRAY['autosummarize=on']", ct)); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.NothingToDo, (await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, Wide, logger, ct)).Outcome); + } + + [Fact] + public async Task TheSummarize_UnderAHeldShareUpdateExclusiveLock_GivesUpBelowDeadlockTimeout_AndNeverShowsAWaiterThatLong() + { + /* A SHARE UPDATE EXCLUSIVE lock stands in for a running autovacuum. The summarize asks for the same lock, so if it waited + deadlock_timeout PostgreSQL would cancel the real autovacuum. It must give up first, and say so. */ + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + await using var blocker = await OpenStoreAsync(scratch, ct); + await using var watcher = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + + await ExecAsync(connection, $"CREATE INDEX {Brin}_legacy ON collect.query_store_interval_wide_legacy USING brin (collection_time) WITH (autosummarize = off)", ct); + await InsertManyAsync(connection, ct); + + var deadlockMs = (int)await CountAsync(connection, "SELECT setting::bigint FROM pg_settings WHERE name = 'deadlock_timeout'", ct); + var longestWaiter = TimeSpan.Zero; + var stop = false; + QueryStoreIntervalPartitions.StepResult held; + var elapsed = TimeSpan.Zero; + Task poll; + await using (var hold = await blocker.BeginTransactionAsync(ct)) + { + await ExecAsync(blocker, "LOCK TABLE collect.query_store_interval_wide_legacy IN SHARE UPDATE EXCLUSIVE MODE", ct); + poll = Task.Run( + async () => + { + var since = (Stopwatch?)null; + while (!Volatile.Read(ref stop)) + { + await using var command = new NpgsqlCommand( + "SELECT count(*) FROM pg_locks WHERE NOT granted AND relation = 'collect.query_store_interval_wide_legacy'::regclass;", watcher); + var waiting = (long)(await command.ExecuteScalarAsync(ct))! > 0; + since = waiting ? since ?? Stopwatch.StartNew() : null; + if (since is not null && since.Elapsed > longestWaiter) + { + longestWaiter = since.Elapsed; + } + + await Task.Delay(10, ct); + } + }, + ct); + + var watch = Stopwatch.StartNew(); + held = await QueryStoreIntervalBrin.SummarizeNewRangesAsync(connection, Wide, logger, ct); + elapsed = watch.Elapsed; + Volatile.Write(ref stop, true); + await poll; + await hold.RollbackAsync(ct); + } + + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.RetryLater, held.Outcome); + Assert.Contains("without the lock", held.Detail); + Assert.True(elapsed < TimeSpan.FromMilliseconds(deadlockMs), $"the summarize took {elapsed.TotalMilliseconds:F0} ms against deadlock_timeout {deadlockMs} ms"); + Assert.True(longestWaiter < TimeSpan.FromMilliseconds(deadlockMs), $"pg_locks showed a waiter for {longestWaiter.TotalMilliseconds:F0} ms against deadlock_timeout {deadlockMs} ms"); + + /* Once the lock is gone the same call summarizes the range the insert left unsummarized, and the next call finds nothing. */ + var after = await QueryStoreIntervalBrin.SummarizeNewRangesAsync(connection, Wide, logger, ct); + Assert.True(after.Outcome == QueryStoreIntervalPartitions.StepOutcome.Done, $"{after.Outcome}: {after.Detail}"); + Assert.True(after.Count >= 1); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.NothingToDo, (await QueryStoreIntervalBrin.SummarizeNewRangesAsync(connection, Wide, logger, ct)).Outcome); + } + + [Fact] + public async Task TheSummarize_WithADeadlockTimeoutTooShortToWaitBelow_DoesNotRun() + { + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + await ExecAsync(connection, $"CREATE INDEX {Brin}_legacy ON collect.query_store_interval_wide_legacy USING brin (collection_time) WITH (autosummarize = off)", ct); + await InsertAsync(connection, Wide, Now.AddHours(-2), 1, ct); + + await ExecAsync(connection, "SET deadlock_timeout = '100ms'", ct); + var step = await QueryStoreIntervalBrin.SummarizeNewRangesAsync(connection, Wide, new ListLogger(), ct); + + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.NotReady, step.Outcome); + Assert.Contains("deadlock_timeout is 100 ms", step.Detail); + } + + private const string LegacyTable = "collect.query_store_interval_wide_legacy"; + + private const string LegacyBrin = "collect." + Brin + "_legacy"; + + [Theory] + [InlineData(1000, 2000)] + [InlineData(200, 1200)] + [InlineData(1, 1001)] + [InlineData(0, 1001)] + [InlineData(4000, 5000)] + [InlineData(60000, 5000)] + [InlineData(int.MaxValue, 5000)] + public void TheTurnOffLockWait_IsDeadlockTimeoutPlusASecond_ButNeverMoreThanFiveSeconds(int deadlockMs, int expectedMs) + { + Assert.Equal(expectedMs, QueryStoreIntervalBrin.TurnOffLockTimeoutMs(deadlockMs)); + } + + /// The statement shapes #5594 review round 1 depends on: a text pin, so a rewrite back to the old shape fails here without a server. + [Fact] + public void TheStatementShapes_StopBetweenRanges_AndCountEveryVacuumThatCannotBeCancelled() + { + /* brin_summarize_new_values can only be cancelled, and a cancel strands a placeholder range that no later pass summarizes. */ + Assert.Contains("brin_summarize_range", QueryStoreIntervalBrin.SummarizeIndexSql); + Assert.Contains("clock_timestamp()", QueryStoreIntervalBrin.SummarizeIndexSql); + Assert.DoesNotContain("brin_summarize_new_values", QueryStoreIntervalBrin.SummarizeIndexSql); + + var holders = QueryStoreIntervalBrin.AutosummarizeOnSql; + Assert.Contains("pg_stat_progress_vacuum", holders); + Assert.Contains("autovacuum_freeze_max_age", holders); + Assert.Contains("autovacuum_multixact_freeze_max_age", holders); + Assert.Contains("autovacuum worker", holders); + + /* #5594 round 2 L1: the activity text autovacuum writes for an anti-wraparound vacuum is a signal of its own, and the age + limit is the smaller of the table's setting and the server's (LEAST), never the table's alone (COALESCE). */ + Assert.Contains("(to prevent wraparound)", holders); + Assert.Contains(QueryStoreIntervalBrin.AntiWraparoundAgeSql, holders); + Assert.Contains("LEAST", QueryStoreIntervalBrin.AntiWraparoundAgeSql); + Assert.DoesNotContain("COALESCE", QueryStoreIntervalBrin.AntiWraparoundAgeSql); + } + + /// + /// #5594 round 2 L2: brin_summarize_range raises XX000 for an index that is gone, so XX000 plus an index that no longer resolves + /// is a drop; XX000 for an index that is still there, or any other error, is a fault that must surface. + /// + [Theory] + [InlineData("XX000", false, true)] + [InlineData("42P01", false, true)] + [InlineData("XX000", true, false)] + [InlineData("42P01", true, false)] + [InlineData("55P03", false, false)] + [InlineData("57014", false, false)] + [InlineData(null, false, false)] + public void ASummarizeError_CountsAsADroppedIndex_OnlyWhenTheIndexIsGone(string? sqlState, bool indexStillExists, bool expectedDropped) + { + Assert.Equal(expectedDropped, QueryStoreIntervalBrin.IsDroppedIndexError(sqlState, indexStillExists)); + } + + /// V171's parent BRIN says off, so a leaf made by PARTITION OF or ATTACH clones off (#5594 review round 1). + [Fact] + public void TheWideRung_CreatesTheParentBrinWithAutosummarizeOff() + { + var rung = PgMigrations.Scripts.Single(m => m.Version == QueryStoreIntervalPartitionRungTests.WideRungVersion); + Assert.Contains("USING brin (collection_time) WITH (autosummarize = off)", rung.Sql); + Assert.DoesNotContain("autosummarize = on", rung.Sql); + } + + /// A second session that watches pg_locks for a request on one relation that is not granted, and how long it stayed ungranted. + private sealed class WaiterWatch : IAsyncDisposable + { + private readonly Task _poll; + private bool _stop; + + public WaiterWatch(NpgsqlConnection watcher, string relation, CancellationToken ct) + { + _poll = Task.Run( + async () => + { + Stopwatch? since = null; + while (!Volatile.Read(ref _stop)) + { + await using var command = new NpgsqlCommand( + $"SELECT count(*) FROM pg_locks WHERE NOT granted AND relation = '{relation}'::regclass;", watcher); + var waiting = (long)(await command.ExecuteScalarAsync(ct))! > 0; + since = waiting ? since ?? Stopwatch.StartNew() : null; + if (since is not null) + { + Seen = true; + if (since.Elapsed > Longest) + { + Longest = since.Elapsed; + } + } + + await Task.Delay(10, ct); + } + }, + ct); + } + + /// Whether a request on the relation was ever seen waiting. + public bool Seen { get; private set; } + + public TimeSpan Longest { get; private set; } + + public async Task StopAsync() + { + Volatile.Write(ref _stop, true); + await _poll; + } + + public async ValueTask DisposeAsync() => await StopAsync(); + } + + /// + /// A manual VACUUM in a second session, throttled to one page per 100 ms so it is still running when the test's step + /// asks (a manual VACUUM is never cancelled by the deadlock check). Disposing cancels it and waits for it to end, which + /// the scratch database's drop needs. + /// + private sealed class SlowVacuum : IAsyncDisposable + { + private readonly NpgsqlConnection _connection; + private readonly CancellationTokenSource _cancel = new(); + private readonly Task _run; + + private SlowVacuum(NpgsqlConnection connection, string table) + { + _connection = connection; + _run = Task.Run( + async () => + { + try + { + await ExecAsync(connection, "SET vacuum_cost_delay = 100; SET vacuum_cost_limit = 1;", CancellationToken.None); + await using var vacuum = new NpgsqlCommand($"VACUUM {table}", connection) { CommandTimeout = 300 }; + await vacuum.ExecuteNonQueryAsync(_cancel.Token); + } + catch (OperationCanceledException) + { + } + catch (PostgresException) + { + } + }); + } + + public static async Task StartAsync(ScratchPostgres scratch, NpgsqlConnection probe, string table, CancellationToken ct) + { + var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + var vacuum = new SlowVacuum(connection, table); + for (var attempt = 0; attempt < 300; attempt++) + { + if (await CountAsync(probe, $"SELECT count(*) FROM pg_stat_progress_vacuum WHERE relid = '{table}'::regclass", ct) > 0) + { + return vacuum; + } + + await Task.Delay(100, ct); + } + + await vacuum.DisposeAsync(); + throw new InvalidOperationException($"the throttled VACUUM of {table} never showed in pg_stat_progress_vacuum"); + } + + public async ValueTask DisposeAsync() + { + await _cancel.CancelAsync(); + await _run; + await _connection.DisposeAsync(); + _cancel.Dispose(); + } + } + + [Fact] + public async Task TheTurnOff_WhileAManualVacuumRunsOnTheTable_LeavesTheIndexOn_AndNeverAsksForTheLock() + { + /* A manual VACUUM is never cancelled by the deadlock check, so an ALTER (ACCESS EXCLUSIVE on the index, which the + VACUUM holds ROW EXCLUSIVE) could only queue the leaf's writers and readers behind it for the whole lock wait. */ + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + await using var watcher = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + + await ExecAsync(connection, $"CREATE INDEX {Brin}_legacy ON {LegacyTable} USING brin (collection_time) WITH (autosummarize = off)", ct); + await InsertManyAsync(connection, ct); + await ExecAsync(connection, $"ALTER INDEX {LegacyBrin} SET (autosummarize = on)", ct); + + QueryStoreIntervalPartitions.StepResult step; + bool waiterSeen; + await using (var vacuum = await SlowVacuum.StartAsync(scratch, connection, LegacyTable, ct)) + { + var watch = new WaiterWatch(watcher, LegacyBrin, ct); + step = await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, Wide, logger, ct); + await watch.StopAsync(); + waiterSeen = watch.Seen; + } + + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.RetryLater, step.Outcome); + Assert.Contains("left on", step.Detail); + Assert.Equal(0, step.Count); + Assert.False(waiterSeen, "the turn-off asked for the index's lock while a manual VACUUM was running"); + Assert.Equal(1L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + + /* Once the VACUUM is gone the next pass turns it off. */ + var after = await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, Wide, logger, ct); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.Done, after.Outcome); + Assert.Equal(0L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + } + + [Fact] + public async Task TheTurnOff_AReaderHoldingTheIndex_TimesOutBelowFiveSeconds_AndTheTableIsStillConverged() + { + /* A reader's ACCESS SHARE on the index is a lock no check sees as uncancellable, so the ALTER does ask and its lock + wait ends the request: rolled back, reported, and (L1) the table's NEXT index is still turned off. */ + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + await using var reader = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + + await ExecAsync(connection, $"CREATE INDEX {Brin}_legacy ON collect.query_store_interval_wide_legacy USING brin (collection_time) WITH (autosummarize = on)", ct); + await ExecAsync(connection, $"CREATE INDEX {Brin} ON ONLY collect.query_store_interval_wide USING brin (collection_time) WITH (autosummarize = off)", ct); + await ExecAsync(connection, $"ALTER INDEX collect.{Brin} ATTACH PARTITION {LegacyBrin}", ct); + await PromoteAsync(connection, Wide, Now, logger, ct); + + /* The legacy leaf sorts first by name; a day leaf is the one that must not be left behind. */ + var dayLeaf = (string)(await ScalarAsync( + connection, + "SELECT format('%I.%I', n.nspname, i.relname) FROM pg_index x JOIN pg_class i ON i.oid = x.indexrelid JOIN pg_namespace n ON n.oid = i.relnamespace " + + "JOIN pg_am am ON am.oid = i.relam WHERE am.amname = 'brin' AND i.relkind = 'i' AND i.relname <> '" + Brin + "_legacy' " + + "AND x.indrelid IN (SELECT relid FROM pg_partition_tree('collect.query_store_interval_wide'::regclass)) ORDER BY 1 LIMIT 1", + ct))!; + await ExecAsync(connection, $"ALTER INDEX {LegacyBrin} SET (autosummarize = on)", ct); + await ExecAsync(connection, $"ALTER INDEX {dayLeaf} SET (autosummarize = on)", ct); + Assert.Equal(2L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + + var deadlockMs = (int)await CountAsync(connection, "SELECT setting::bigint FROM pg_settings WHERE name = 'deadlock_timeout'", ct); + var lockMs = QueryStoreIntervalBrin.TurnOffLockTimeoutMs(deadlockMs); + QueryStoreIntervalPartitions.StepResult step; + var elapsed = TimeSpan.Zero; + await using (var hold = await reader.BeginTransactionAsync(ct)) + { + await ExecAsync(reader, $"SELECT count(*) FROM {LegacyTable}", ct); + Assert.True( + await CountAsync(connection, $"SELECT count(*) FROM pg_locks WHERE granted AND mode = 'AccessShareLock' AND relation = '{LegacyBrin}'::regclass", ct) > 0, + "the reader's transaction must hold the legacy BRIN index"); + + var watch = Stopwatch.StartNew(); + step = await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, Wide, logger, ct); + elapsed = watch.Elapsed; + await hold.RollbackAsync(ct); + } + + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.RetryLater, step.Outcome); + Assert.Equal(1, step.Count); + Assert.Contains("1 lock timeout", step.Detail); + Assert.True(elapsed >= TimeSpan.FromMilliseconds(lockMs - 100), $"the legacy index's turn-off returned after {elapsed.TotalMilliseconds:F0} ms, before its {lockMs} ms lock timeout"); + Assert.True(elapsed < TimeSpan.FromMilliseconds(lockMs + 3000), $"the turn-off took {elapsed.TotalMilliseconds:F0} ms against a {lockMs} ms lock timeout"); + Assert.True(logger.Lines.Exists(l => l.Level == LogLevel.Warning && l.Message.Contains("lock timeout", StringComparison.Ordinal)), logger.Dump()); + + /* The day leaf, which sorts after the legacy leaf, was altered; the legacy leaf was rolled back and is still on. */ + Assert.Equal(1L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + Assert.Equal(1L, await CountAsync(connection, $"SELECT count(*) FROM pg_class WHERE oid = '{LegacyBrin}'::regclass AND reloptions @> ARRAY['autosummarize=on']", ct)); + + /* With the reader gone the next pass finishes. */ + var after = await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, Wide, logger, ct); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.Done, after.Outcome); + Assert.Equal(0L, await CountAsync(connection, LeafBrinOnCountSql, ct)); + } + + [Fact] + public async Task TheSummarize_WhileAManualVacuumRunsOnTheTable_SkipsTheIndex_AndNeverAsksForTheLock() + { + /* The pg_stat_progress_vacuum skip: a throttled manual VACUUM holds SHARE UPDATE EXCLUSIVE on the table, and the + summarize must not queue for it. */ + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + await using var watcher = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + + await ExecAsync(connection, $"CREATE INDEX {Brin}_legacy ON {LegacyTable} USING brin (collection_time) WITH (autosummarize = off)", ct); + await InsertManyAsync(connection, ct); + + QueryStoreIntervalPartitions.StepResult step; + bool waiterSeen; + await using (var vacuum = await SlowVacuum.StartAsync(scratch, connection, LegacyTable, ct)) + { + var watch = new WaiterWatch(watcher, LegacyTable, ct); + step = await QueryStoreIntervalBrin.SummarizeNewRangesAsync(connection, Wide, logger, ct); + await watch.StopAsync(); + waiterSeen = watch.Seen; + } + + Assert.Contains("skipped 1 being vacuumed", step.Detail); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.NothingToDo, step.Outcome); + Assert.False(waiterSeen, "the summarize asked for the table's lock while a manual VACUUM was running"); + + /* Nothing was summarized, and once the VACUUM is gone the next pass does it. */ + var after = await QueryStoreIntervalBrin.SummarizeNewRangesAsync(connection, Wide, logger, ct); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.Done, after.Outcome); + Assert.True(after.Count >= 1); + } + + [Fact] + public async Task TheSummarizeStatement_StopsBetweenRanges_AndTheNextPassFinishesWithNothingLost() + { + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + + await ExecAsync(connection, $"CREATE INDEX {Brin}_legacy ON {LegacyTable} USING brin (collection_time) WITH (autosummarize = off)", ct); + await InsertManyAsync(connection, ct); + var blocks = await CountAsync(connection, $"SELECT pg_relation_size('{LegacyTable}') / 8192", ct); + var fullRanges = blocks / 128; + var rangesWalked = (blocks + 127) / 128; + Assert.True(fullRanges >= 2, $"expected several complete 128-block ranges; the table has {blocks} blocks"); + + /* A spent budget starts no range: nothing is summarized, every range is reported as not started. */ + var (done, notStarted) = await RunSummarizeAsync(connection, LegacyBrin, 0.0, ct); + Assert.Equal(0L, done); + Assert.Equal(rangesWalked, notStarted); + + /* The next pass, with a budget it will not spend, does all of it (the last, partial range is summarized too: brin_summarize_range passes include_partial). */ + (done, notStarted) = await RunSummarizeAsync(connection, LegacyBrin, 60.0, ct); + Assert.InRange(done, fullRanges - 1, rangesWalked); + Assert.Equal(0L, notStarted); + (done, notStarted) = await RunSummarizeAsync(connection, LegacyBrin, 60.0, ct); + Assert.Equal((0L, 0L), (done, notStarted)); + + /* A leaf dropped after the target list was read reads no rows: (0, 0), not a NULL scalar that throws. */ + (done, notStarted) = await RunSummarizeAsync(connection, "collect.no_such_brin_leaf", 60.0, ct); + Assert.Equal((0L, 0L), (done, notStarted)); + } + + private static async Task AntiWraparoundAsync(NpgsqlConnection connection, string table, CancellationToken ct) + { + await using var command = new NpgsqlCommand( + $"SELECT {QueryStoreIntervalBrin.AntiWraparoundAgeSql} FROM pg_class AS t WHERE t.oid = $1::regclass", connection) { CommandTimeout = 120 }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text, Value = table }); + return (bool)(await command.ExecuteScalarAsync(ct))!; + } + + /// + /// #5594 round 2 L3: the anti-wraparound age fragment reads false below the table's freeze limit and true past it, uses the + /// smaller of the table's and the server's limit, and a table with no setting of its own follows the server's. + /// + [Fact] + public async Task TheAntiWraparoundAge_ReadsTrueOnlyPastTheSmallerOfTheTablesAndTheServersFreezeLimit() + { + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + + /* 100000 is the smallest autovacuum_freeze_max_age a table may set; 2000000000 the largest, far above the server's 200 M. */ + await ExecAsync(connection, "CREATE TABLE collect.wrap_low (c int) WITH (autovacuum_freeze_max_age = 100000)", ct); + await ExecAsync(connection, "CREATE TABLE collect.wrap_high (c int) WITH (autovacuum_freeze_max_age = 2000000000)", ct); + await ExecAsync(connection, "CREATE TABLE collect.wrap_plain (c int)", ct); + await ExecAsync( + connection, + "CREATE PROCEDURE collect.burn_xids(n int) LANGUAGE plpgsql AS $$ BEGIN FOR i IN 1..n LOOP PERFORM pg_current_xact_id(); COMMIT; END LOOP; END $$", + ct); + + /* An open transaction that holds an XID keeps VACUUM from freezing past it, so no autovacuum (the 100000 table is due for + one as soon as it is old enough) can reset the age this test is about to build. */ + await using var holder = new NpgsqlConnection(scratch.ConnectionString); + await holder.OpenAsync(ct); + await using var holderTransaction = await holder.BeginTransactionAsync(ct); + await using (var xid = new NpgsqlCommand("SELECT pg_current_xact_id()::text", holder, holderTransaction)) + { + await xid.ExecuteScalarAsync(ct); + } + + Assert.False(await AntiWraparoundAsync(connection, "collect.wrap_low", ct)); + Assert.False(await AntiWraparoundAsync(connection, "collect.wrap_high", ct)); + Assert.False(await AntiWraparoundAsync(connection, "collect.wrap_plain", ct)); + + /* Just over the 100000 minimum. Commits are not flushed one by one (synchronous_commit off), and each one burns an XID. */ + var burn = Stopwatch.StartNew(); + await ExecAsync(connection, "SET synchronous_commit = off", ct); + await using (var call = new NpgsqlCommand("CALL collect.burn_xids(100200)", connection) { CommandTimeout = 600 }) + { + await call.ExecuteNonQueryAsync(ct); + } + + burn.Stop(); + TestContext.Current.SendDiagnosticMessage($"burned 100200 XIDs in {burn.Elapsed.TotalSeconds:F1} s"); + + Assert.True(await AntiWraparoundAsync(connection, "collect.wrap_low", ct)); + Assert.False(await AntiWraparoundAsync(connection, "collect.wrap_high", ct), "a per-table limit above the server's 200 M must not count as past it"); + Assert.False(await AntiWraparoundAsync(connection, "collect.wrap_plain", ct)); + + /* The server setting cannot change without a restart, so the smaller-limit leg lowers the setting the fragment reads: a + function earlier in search_path answers for current_setting. The table's limit (2 billion) is now the larger of the two, + and autovacuum would use the server's, so the fragment must too; the table's alone (the old COALESCE) reads false. */ + await ExecAsync( + connection, + "CREATE SCHEMA shadow; " + + "CREATE FUNCTION shadow.current_setting(text) RETURNS text LANGUAGE sql STABLE AS " + + "$$ SELECT CASE WHEN $1 = 'autovacuum_freeze_max_age' THEN '100000' ELSE pg_catalog.current_setting($1) END $$; " + + "SET search_path = shadow, pg_catalog", + ct); + Assert.True(await AntiWraparoundAsync(connection, "collect.wrap_high", ct), "the smaller of the table's and the server's limit decides"); + Assert.True(await AntiWraparoundAsync(connection, "collect.wrap_plain", ct), "a table with no limit of its own follows the server's"); + } + + private static async Task<(long Done, long NotStarted)> RunSummarizeAsync(NpgsqlConnection connection, string index, double budgetSeconds, CancellationToken ct) + { + await using var command = new NpgsqlCommand(QueryStoreIntervalBrin.SummarizeIndexSql, connection) { CommandTimeout = 120 }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text, Value = index }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Double, Value = budgetSeconds }); + await using var reader = await command.ExecuteReaderAsync(ct); + Assert.True(await reader.ReadAsync(ct)); + return (reader.GetInt64(0), reader.GetInt64(1)); + } + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 120 }; + return await command.ExecuteScalarAsync(ct); + } +} diff --git a/Darling/Darling.Tests/QueryStoreIntervalConvergenceLiveTests.cs b/Darling/Darling.Tests/QueryStoreIntervalConvergenceLiveTests.cs index ca1edb77f6..c33711b0e6 100644 --- a/Darling/Darling.Tests/QueryStoreIntervalConvergenceLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreIntervalConvergenceLiveTests.cs @@ -478,6 +478,135 @@ public async Task ALegacyRowDatedMoreThanOneHorizonAhead_IsNotArmedOver_WarnsOnc Assert.Equal(2, FarMaxWarnings(logger)); } + [Fact] + public async Task AFarFutureLegacyRow_AnHourOfBothLoops_RequestsNoLock_WhileAnAutovacuumLikeLockIsHeld() + { + /* #5589. A SHARE UPDATE EXCLUSIVE lock stands in for a running autovacuum: any lock request that conflicts with it would wait + (and in production cancel the autovacuum). With the newest row beyond the horizon, neither the hourly convergence step nor + the background loop's pass (arm, validate, promote, analyze) may request one: every call returns NotReady in well under + the 5 s lock_timeout, and a third connection never sees a backend waiting on the legacy table. */ + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + await using var loopConnection = await OpenStoreAsync(scratch, ct); + await using var blocker = await OpenStoreAsync(scratch, ct); + await using var watcher = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + QueryStoreIntervalPartitions.ResetFarFutureMaxWarnings(); + + await InsertAsync(connection, Wide, new DateTime(2026, 10, 19, 0, 0, 0, DateTimeKind.Unspecified), 1, ct); + + var waiters = 0; + var stop = false; + var steps = new List<(string What, QueryStoreIntervalPartitions.StepOutcome Outcome, TimeSpan Elapsed)>(); + Task poll; + await using (var hold = await blocker.BeginTransactionAsync(ct)) + { + await ExecAsync(blocker, $"LOCK TABLE {Wide.Legacy} IN SHARE UPDATE EXCLUSIVE MODE", ct); + + poll = Task.Run( + async () => + { + while (!Volatile.Read(ref stop)) + { + await using var command = new NpgsqlCommand( + $"SELECT count(*) FROM pg_locks WHERE NOT granted AND relation = '{Wide.Legacy}'::regclass;", watcher); + waiters = Math.Max(waiters, (int)(long)(await command.ExecuteScalarAsync(ct))!); + await Task.Delay(20, ct); + } + }, + ct); + + /* The hourly convergence step, a few hours running. */ + for (var hour = 0; hour < 3; hour++) + { + var watch = System.Diagnostics.Stopwatch.StartNew(); + var step = await QueryStoreIntervalPartitions.ConvergeUnpromotedAsync(connection, Wide, Now.AddHours(hour), logger, ct); + steps.Add(("converge", step.Outcome, watch.Elapsed)); + } + + /* The background loop's own body, through the loop entry: its promotion step is RunPromotionAsync and its analyze step is + AnalyzeIfDueAsync, each on its own connection exactly as the production lambdas do. The index ensure is a no-op here: it + runs once per start, not hourly. */ + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + var passes = 0; + await QueryStoreIntervalPartitions.RunDelayedAsync( + logger, + TimeSpan.Zero, + TimeSpan.FromMilliseconds(20), + OnlyWide, + async (table, token) => + { + var watch = System.Diagnostics.Stopwatch.StartNew(); + var result = await QueryStoreIntervalPartitions.RunPromotionAsync(loopConnection, table, Now.AddHours(passes), logger, token); + steps.Add(("promotion", result.Outcome, watch.Elapsed)); + return result; + }, + async (table, token) => + { + var watch = System.Diagnostics.Stopwatch.StartNew(); + var result = await QueryStoreIntervalPartitions.AnalyzeIfDueAsync(loopConnection, table, Now.AddHours(passes), logger, token); + steps.Add(("analyze", result.Outcome, watch.Elapsed)); + if (++passes >= 3) + { + cts.Cancel(); + } + + return result; + }, + _ => Task.CompletedTask, + cts.Token); + + Volatile.Write(ref stop, true); + await poll; + await hold.RollbackAsync(ct); + } + + Assert.Equal(3, steps.Count(x => x.What == "converge")); + Assert.Equal(3, steps.Count(x => x.What == "promotion")); + Assert.Equal(3, steps.Count(x => x.What == "analyze")); + foreach (var (what, outcome, elapsed) in steps) + { + Assert.True(outcome == QueryStoreIntervalPartitions.StepOutcome.NotReady, $"{what} ended {outcome} (RetryLater means it requested a conflicting lock)"); + Assert.True(elapsed < TimeSpan.FromSeconds(3), $"{what} took {elapsed.TotalSeconds:F1} s: it waited on a lock"); + } + + Assert.Equal(0, waiters); + Assert.Equal(1, FarMaxWarnings(logger)); + Assert.False((await StateAsync(connection, ct)).CheckPresent); + } + + [Fact] + public async Task ALegacyRowInsideTheHorizon_StillTakesTheArmLock_AndIsRetryLaterWhileAnAutovacuumLikeLockIsHeld() + { + /* #5589. The pre-check is only a refusal: a maximum inside the horizon goes on to the locked, authoritative arm. */ + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), SkipText); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + await using var connection = await OpenStoreAsync(scratch, ct); + await using var blocker = await OpenStoreAsync(scratch, ct); + var logger = new ListLogger(); + QueryStoreIntervalPartitions.ResetFarFutureMaxWarnings(); + + await InsertAsync(connection, Wide, new DateTime(2026, 10, 12, 0, 0, 0, DateTimeKind.Unspecified), 1, ct); + + await using (var hold = await blocker.BeginTransactionAsync(ct)) + { + await ExecAsync(blocker, $"LOCK TABLE {Wide.Legacy} IN SHARE UPDATE EXCLUSIVE MODE", ct); + var blocked = await QueryStoreIntervalPartitions.ConvergeUnpromotedAsync(connection, Wide, Now, logger, ct); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.RetryLater, blocked.Outcome); + await hold.RollbackAsync(ct); + } + + var armed = await QueryStoreIntervalPartitions.ConvergeUnpromotedAsync(connection, Wide, Now, logger, ct); + Assert.Equal(QueryStoreIntervalPartitions.StepOutcome.Done, armed.Outcome); + Assert.True((await StateAsync(connection, ct)).CheckPresent); + Assert.Equal(0, FarMaxWarnings(logger)); + } + [Fact] public async Task ALegacyRowOneMicrosecondInsideTheHorizon_IsArmedOver_AndTheValidatePasses() { @@ -744,10 +873,20 @@ public void TheRound2Fixes_AreInTheProductText() /* M1: a far-future legacy maximum is never armed over; the transaction ends before the CHECK is added. */ Assert.True( - arm.IndexOf("IsLegacyMaxBeyondHorizon(", StringComparison.Ordinal) is var m1 && m1 > arm.IndexOf("ReadLegacyMaxAsync(", StringComparison.Ordinal) + arm.LastIndexOf("IsLegacyMaxBeyondHorizon(", StringComparison.Ordinal) is var m1 && m1 > arm.IndexOf("ReadLegacyMaxAsync(", StringComparison.Ordinal) && m1 < arm.IndexOf("AddCheckSql(", StringComparison.Ordinal), "the horizon test sits between the maximum read and the ADD CONSTRAINT"); - Assert.Contains("RollbackAsync", arm[arm.IndexOf("IsLegacyMaxBeyondHorizon(", StringComparison.Ordinal)..arm.IndexOf("AddCheckSql(", StringComparison.Ordinal)], StringComparison.Ordinal); + Assert.Contains("RollbackAsync", arm[arm.LastIndexOf("IsLegacyMaxBeyondHorizon(", StringComparison.Ordinal)..arm.IndexOf("AddCheckSql(", StringComparison.Ordinal)], StringComparison.Ordinal); + + /* #5589: the same test also runs first, on an unlocked read, so a far-future row never requests ACCESS EXCLUSIVE (it would cancel autovacuum). */ + var firstTest = arm.IndexOf("IsLegacyMaxBeyondHorizon(", StringComparison.Ordinal); + Assert.True(firstTest < arm.LastIndexOf("IsLegacyMaxBeyondHorizon(", StringComparison.Ordinal), "the horizon test runs twice"); + Assert.True( + arm.IndexOf("ReadLegacyMaxUnlockedAsync(", StringComparison.Ordinal) is var pre && pre > 0 + && pre < firstTest && firstTest < arm.IndexOf("BeginTransactionAsync(", StringComparison.Ordinal) + && firstTest < arm.IndexOf("ACCESS EXCLUSIVE MODE", StringComparison.Ordinal), + "the unlocked read and its horizon test come before the lock transaction"); + Assert.Contains("WarnFarFutureMax(", arm[firstTest..arm.IndexOf("BeginTransactionAsync(", StringComparison.Ordinal)], StringComparison.Ordinal); /* L1: the guard is derived from the deadline and the re-arm window. */ Assert.Contains("ValidateGuard = TimeSpan.FromSeconds(ValidateTimeoutSeconds) + ReArmWithin;", source, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs b/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs index f2436f9bad..b29cbe431a 100644 --- a/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs +++ b/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs @@ -363,6 +363,13 @@ public async Task UpgradeFromV170_WithAValidBtreeAndAValidBrin_AttachesBothWitho /* The same physical files under the renamed indexes: nothing was rebuilt. */ Assert.Equal(filesBefore, await TextAsync(connection, "SELECT string_agg(relfilenode::text, ',' ORDER BY replace(relname, '_legacy', '')) FROM pg_class WHERE relname IN ('ux_query_store_interval_wide_legacy', 'ix_query_store_interval_wide_server_first_exec_legacy', 'ix_query_store_interval_wide_collection_time_brin_legacy', 'idx_query_store_interval_wide_first_exec_legacy')", ct)); + + /* #5594: the parent's BRIN says off, so a day partition made later clones off; the legacy copy keeps what it was built with + (it cannot change without a lock) until the runtime step turns it off. */ + Assert.Equal("autosummarize=off", await TextAsync(connection, + "SELECT array_to_string(reloptions, ',') FROM pg_class WHERE oid = 'collect.ix_query_store_interval_wide_collection_time_brin'::regclass", ct)); + Assert.Equal("autosummarize=on", await TextAsync(connection, + "SELECT array_to_string(reloptions, ',') FROM pg_class WHERE oid = 'collect.ix_query_store_interval_wide_collection_time_brin_legacy'::regclass", ct)); bodySucceeded = true; } finally diff --git a/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexLiveTests.cs b/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexLiveTests.cs index 96c2f58dcc..43032b6ac2 100644 --- a/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexLiveTests.cs @@ -131,7 +131,7 @@ public async Task Ensure_BuildsAValidBrinIndexOnce_AndASecondCallChangesNothing( $"SELECT indisvalid FROM pg_index WHERE indexrelid = '{Index}'::regclass", ct))!); var definition = (string)(await ScalarAsync(connection, $"SELECT pg_get_indexdef('{Index}'::regclass)", ct))!; Assert.Contains("USING brin (collection_time)", definition); - Assert.Contains("autosummarize='on'", definition); + Assert.Contains("autosummarize=off", definition); var oid = await ScalarAsync(connection, $"SELECT '{Index}'::regclass::oid", ct); await EnsureAsync(connection, ct); @@ -360,7 +360,7 @@ public async Task AnInvalidLeftover_IsDroppedAndRebuilt() Assert.NotEqual(oldLeafOid, await ScalarAsync(connection, $"SELECT '{Index}_legacy'::regclass::oid", ct)); var definition = (string)(await ScalarAsync(connection, $"SELECT pg_get_indexdef('{Index}_legacy'::regclass)", ct))!; Assert.Contains("USING brin (collection_time)", definition); - Assert.Contains("autosummarize='on'", definition); + Assert.Contains("autosummarize=off", definition); } /// diff --git a/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexTests.cs b/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexTests.cs index ba9e828d99..916aaa9dbf 100644 --- a/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexTests.cs +++ b/Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexTests.cs @@ -58,7 +58,10 @@ public void TheBuildSql_IsConcurrent_Idempotent_AndABrinOnCollectionTime() var sql = QueryStoreIntervalWideBrinIndex.CreateSql; Assert.Contains("CREATE INDEX CONCURRENTLY IF NOT EXISTS", sql); Assert.Contains("USING brin (collection_time)", sql); - Assert.Contains("autosummarize = on", sql); + /* #5594: off. An autosummarize work item waits for the table's lock and cancels a running autovacuum. */ + Assert.Contains("autosummarize = off", sql); + Assert.DoesNotContain("autosummarize = on", sql); + Assert.Contains("autosummarize = off", QueryStoreIntervalWideBrinIndex.Spec.IndexDefinition); Assert.Contains("ix_query_store_interval_wide_collection_time_brin", sql); Assert.Contains("DROP INDEX CONCURRENTLY IF EXISTS", QueryStoreIntervalWideBrinIndex.DropSql); } diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index d1d6b67e7c..ce695461af 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -974,7 +974,7 @@ CREATE TABLE collect.__TABLE__ (LIKE collect.__TABLE___legacy INCLUDING DEFAULTS .Replace("__DEFINITION__", "(server_id, first_execution_time)", StringComparison.Ordinal) + (wide ? optionalIndex.Replace("__NAME__", "ix___TABLE___collection_time_brin", StringComparison.Ordinal) - .Replace("__DEFINITION__", "USING brin (collection_time) WITH (autosummarize = on)", StringComparison.Ordinal) + .Replace("__DEFINITION__", "USING brin (collection_time) WITH (autosummarize = off)", StringComparison.Ordinal) : @" CREATE TRIGGER trg_plan_regression_daily_late diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreBackgroundIndexes.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreBackgroundIndexes.cs index e6b64fee67..aa50080899 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreBackgroundIndexes.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreBackgroundIndexes.cs @@ -95,7 +95,7 @@ public enum IndexAction /// Why a server below the floor must not build it, for the log. /// /// What follows ON <table> in the index's definition ((server_id, first_execution_time), - /// USING brin (collection_time) WITH (autosummarize = on)). The partitioned path (#5571) builds the parent + /// USING brin (collection_time) WITH (autosummarize = off)). The partitioned path (#5571) builds the parent /// ON ONLY and each leaf's child from it. Empty for a spec that is never aimed at a partitioned table. /// public sealed record IndexSpec( diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalBrin.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalBrin.cs new file mode 100644 index 0000000000..cd80399289 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalBrin.cs @@ -0,0 +1,550 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The BRIN indexes on the Query Store interval tables and autovacuum (#5594): they are summarized by the service, never +/// by an autovacuum work item, so a running VACUUM of a big table is not cancelled for them. +/// +/// The defect. autosummarize = on (#4862) queues a "BRIN summarize" autovacuum work item for every new +/// block range. The work item needs SHARE UPDATE EXCLUSIVE on the table, the same lock a VACUUM holds, so it waits; +/// after deadlock_timeout (1 s) PostgreSQL cancels the blocking autovacuum for any waiter. On a store that had just +/// drained 17.8 M rows from a 12.3 M-block table, the table's autovacuum was cancelled 8 times in 14 minutes and had not +/// finished since: each attempt got 170 K to 313 K blocks in. On a throwaway PostgreSQL 18 rig (a 1 GB table, a throttled +/// VACUUM, 3,000 rows inserted every 0.7 s) it was cancelled 69 times in 240 s and never finished. +/// +/// Off, everywhere. New builds say autosummarize = off (, and +/// V171's parent index, so a day partition made by PARTITION OF or ATTACH clones off). +/// turns it off on every existing BRIN leaf of a table (the legacy table's, each +/// day leaf's, DEFAULT's): it reads the catalog and alters only an index that is still on, so a converged store pays one +/// catalog read. It runs on the hourly convergence pass, which also runs on the start path before the collectors. A store +/// that ran V171 before the parent said off keeps an on parent, and the partitioned parent index cannot be +/// altered ("This operation is not supported for partitioned indexes", PostgreSQL 18; a parent index is a catalog template +/// with no storage, so its setting changes nothing by itself), so such a store keeps cloning on. Promotion and +/// create-ahead therefore turn it off inside the transaction that makes the leaf, while the leaf is new and nothing else +/// holds it. +/// +/// The lock the ALTER takes. ALTER INDEX ... SET (autosummarize = off) takes ACCESS EXCLUSIVE on the INDEX, +/// not the table (measured with pg_locks on PostgreSQL 18.6). A running VACUUM holds ROW EXCLUSIVE on each of its table's +/// indexes for its whole run, so the ALTER waits for it, and a queued ACCESS EXCLUSIVE request blocks everything that asks +/// for the index after it: the collectors' inserts (ROW EXCLUSIVE) and also every read that does not prune the leaf (the +/// planner takes ACCESS SHARE on each index of a leaf it plans against). So the wait is felt by readers as well as writers. +/// What the holder is decides whether waiting does any good: +/// +/// A regular autovacuum is cancelled by the deadlock check after deadlock_timeout, and the ALTER goes through +/// (measured: 1,008 ms, one cancel). That is deliberately accepted: it happens once per index, ends the cancel loop for +/// good, and the VACUUM it cancels was being cancelled every couple of minutes anyway. +/// An anti-wraparound autovacuum and a manual VACUUM (any role) are never cancelled: waiting for them only queues +/// the ALTER, and the leaf's writers and readers with it. A store that never finished vacuuming its big legacy table is the +/// likely one to be in anti-wraparound. So an index whose table pg_stat_progress_vacuum shows being vacuumed by one +/// of those is not asked for at all: the step leaves it on, says so, and the next pass tries again +/// (). A vacuum is anti-wraparound by either of two signals. +/// First, its table's age(relfrozenxid) or mxid_age(relminmxid) is past autovacuum_freeze_max_age or +/// autovacuum_multixact_freeze_max_age, the smaller of the table's own setting and the server's (autovacuum does the +/// same in relation_needs_vacanalyze; ). Second, the worker's own activity text ends +/// in (to prevent wraparound), which autovacuum writes for exactly that flag; it also covers a multixact member-space +/// limit that SQL cannot read. A vacuum whose backend is not an autovacuum worker is a manual one. A role that may not see another role's worker reads NULL from +/// pg_stat_progress_vacuum and pg_stat_activity (measured), and then only the next point applies. +/// What no check can see waits at most min(5 s, deadlock_timeout + 1 s) (, +/// read at run time in the same session): a regular autovacuum still gets its deadlock_timeout to be cancelled plus a +/// second to let go, and a futile wait is about 2 s on a default server, not 5. +/// +/// The turn-off and the summarize run on different tasks (the convergence pass and the background task, 20 minutes +/// apart after a start and each hourly), each on its own connection, so nothing keeps them from overlapping. They do not +/// get in each other's way: brin_summarize_range takes its SHARE UPDATE EXCLUSIVE locks inside the call and +/// releases them when the call ends (measured with pg_locks: none held after it), so the summarize holds the index for one +/// range at a time, not for its run. Measured on PostgreSQL 18.6, an ALTER sent 1.5 s into a 10.7 s summarize of a +/// 156 K-block table took 150 ms and the summarize finished every range. A lock timeout or a skipped index moves on to the +/// next index of the table; the step reports RetryLater after the loop. +/// +/// Summarizing, explicitly. Without autosummarize a range is summarized only when VACUUM reaches the end of +/// its table, and an unsummarized range is always read, so every collection_time read through the BRIN scans the +/// unsummarized tail of every leaf it touches. Measured on a throwaway rig (a 111 K-block table, a 6 h window of 24 h): +/// 27,904 blocks with no tail, 38,895 with a 10% tail (+39%), 50,006 with a 20% tail (+79%), 27,904 again once +/// summarized; and a leaf that stops growing keeps its tail until another VACUUM comes. Insert-driven autovacuum fires +/// after 20% growth, so the tail is real. closes it hourly. Summarizing takes the same +/// SHARE UPDATE EXCLUSIVE a VACUUM holds, so it must never be the waiter that cancels one: it reads +/// deadlock_timeout at run time, runs each index in its own transaction under a lock_timeout of half of it +/// (capped at ), skips an index whose table pg_stat_progress_vacuum shows +/// being vacuumed without asking for a lock, and does nothing when deadlock_timeout is under +/// . +/// +/// Stopping between ranges. brin_summarize_new_values cannot be stopped cleanly. A range is summarized by +/// first inserting a placeholder tuple for it and then reading its 128 blocks, so a statement_timeout cancel lands +/// inside a range nearly every time, and a range that has any tuple (a placeholder included) is skipped by every later +/// summarize. Measured on PostgreSQL 18.6: a 150 ms statement_timeout on a 47 K-block table left a placeholder; a second +/// brin_summarize_new_values and a brin_summarize_range on that very block both returned without touching it; +/// only brin_desummarize_range and then a summarize healed it. A placeholder is always returned by a scan, so that +/// range (128 blocks) would be read by every query for good, one per deadline hit. So the summarize is one statement per +/// index that calls brin_summarize_range for each range of the heap in block order, each call guarded by a clock check +/// against : when the budget is spent the remaining ranges are not started, and the +/// next pass continues at the first range still missing. Measured: a 200 ms budget summarized 24 ranges, left no +/// placeholder, and the next pass finished the other 342. A statement_timeout of +/// past the budget stays as a backstop for a read that stalls; only that one can leave a placeholder. The cost of testing +/// every range each pass is about 3 microseconds a range (measured on summarized ranges), about 0.3 s for a 12.3 M-block table. The +/// budget is reached only by a tail nothing has ever summarized: at the 14,600 to 36,000 blocks a second measured on the rig +/// (a 156 K-block table took 10.7 s, a 47 K-block one 1.2 s), the 12.3 M-block legacy table of the store in #5594 is 6 to 14 +/// minutes of reading, more from a slower disk, so several hourly passes; one hour's growth of a day leaf (one 24th of a +/// leaf, for example 17 K blocks of 400 K) is a second or two. A call with nothing to summarize costs a few +/// milliseconds. +/// +/// Only leaves. A partitioned index (relkind I) has no storage: it can be neither altered nor +/// summarized. Every statement here names a leaf index found in the catalog (format('%I.%I'), never a built-up +/// string), through the table's partition tree, so a future leaf is covered the day it exists. A leaf dropped between the +/// catalog read and its statement (the sweep drops expired days) is skipped. +/// +public static class QueryStoreIntervalBrin +{ + /// The smallest deadlock_timeout (milliseconds) that leaves room for a lock wait strictly below it. + public const int MinimumDeadlockTimeoutMs = 200; + + /// The longest lock wait of a summarize, in milliseconds, whatever deadlock_timeout is. + public const int MaximumSummarizeLockTimeoutMs = 2000; + + /// The time one index's summarize may spend starting new ranges, in seconds; it stops between ranges. + public const int SummarizeBudgetSeconds = 60; + + /// How long after the budget the server cancels a summarize statement that is stuck inside a range, in seconds. + public const int SummarizeBackstopSeconds = 30; + + /// Deadline for the catalog reads, in seconds. + public const int CatalogReadTimeoutSeconds = 30; + + /* The BRIN leaf indexes of $1 and its partition tree. pg_partition_tree is strict, so a missing table reads nothing; a plain + table is found by the first predicate. Relkind i only: the partitioned parent index (I) has no storage and no ALTER. */ + private const string LeafBrinIndexesFrom = @" +FROM pg_class AS i +JOIN pg_index AS x ON x.indexrelid = i.oid +JOIN pg_namespace AS n ON n.oid = i.relnamespace +JOIN pg_am AS am ON am.oid = i.relam +WHERE am.amname = 'brin' +AND i.relkind = 'i' +AND (x.indrelid = to_regclass($1) OR x.indrelid IN (SELECT pt.relid FROM pg_partition_tree(to_regclass($1)) AS pt))"; + + /* Whether the table (the pg_class row aliased t) is old enough that autovacuum runs an anti-wraparound vacuum on it (#5594 + round 2 L1, L3): the XID age or the multixact age past the freeze limit. Autovacuum's limit is the SMALLER of the table's + reloption and the server setting (relation_needs_vacanalyze in autovacuum.c), so LEAST, which skips the NULL of an unset + reloption; a per-table value above the server's is ignored by autovacuum and must be ignored here. A fragment of its own + so a test can evaluate it alone against a table (SELECT FROM pg_class AS t WHERE t.oid = $1::regclass). */ + internal const string AntiWraparoundAgeSql = @" +( + age(t.relfrozenxid) > + LEAST + ( + (SELECT o.option_value::int FROM pg_options_to_table(t.reloptions) AS o WHERE o.option_name = 'autovacuum_freeze_max_age'), + current_setting('autovacuum_freeze_max_age')::int + ) + OR mxid_age(t.relminmxid) > + LEAST + ( + (SELECT o.option_value::int FROM pg_options_to_table(t.reloptions) AS o WHERE o.option_name = 'autovacuum_multixact_freeze_max_age'), + current_setting('autovacuum_multixact_freeze_max_age')::int + ) +)"; + + /* Every BRIN leaf index of $1 whose autosummarize is on, and whether its table is being vacuumed by a vacuum the deadlock + check never cancels (#5594 M1): a manual VACUUM (a backend that is not an autovacuum worker), or an autovacuum that is + anti-wraparound. Two signals for the second: the table's age (AntiWraparoundAgeSql), and the worker's activity text, which + autovacuum ends with ' (to prevent wraparound)' for an anti-wraparound vacuum (autovac_report_activity in autovacuum.c: + tab->at_params.is_wraparound ? " (to prevent wraparound)" : "", PostgreSQL 16-18). The text also covers the multixact + member-space limit, which lowers autovacuum's real multixact limit below the setting and which SQL cannot read. + pg_stat_progress_vacuum and pg_stat_activity read NULL for another role's worker the caller may not see, which makes + this false for it, and the lock_timeout decides instead. The option text is whatever the DDL said (on, true, yes, 1). */ + internal const string AutosummarizeOnSql = @" +SELECT format('%I.%I', n.nspname, i.relname), + EXISTS + ( + SELECT 1 + FROM pg_stat_progress_vacuum AS v + JOIN pg_class AS t ON t.oid = v.relid + WHERE v.datid = (SELECT d.oid FROM pg_database AS d WHERE d.datname = current_database()) + AND v.relid = x.indrelid + AND + ( + " + AntiWraparoundAgeSql + @" + OR EXISTS + ( + SELECT 1 + FROM pg_stat_activity AS a + WHERE a.pid = v.pid + AND (a.backend_type <> 'autovacuum worker' OR a.query LIKE '%(to prevent wraparound)') + ) + ) + )" + LeafBrinIndexesFrom + @" +AND EXISTS +( + SELECT 1 + FROM pg_options_to_table(i.reloptions) AS o + WHERE o.option_name = 'autosummarize' + AND lower(o.option_value) IN ('on', 'true', 'yes', 't', 'y', '1') +) +ORDER BY 1;"; + + /* Every valid BRIN leaf index of $1, and whether its table is being vacuumed right now. pg_stat_progress_vacuum is + per database, and a role that may not see another role's worker reads NULL there, which only means the lock timeout + decides instead of this check. */ + internal const string SummarizeTargetsSql = @" +SELECT format('%I.%I', n.nspname, i.relname), + EXISTS + ( + SELECT 1 + FROM pg_stat_progress_vacuum AS v + WHERE v.datid = (SELECT d.oid FROM pg_database AS d WHERE d.datname = current_database()) + AND v.relid = x.indrelid + )" + LeafBrinIndexesFrom + @" +AND x.indisvalid +ORDER BY 1;"; + + /* One index's summarize ($1 the index, $2 the budget in seconds): brin_summarize_range for each range of the heap in block + order, each call behind a clock check, so the statement stops BETWEEN ranges (brin_summarize_new_values can only be + cancelled, which strands a placeholder; see the class remarks). Column 0 is the ranges summarized, column 1 the ranges + not started because the budget was spent. The index is resolved once in the inner query: a dropped index reads no rows + (0, 0) instead of a NULL scalar. The last, partial range is summarized too (brin_summarize_range passes include_partial). pages_per_range is the + index's own setting, 128 when unset. */ + internal const string SummarizeIndexSql = @" +SELECT COALESCE(sum(s.r), 0)::bigint, + count(*) FILTER (WHERE s.r IS NULL)::bigint +FROM +( + SELECT CASE WHEN clock_timestamp() < statement_timestamp() + make_interval(secs => $2) + THEN brin_summarize_range(ix.indexrelid, g.b) + END AS r + FROM + ( + SELECT x.indexrelid, + x.indrelid, + COALESCE((SELECT o.option_value::int FROM pg_options_to_table(i.reloptions) AS o WHERE o.option_name = 'pages_per_range'), 128) AS pages_per_range + FROM pg_index AS x + JOIN pg_class AS i ON i.oid = x.indexrelid + WHERE x.indexrelid = to_regclass($1) + ) AS ix + CROSS JOIN LATERAL generate_series(0::bigint, pg_relation_size(ix.indrelid) / current_setting('block_size')::int - 1, ix.pages_per_range) AS g(b) +) AS s;"; + + internal const string DeadlockTimeoutSql = "SELECT setting::int FROM pg_settings WHERE name = 'deadlock_timeout';"; + + /// + /// The lock wait of a summarize in milliseconds for a server whose deadlock_timeout is + /// : half of it, at most , so always + /// strictly below it; 0 when is under , which + /// means the step does not run. + /// + public static int SummarizeLockTimeoutMs(int deadlockTimeoutMs) => + deadlockTimeoutMs < MinimumDeadlockTimeoutMs ? 0 : Math.Min(deadlockTimeoutMs / 2, MaximumSummarizeLockTimeoutMs); + + /// + /// The lock wait of the turn-off ALTER in milliseconds for a server whose deadlock_timeout is + /// : deadlock_timeout plus one second, at most the step's usual + /// . A regular autovacuum is cancelled by the deadlock check + /// at deadlock_timeout, so this gives it that and a second to let go; a wait that long on a holder that is not + /// cancelled is the most a request that queues readers and writers should cost. + /// + public static int TurnOffLockTimeoutMs(int deadlockTimeoutMs) => + (int)Math.Min(QueryStoreIntervalPartitions.LockTimeoutSeconds * 1000L, Math.Max(deadlockTimeoutMs, 1) + 1000L); + + private static string AlterOffSql(string index) => $"ALTER INDEX {index} SET (autosummarize = off);"; + + private static async Task ReadDeadlockTimeoutMsAsync(NpgsqlConnection connection, CancellationToken cancellationToken) + { + await using var read = new NpgsqlCommand(DeadlockTimeoutSql, connection) { CommandTimeout = CatalogReadTimeoutSeconds }; + return Convert.ToInt32(await read.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false), System.Globalization.CultureInfo.InvariantCulture); + } + + private static async Task> ReadAutosummarizeOnAsync( + NpgsqlConnection connection, string relation, NpgsqlTransaction? transaction, CancellationToken cancellationToken) + { + var found = new List<(string Index, bool Held)>(); + await using var command = new NpgsqlCommand(AutosummarizeOnSql, connection, transaction) { CommandTimeout = CatalogReadTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text, Value = relation }); + await using var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + found.Add((reader.GetString(0), reader.GetBoolean(1))); + } + + return found; + } + + /// + /// Turns autosummarize off on every BRIN leaf index of and its partitions that still has + /// it on, inside (which carries its own lock_timeout). For a partition the same + /// transaction has just created or locked, so the ALTER waits for nothing. Returns how many indexes it altered. + /// + internal static async Task TurnOffInTransactionAsync( + NpgsqlConnection connection, NpgsqlTransaction transaction, string relation, CancellationToken cancellationToken) + { + var on = await ReadAutosummarizeOnAsync(connection, relation, transaction, cancellationToken).ConfigureAwait(false); + foreach (var (index, _) in on) + { + await using var alter = new NpgsqlCommand(AlterOffSql(index), connection, transaction) { CommandTimeout = QueryStoreIntervalPartitions.ShortTimeoutSeconds }; + await alter.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + return on.Count; + } + + /// + /// The convergence step: every BRIN leaf index of that still has autosummarize on (an + /// index built by #4862, V171's legacy and parent copies, a day leaf that cloned the parent's option) is altered off, + /// each in its own transaction under . An index a holder that cannot be cancelled has + /// (see the class remarks) is skipped without a lock request; a lock timeout rolls that index back. Neither stops the + /// other indexes of the table; either one makes the outcome + /// , and the indexes already done stay done. Count is the + /// number of indexes altered. + /// + public static async Task TurnOffAutosummarizeAsync( + NpgsqlConnection connection, QueryStoreIntervalPartitions.IntervalTable table, ILogger logger, CancellationToken cancellationToken) + { + var on = await ReadAutosummarizeOnAsync(connection, table.Parent, null, cancellationToken).ConfigureAwait(false); + if (on.Count == 0) + { + return new QueryStoreIntervalPartitions.StepResult(QueryStoreIntervalPartitions.StepOutcome.NothingToDo, "no BRIN index has autosummarize on"); + } + + var altered = 0; + var held = 0; + var timedOut = 0; + var lockTimeoutMs = 0; + foreach (var (index, uncancellableHolder) in on) + { + if (uncancellableHolder) + { + held++; + logger.LogDebug( + "Query Store interval table {Table}: autosummarize stays on for {Index} this pass: its table is being vacuumed by a vacuum " + + "that is never cancelled (a manual VACUUM or an anti-wraparound autovacuum), so asking for the lock would only queue readers and writers.", + table.Parent, index); + continue; + } + + if (lockTimeoutMs == 0) + { + lockTimeoutMs = TurnOffLockTimeoutMs(await ReadDeadlockTimeoutMsAsync(connection, cancellationToken).ConfigureAwait(false)); + } + + try + { + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await using (var timeout = new NpgsqlCommand( + $"SET LOCAL lock_timeout = '{lockTimeoutMs}ms';", connection, transaction) + { CommandTimeout = QueryStoreIntervalPartitions.ShortTimeoutSeconds }) + { + await timeout.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + await using (var alter = new NpgsqlCommand(AlterOffSql(index), connection, transaction) { CommandTimeout = QueryStoreIntervalPartitions.ShortTimeoutSeconds }) + { + await alter.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + altered++; + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.LockNotAvailable) + { + timedOut++; + logger.LogDebug( + "Query Store interval table {Table}: turning autosummarize off on {Index} hit its {LockMs} ms lock timeout and rolled back.", + table.Parent, index, lockTimeoutMs); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.UndefinedTable) + { + /* The leaf was dropped (the sweep drops expired days) between the catalog read and the ALTER: nothing to turn off. */ + logger.LogDebug("Query Store interval table {Table}: {Index} was dropped before autosummarize could be turned off on it.", table.Parent, index); + } + } + + if (altered > 0) + { + logger.LogInformation( + "Query Store interval table {Table}: turned autosummarize off on {Count} BRIN index(es), so a BRIN summarize work item " + + "no longer waits for the table's lock and cancels its autovacuum (#5594).", + table.Parent, altered); + } + + if (held + timedOut > 0) + { + var detail = $"autosummarize off on {altered} BRIN index(es); {held} left on while a vacuum that cannot be cancelled runs on their table, {timedOut} lock timeout(s)"; + if (timedOut > 0) + { + logger.LogWarning( + "Query Store interval table {Table}: turning autosummarize off hit a lock timeout on {TimedOut} BRIN index(es) and rolled them back, " + + "and left {Held} on while a vacuum that cannot be cancelled runs on their table; the next pass retries.", + table.Parent, timedOut, held); + } + else + { + logger.LogInformation( + "Query Store interval table {Table}: autosummarize stays on for {Held} BRIN index(es) while a vacuum that cannot be " + + "cancelled runs on their table; the next pass retries.", + table.Parent, held); + } + + return new QueryStoreIntervalPartitions.StepResult(QueryStoreIntervalPartitions.StepOutcome.RetryLater, detail, altered); + } + + return new QueryStoreIntervalPartitions.StepResult( + QueryStoreIntervalPartitions.StepOutcome.Done, $"autosummarize off on {altered} BRIN index(es)", altered); + } + + /// + /// The hourly summarize (see the class remarks): every range not yet summarized on each valid BRIN leaf index of + /// , each index in its own transaction, under a lock_timeout strictly below + /// deadlock_timeout and a budget that stops it between ranges. An index whose + /// table is being vacuumed is skipped without a lock request. Count is the number of ranges summarized. A lock timeout + /// is ; a deadlock_timeout too short to leave + /// room is . Never throws for a lock, a budget or a + /// dropped leaf; shutdown propagates. + /// + public static async Task SummarizeNewRangesAsync( + NpgsqlConnection connection, QueryStoreIntervalPartitions.IntervalTable table, ILogger logger, CancellationToken cancellationToken) + { + var deadlockMs = await ReadDeadlockTimeoutMsAsync(connection, cancellationToken).ConfigureAwait(false); + var lockMs = SummarizeLockTimeoutMs(deadlockMs); + if (lockMs == 0) + { + return new QueryStoreIntervalPartitions.StepResult( + QueryStoreIntervalPartitions.StepOutcome.NotReady, + $"deadlock_timeout is {deadlockMs} ms, too short to wait for a lock below it, so no BRIN summarize runs"); + } + + var targets = new List<(string Index, bool BeingVacuumed)>(); + await using (var list = new NpgsqlCommand(SummarizeTargetsSql, connection) { CommandTimeout = CatalogReadTimeoutSeconds }) + { + list.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text, Value = table.Parent }); + await using var reader = await list.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + targets.Add((reader.GetString(0), reader.GetBoolean(1))); + } + } + + if (targets.Count == 0) + { + return new QueryStoreIntervalPartitions.StepResult(QueryStoreIntervalPartitions.StepOutcome.NothingToDo, "no valid BRIN index"); + } + + long summarized = 0; + var vacuumed = 0; + var lockSkipped = 0; + var stopped = 0; + var dropped = 0; + foreach (var (index, beingVacuumed) in targets) + { + if (beingVacuumed) + { + vacuumed++; + continue; + } + + try + { + await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await using (var timeouts = new NpgsqlCommand( + $"SET LOCAL lock_timeout = '{lockMs}ms'; SET LOCAL statement_timeout = '{SummarizeBudgetSeconds + SummarizeBackstopSeconds}s';", + connection, + transaction) + { CommandTimeout = CatalogReadTimeoutSeconds }) + { + await timeouts.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + long done; + long notStarted; + await using (var summarize = new NpgsqlCommand(SummarizeIndexSql, connection, transaction) + { CommandTimeout = SummarizeBudgetSeconds + SummarizeBackstopSeconds + CatalogReadTimeoutSeconds }) + { + summarize.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text, Value = index }); + summarize.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Double, Value = (double)SummarizeBudgetSeconds }); + await using var result = await summarize.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + await result.ReadAsync(cancellationToken).ConfigureAwait(false); + done = result.GetInt64(0); + notStarted = result.GetInt64(1); + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + summarized += done; + if (notStarted > 0) + { + stopped++; + logger.LogInformation( + "Query Store interval table {Table}: BRIN summarize of {Index} spent its {Seconds} s budget after {Done} range(s) and stopped between " + + "ranges; those stay summarized and the next pass continues at the first range still missing.", + table.Parent, index, SummarizeBudgetSeconds, done); + } + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.LockNotAvailable) + { + lockSkipped++; + logger.LogDebug( + "Query Store interval table {Table}: BRIN summarize of {Index} gave up after {LockMs} ms without the lock " + + "(deadlock_timeout is {DeadlockMs} ms); the next pass retries.", + table.Parent, index, lockMs, deadlockMs); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.InternalError || ex.SqlState == PostgresErrorCodes.UndefinedTable) + { + /* A leaf the sweep dropped (it drops expired days) after the target list was read: brin_summarize_range cannot open it + and raises XX000 ("could not open relation with OID", index_open in brin.c), not 42P01 (#5594 round 2 L2). XX000 is + also what any internal error says, so the index must read as gone before it counts as dropped. The transaction + is already rolled back here (its scope ended with the try), so the check runs on a clean connection. */ + var stillThere = await IndexExistsAsync(connection, index, cancellationToken).ConfigureAwait(false); + if (!IsDroppedIndexError(ex.SqlState, stillThere)) + { + throw; + } + + dropped++; + logger.LogDebug("Query Store interval table {Table}: BRIN summarize of {Index} found it dropped.", table.Parent, index); + } + catch (PostgresException ex) when (ex.SqlState == PostgresErrorCodes.QueryCanceled && !cancellationToken.IsCancellationRequested) + { + stopped++; + logger.LogInformation( + "Query Store interval table {Table}: BRIN summarize of {Index} was cancelled by its {Seconds} s backstop, stuck inside one range; " + + "what it finished stays summarized, that range stays a placeholder every scan reads until the index is rebuilt, and the next pass continues.", + table.Parent, index, SummarizeBudgetSeconds + SummarizeBackstopSeconds); + } + } + + var detail = $"summarized {summarized} range(s) on {targets.Count - vacuumed - lockSkipped - dropped} index(es); " + + $"skipped {vacuumed} being vacuumed, {lockSkipped} without the lock, {dropped} dropped, {stopped} stopped at the deadline"; + if (summarized > 0) + { + return new QueryStoreIntervalPartitions.StepResult( + QueryStoreIntervalPartitions.StepOutcome.Done, detail, (int)Math.Min(summarized, int.MaxValue)); + } + + return new QueryStoreIntervalPartitions.StepResult( + lockSkipped > 0 ? QueryStoreIntervalPartitions.StepOutcome.RetryLater : QueryStoreIntervalPartitions.StepOutcome.NothingToDo, + detail); + } + + /// + /// Whether a failed summarize means the index was dropped under it: the error is the one brin_summarize_range raises for + /// a missing index (XX000, or 42P01 from a name that no longer resolves) AND the index no longer resolves. Any other error, or + /// an XX000 for an index that is still there, is a real fault and is rethrown (#5594 round 2 L2). + /// + internal static bool IsDroppedIndexError(string? sqlState, bool indexStillExists) => + !indexStillExists + && (sqlState == PostgresErrorCodes.InternalError || sqlState == PostgresErrorCodes.UndefinedTable); + + private static async Task IndexExistsAsync(NpgsqlConnection connection, string index, CancellationToken cancellationToken) + { + await using var command = new NpgsqlCommand("SELECT to_regclass($1) IS NOT NULL", connection) { CommandTimeout = CatalogReadTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Text, Value = index }); + return (bool)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!; + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalPartitions.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalPartitions.cs index 3078f267d9..6d4ea49b22 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalPartitions.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalPartitions.cs @@ -46,13 +46,15 @@ namespace PerformanceMonitor.Darling.Storage; /// off the sweep loop, every hour until shutdown) does the long work: a VALIDATE, which never starts when S is closer than /// (it re-arms first), and the promotion retries. S is the later of and /// the day after the legacy table's newest first_execution_time, read under the arm's own lock through the legacy -/// first-execution index; a maximum more than one retention horizon past the normal S is not armed over (see +/// first-execution index (and once before the lock, to refuse a far-future row without it: #5589); a maximum more than one retention horizon past the normal S is not armed over (see /// : the table stays unpartitioned, with one warning, until that row is gone). A /// VALIDATE that fails with 23514 drops its CHECK and logs that newest time; when the time could be read the same /// background pass arms again with a later S, and when it could not the next pass does. No day partition is created below S ( starts at the newest upper bound, /// which is S while only the legacy table bounds it), and the legacy table is dropped only when S is at or below the cutoff. /// -/// Locks. Arm: ACCESS EXCLUSIVE on legacy for the ADD CONSTRAINT, 5 s lock_timeout, 55P03 retried three +/// Locks. Arm: first an unlocked read of the legacy maximum, which returns not-ready for a row beyond the +/// horizon without any lock request (a request for ACCESS EXCLUSIVE cancels a running autovacuum, and a stuck table would +/// ask hourly: #5589); then ACCESS EXCLUSIVE on legacy for the ADD CONSTRAINT, 5 s lock_timeout, 55P03 retried three /// times 30 s apart. Validate: SHARE UPDATE EXCLUSIVE on legacy (upserts and reads continue), lock_timeout 5 s, a /// 7200 s command deadline. Promote: ACCESS EXCLUSIVE on the parent for the DETACH, ATTACH and the CREATEs, held for /// milliseconds plus a lock wait of 5 s at most. Create-ahead: SHARE UPDATE EXCLUSIVE on the parent and ACCESS @@ -359,7 +361,9 @@ public static DateTime ArmBoundFor(DateTime utcNow, DateTime? legacyMax) /// that ignores it (the CHECK could then never validate). A /// promoted table is never re-armed, so one such row would keep every new row in the legacy table for as long as it /// is dated ahead, and deleting it afterwards would not help. The arm waits until the row is gone instead - /// (#5571 review round 2 M1). A maximum inside the horizon is armed over, with S the day after it. + /// (#5571 review round 2 M1). A maximum inside the horizon is armed over, with S the day after it. The arm checks this + /// before it takes the ACCESS EXCLUSIVE lock as well as under it (#5589): a later row only moves the maximum later, so + /// a verdict without the lock holds under it, and the hourly retry of a stuck table does not cancel autovacuum. /// public static bool IsLegacyMaxBeyondHorizon(DateTime utcNow, DateTime? legacyMax, int horizonDays) { @@ -666,6 +670,20 @@ await ExecuteAsync( } } + /// + /// in its own short transaction that takes no lock beyond the read's ACCESS SHARE + /// (#5589): the arm's far-future check before it requests ACCESS EXCLUSIVE, which would cancel a running autovacuum. + /// The transaction is only there for the transaction-local statement timeout and is rolled back. + /// + private static async Task ReadLegacyMaxUnlockedAsync( + NpgsqlConnection connection, IntervalTable table, ILogger logger, CancellationToken cancellationToken) + { + await using var read = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var max = await ReadLegacyMaxAsync(connection, table, read, logger, cancellationToken).ConfigureAwait(false); + await read.RollbackAsync(cancellationToken).ConfigureAwait(false); + return max; + } + /// /// Phase A step 1. Adds CHECK (first_execution_time < S) NOT VALID to the legacy table, or re-arms a CHECK /// that is still not valid with S closer than . A valid CHECK is left alone here @@ -683,7 +701,10 @@ internal static Task ArmAsync( ArmCoreAsync(connection, table, utcNow, logger, retryDelay, ArmRetries, reArmValid: false, cancellationToken); /// - /// Arm or re-arm in one transaction: lock_timeout, LOCK TABLE legacy IN ACCESS EXCLUSIVE MODE (so no + /// First, before any lock, the far-future check (#5589): the legacy maximum is read with no ACCESS EXCLUSIVE and, when + /// it is beyond the horizon, the step returns without locking (a lock request + /// cancels a running autovacuum, and a table stuck in that state would ask every hour). Otherwise it arms or re-arms + /// in one transaction: lock_timeout, LOCK TABLE legacy IN ACCESS EXCLUSIVE MODE (so no /// row can arrive between the next two reads and the CHECK), the state again (a concurrent arm, re-arm or promotion /// wins and this step does nothing), the legacy maximum, then the DROP of the old CHECK when re-arming and the ADD /// of the new one. lock timeouts are retried apart; the @@ -709,6 +730,19 @@ internal static async Task ArmCoreAsync( return decided; } + /* The far-future check first, with no lock (#5589). A newest time beyond the horizon stays beyond it under the + lock (a later row only moves it later), so a table in that state returns here every hour without ever + requesting ACCESS EXCLUSIVE. That request cancels a running autovacuum, and while the table is unpromoted + the legacy table is the whole live table, so a ~5.5 h autovacuum would never finish. The read under the lock + below stays the one the arm trusts; a null here (no usable index, timeout, empty) falls through to it. */ + var preMax = await ReadLegacyMaxUnlockedAsync(connection, table, logger, cancellationToken).ConfigureAwait(false); + if (IsLegacyMaxBeyondHorizon(utcNow, preMax, table.HorizonDays)) + { + WarnFarFutureMax(logger, table, preMax!.Value, utcNow); + return new StepResult( + StepOutcome.NotReady, $"{table.Legacy} holds a row dated {Literal(preMax.Value)}, too far ahead to arm over"); + } + try { await using var transaction = await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); @@ -1060,6 +1094,10 @@ await ExecuteAsync( } await ExecuteAsync(connection, CreateDefaultSql(table), ShortTimeoutSeconds, transaction, cancellationToken).ConfigureAwait(false); + + /* #5594: the day partitions and DEFAULT cloned the parent's BRIN index with its autosummarize option (on when V171 + made the parent). The LOCK TABLE above holds every partition, so turning the whole family off waits for nothing. */ + await QueryStoreIntervalBrin.TurnOffInTransactionAsync(connection, transaction, table.Parent, cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); logger.LogInformation( "Query Store interval table {Table}: promoted. {Legacy} is bounded below {Bound:O}; {Days} day partition(s) and DEFAULT created.", @@ -1283,6 +1321,11 @@ await ExecuteAsync( ShortTimeoutSeconds, transaction, cancellationToken).ConfigureAwait(false); + + /* #5594: the attach cloned the parent's BRIN index with the parent's autosummarize, which is on when V171 made + the parent. The partitioned parent index cannot be altered, so the new leaf's copy is turned off here, in the + transaction that makes it, while nothing else can hold it. */ + await QueryStoreIntervalBrin.TurnOffInTransactionAsync(connection, transaction, name, cancellationToken).ConfigureAwait(false); await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); created++; drainedTotal += drained; @@ -1425,6 +1468,28 @@ internal static async Task RunMaintenancePassAsync( var failed = 0; foreach (var table in tables) { + /* #5594: first, before any partition work, so the cancel loop ends even when the rest of the pass finds nothing to + do or fails. Its own catch: a BRIN step that cannot run (a lock, a permission) is counted as a failure but never + stops the table's partitions from being maintained. */ + try + { + var brin = await QueryStoreIntervalBrin.TurnOffAutosummarizeAsync(connection, table, logger, cancellationToken).ConfigureAwait(false); + + /* RetryLater still altered the indexes it got to (a lock timeout or a skipped index does not undo the others). */ + if (brin.Outcome is StepOutcome.Done or StepOutcome.RetryLater) + { + changed += brin.Count; + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + failed++; + logger.LogWarning( + "Query Store interval table {Table}: turning BRIN autosummarize off failed ({Message}); the partitions are still maintained and the next pass retries.", + table.Parent, ex.Message); + await ReopenAsync(connection, logger, cancellationToken).ConfigureAwait(false); + } + try { var result = await RunMaintenanceAsync(connection, table, utcNow, logger, cancellationToken).ConfigureAwait(false); @@ -1440,24 +1505,32 @@ internal static async Task RunMaintenancePassAsync( "Query Store interval table {Table}: partition maintenance failed ({Message}); the other table still runs and the next pass retries.", table.Parent, ex.Message); - if (connection.State != System.Data.ConnectionState.Open) - { - try - { - await connection.CloseAsync().ConfigureAwait(false); - await connection.OpenAsync(cancellationToken).ConfigureAwait(false); - } - catch (Exception reopen) when (reopen is not OperationCanceledException) - { - logger.LogWarning("Query Store interval partition maintenance could not reopen its connection: {Message}", reopen.Message); - } - } + await ReopenAsync(connection, logger, cancellationToken).ConfigureAwait(false); } } return new PassResult(changed, failed); } + /* A failure that closed the connection leaves the next table (and the next step) nothing to run on, so it is reopened. */ + private static async Task ReopenAsync(NpgsqlConnection connection, ILogger logger, CancellationToken cancellationToken) + { + if (connection.State == System.Data.ConnectionState.Open) + { + return; + } + + try + { + await connection.CloseAsync().ConfigureAwait(false); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception reopen) when (reopen is not OperationCanceledException) + { + logger.LogWarning("Query Store interval partition maintenance could not reopen its connection: {Message}", reopen.Message); + } + } + /// /// The background task (#5571), launched at start, never awaited on the startup path and never throwing: wait /// , then loop until shutdown, every : Phase A for each table @@ -1492,7 +1565,12 @@ public static Task RunDelayedAsync( return await AnalyzeIfDueAsync(connection, table, DateTime.UtcNow, logger, token).ConfigureAwait(false); }, token => QueryStoreBackgroundIndexes.RunDelayedAsync(postgres, logger, TimeSpan.Zero, specs, token), - cancellationToken); + cancellationToken, + async (table, token) => + { + await using var connection = await postgres.OpenConnectionAsync(token).ConfigureAwait(false); + return await QueryStoreIntervalBrin.SummarizeNewRangesAsync(connection, table, logger, token).ConfigureAwait(false); + }); /// with the three actions injected, so the order, the repetition and the isolation run without a store. internal static async Task RunDelayedAsync( @@ -1503,7 +1581,8 @@ internal static async Task RunDelayedAsync( Func> promote, Func> analyze, Func ensureIndexes, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + Func>? summarize = null) { try { @@ -1545,6 +1624,16 @@ internal static async Task RunDelayedAsync( await TryStepAsync(logger, table, "analyze", analyze, quiet: true, cancellationToken).ConfigureAwait(false); } + /* #5594: the BRIN indexes are summarized here, hourly, because autosummarize is off and a VACUUM only summarizes + when it finishes. Never the waiter that cancels an autovacuum (see QueryStoreIntervalBrin). */ + if (summarize is not null) + { + foreach (var table in tables) + { + await TryStepAsync(logger, table, "BRIN summarize", summarize, quiet: true, cancellationToken).ConfigureAwait(false); + } + } + await Task.Delay(interval, cancellationToken).ConfigureAwait(false); } } diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalWideBrinIndex.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalWideBrinIndex.cs index a0538cce31..8fd9163629 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalWideBrinIndex.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreIntervalWideBrinIndex.cs @@ -27,7 +27,8 @@ namespace PerformanceMonitor.Darling.Storage; /// rows, so BRIN ranges widen: windows ending now should stay selective, while historical windows lose /// selectivity and the plan may flip back to a Seq Scan for part of the cycle. Results stay exact and /// nothing is slower than without the index. If that matters, the remedy is a periodic -/// REINDEX INDEX CONCURRENTLY or re-summarizing the ranges, which this class does not do. +/// REINDEX INDEX CONCURRENTLY, which this class does not do. (The hourly summarize of new ranges is +/// 's; it does not re-summarize a range that widened.) /// /// Why BRIN and not a btree. The writer's upsert (QueryStoreIntervalWide.cs, /// ON CONFLICT ... DO UPDATE SET collection_time = EXCLUDED.collection_time, ...) rewrites @@ -55,9 +56,14 @@ namespace PerformanceMonitor.Darling.Storage; /// on a hypertable, so if the table is ever converted the ensure skips with a warning instead of failing /// or retrying. /// -/// Autosummarize. autosummarize = on lets autovacuum work items summarize each new block -/// range as the table grows. A range that is not summarized yet is always read by the scan, so results stay -/// correct while the summary catches up; the index is 856 kB on that store. +/// Autosummarize is off (#5594; #4862 turned it on). autosummarize = on lets an autovacuum work item +/// summarize each new block range, but the item waits for the table's SHARE UPDATE EXCLUSIVE lock, the one a +/// running VACUUM holds, and after deadlock_timeout PostgreSQL cancels that VACUUM. After a big retention drain the +/// table's vacuum was cancelled every couple of minutes and never finished. Off, a VACUUM summarizes new ranges when it +/// ends and summarizes hourly in between, without ever +/// cancelling a vacuum. A range that is not summarized yet is always read by the scan, so results stay correct while the +/// summary catches up; the index is 856 kB on that store. An index built with on by an earlier build is turned off +/// by . /// /// The start delay, the validity read and the build. They are /// 's, shared with the btree #4952 adds: it waits @@ -83,7 +89,7 @@ public static class QueryStoreIntervalWideBrinIndex internal const string CreateSql = "CREATE INDEX CONCURRENTLY IF NOT EXISTS ix_query_store_interval_wide_collection_time_brin " - + "ON collect.query_store_interval_wide USING brin (collection_time) WITH (autosummarize = on);"; + + "ON collect.query_store_interval_wide USING brin (collection_time) WITH (autosummarize = off);"; internal const string DropSql = "DROP INDEX CONCURRENTLY IF EXISTS collect.ix_query_store_interval_wide_collection_time_brin;"; @@ -99,7 +105,7 @@ public static class QueryStoreIntervalWideBrinIndex DropSql, MinimumServerVersionNum, "a BRIN index on collection_time would make the upsert non-HOT below PG 16", - "USING brin (collection_time) WITH (autosummarize = on)"); + "USING brin (collection_time) WITH (autosummarize = off)"); /// What the ensure does about the index. public enum BrinAction