Darling builds a background index so get_query_store_top no longer reads every entry a server holds (#4952)

Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexLiveTests.cs

@@ -45,13 +45,13 @@ private static async Task<NpgsqlConnection> OpenMigratedAsync(ScratchPostgres sc
         return connection;
     }
 
-    private static async Task<object?> ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct)
+    internal static async Task<object?> ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct)
     {
         await using var command = new NpgsqlCommand(sql, connection);
         return await command.ExecuteScalarAsync(ct);
     }
 
-    private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct)
+    internal static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct)
     {
         await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
         await command.ExecuteNonQueryAsync(ct);
@@ -79,7 +79,7 @@ FROM generate_series(1, {rows}) AS g
         "collection_time = collection_time + interval '1 minute', last_execution_time = last_execution_time + interval '1 minute', "
         + "execution_count = execution_count + 1, avg_duration_us = avg_duration_us + 1";
 
-    private static async Task<(long Updated, long Hot)> UpdateAndReadHotAsync(
+    internal static async Task<(long Updated, long Hot)> UpdateAndReadHotAsync(
         NpgsqlConnection connection, string setList, string where, CancellationToken ct)
     {
         /* The pg_stat_xact_* counters are the session's still-pending counts, so an earlier measurement on this
@@ -165,7 +165,7 @@ public async Task TheUpsertsCollectionTimeRewrite_StaysHotWithTheBrin_AndNotWith
     }
 
     /// <summary>The column names the product's upsert sets in <c>DO UPDATE SET</c>, parsed from its own SQL.</summary>
-    private static HashSet<string> UpsertSetColumns()
+    internal static HashSet<string> UpsertSetColumns()
     {
         var sql = QueryStoreIntervalWide.UpsertSql;
         var start = sql.IndexOf("DO UPDATE SET", StringComparison.Ordinal);
@@ -230,7 +230,16 @@ public async Task NoNonBrinIndex_CoversAColumnTheUpsertSets_SoTheUpsertStaysHotF
 
         await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct);
         await using var connection = await OpenMigratedAsync(scratch, ct);
-        await EnsureAsync(connection, ct);
+
+        /* Every background index the service builds, not the BRIN alone: the census reads the catalog after the product's
+           own list has been ensured, so the next member of QueryStoreBackgroundIndexes.All is scanned without an edit here. */
+        await QueryStoreBackgroundIndexesLiveTests.EnsureAllAsync(connection, ct);
+        foreach (var spec in QueryStoreBackgroundIndexes.All)
+        {
+            Assert.True(
+                (bool)(await ScalarAsync(connection, $"SELECT indisvalid FROM pg_index WHERE indexrelid = '{spec.IndexName}'::regclass", ct))!,
+                $"{spec.IndexName} must be built and valid before the census reads the catalog");
+        }
 
         /* Data-independent: the catalog, not a seeded update. The scan must see the identity index and the
            first_execution_time index, or it reads nothing. */

Darling/Darling.Tests/QueryStoreIntervalWideBrinIndexTests.cs

@@ -66,7 +66,8 @@ public void TheBuildSql_IsConcurrent_Idempotent_AndABrinOnCollectionTime()
     [Fact]
     public void TheEnsure_ReadsValidityFirst_AndGuardsHypertablesThroughAViewCheck()
     {
-        var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Storage", "QueryStoreIntervalWideBrinIndex.cs");
+        /* The machinery moved to QueryStoreBackgroundIndexes (#4952) so the two new btrees share it; the pins moved with it. */
+        var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Storage", "QueryStoreBackgroundIndexes.cs");
 
         /* The validity read is what tells an INVALID leftover from a good index; IF NOT EXISTS alone keeps both. */
         Assert.Contains("i.indisvalid", source);
@@ -75,7 +76,7 @@ public void TheEnsure_ReadsValidityFirst_AndGuardsHypertablesThroughAViewCheck()
         /* The hypertable catalog view is queried only after to_regclass says it exists (no TimescaleDB, no view). */
         Assert.Contains("to_regclass('timescaledb_information.hypertables')", source);
         Assert.Contains("if (hasHypertableView)", source);
-        Assert.Contains("BrinAction.SkipHypertable", source);
+        Assert.Contains("IndexAction.SkipHypertable", source);
         Assert.Contains("LogWarning", source);
     }
 
@@ -85,14 +86,14 @@ public void TheWorker_SchedulesTheEnsureAfterMigrations_WithoutAwaitingItOnTheSt
         var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Service", "DarlingWorker.cs").Replace("\r\n", "\n");
 
         var migrate = source.IndexOf("PgMigrations.MigrateAsync(migrateConnection", System.StringComparison.Ordinal);
-        var launch = source.IndexOf("var intervalWideBrin = QueryStoreIntervalWideBrinIndex.RunDelayedAsync(", System.StringComparison.Ordinal);
-        var drain = source.IndexOf("await intervalWideBrin;", System.StringComparison.Ordinal);
+        var launch = source.IndexOf("var queryStoreIndexes = QueryStoreBackgroundIndexes.RunDelayedAsync(", System.StringComparison.Ordinal);
+        var drain = source.IndexOf("await queryStoreIndexes;", System.StringComparison.Ordinal);
         var loopStop = source.IndexOf("PerformanceMonitor Darling collection loop stopped", System.StringComparison.Ordinal);
 
         Assert.True(migrate > 0 && launch > migrate, "the ensure must launch after migrations");
         Assert.True(drain > launch && drain < loopStop, "the ensure is drained only at shutdown, after the collection loop");
-        Assert.Single(Regex.Matches(source, @"await intervalWideBrin;"));
-        Assert.Contains("QueryStoreIntervalWideBrinIndex.StartDelay", source);
+        Assert.Single(Regex.Matches(source, @"await queryStoreIndexes;"));
+        Assert.Contains("QueryStoreBackgroundIndexes.StartDelay", source);
     }
 
     [Fact]
@@ -103,7 +104,7 @@ public async Task RunDelayed_CancelledDuringTheDelay_LogsAtDebug_NotInformation(
         using var cts = new CancellationTokenSource();
         cts.CancelAfter(TimeSpan.FromMilliseconds(200));
 
-        await QueryStoreIntervalWideBrinIndex.RunDelayedAsync(dataSource, logger, TimeSpan.FromMinutes(20), cts.Token);
+        await QueryStoreBackgroundIndexes.RunDelayedAsync(dataSource, logger, TimeSpan.FromMinutes(20), QueryStoreBackgroundIndexes.All, cts.Token);
 
         var entry = Assert.Single(logger.Entries);
         Assert.Equal(LogLevel.Debug, entry.Level);
@@ -116,7 +117,7 @@ public void RunDelayed_KeepsTheInformationLineOnlyForACancelAfterTheDelay()
         /* A cancel that lands after the delay needs a connection held open mid-handshake; Npgsql then may report
            the stop as its own exception, so that branch is pinned on the source: the flag is set right after the
            delay, the Debug branch is filtered on it, and the Information branch is the unfiltered catch. */
-        var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Storage", "QueryStoreIntervalWideBrinIndex.cs")
+        var source = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Storage", "QueryStoreBackgroundIndexes.cs")
             .Replace("\r\n", "\n");
 
         var delay = source.IndexOf("await Task.Delay(delay, cancellationToken)", System.StringComparison.Ordinal);

Darling/Darling.Tests/QueryStoreIntervalWideFirstExecFloorTests.cs

@@ -17,12 +17,14 @@
 namespace Darling.Tests;
 
 /// <summary>
-/// #4605: among the indexes of <c>collect.query_store_interval_wide</c> are the unique key (it leads with
-/// <c>server_id</c> and holds <c>first_execution_time</c> as a key column) and
-/// <c>idx_query_store_interval_wide_first_exec</c>. The three per-server reads of the table (the MCP Query Store top,
+/// #4605: among the btrees of <c>collect.query_store_interval_wide</c> are the unique key (it leads with
+/// <c>server_id</c> and holds <c>first_execution_time</c> as a key column),
+/// <c>idx_query_store_interval_wide_first_exec</c>, and, where the service has built it, the wide btree on
+/// <c>(server_id, first_execution_time)</c> (#4952), which the service builds in the background rather than a
+/// migration. The three per-server reads of the table (the MCP Query Store top,
 /// the Queries grid and the Trends chart, all <c>WHERE server_id = $1</c>) filter <c>collection_time</c> (the Trends
-/// interval arm <c>interval_start_time_utc</c>), which neither serves, so each walked all of the server's rows. Each
-/// now also carries
+/// interval arm <c>interval_start_time_utc</c>), which none of them serves, so each walked all of the server's rows.
+/// Each now also carries
 /// <c>first_execution_time &gt;= &lt;window start&gt; - </c><see cref="QueryStoreIntervalWide.PurgeEdgeMarginSql"/>.
 /// The Custom Views route, for all servers or some, carries no floor; its compose text is pinned in
 /// <c>DarlingComposeTests.Compile_QueryStoreWideEligible_CarriesNoFirstExecutionTimeFloor</c>. This class pins the

Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs

@@ -2311,12 +2311,14 @@ startup work below. */
            its own connection, its own catch, drained with the other background startup work below. */
         var planForceDetailScrub = RunPlanForceActionDetailScrubAsync(postgres, stoppingToken);
 
-        /* #4605: the BRIN index on collect.query_store_interval_wide (collection_time), built CONCURRENTLY in the
-           background QueryStoreIntervalWideBrinIndex.StartDelay after start so the full-heap read stays off the
-           post-restart IO burst. Launched after migrations confirm the table exists, never awaited on the startup
-           path, one attempt per start, and RunDelayedAsync never throws. Drained with the other background work. */
-        var intervalWideBrin = QueryStoreIntervalWideBrinIndex.RunDelayedAsync(
-            postgres, _logger, QueryStoreIntervalWideBrinIndex.StartDelay, stoppingToken);
+        /* #4605, #4952: the Query Store read indexes - the BRIN on collect.query_store_interval_wide (collection_time)
+           and the btree on its (server_id, first_execution_time) - built in the background
+           QueryStoreBackgroundIndexes.StartDelay after start so their heap reads stay off the post-restart IO burst,
+           one after another, each failure-isolated. Launched after migrations confirm the tables exist, never awaited
+           on the startup path, one attempt per start, and RunDelayedAsync never throws. Drained with the other
+           background work. */
+        var queryStoreIndexes = QueryStoreBackgroundIndexes.RunDelayedAsync(
+            postgres, _logger, QueryStoreBackgroundIndexes.StartDelay, QueryStoreBackgroundIndexes.All, stoppingToken);
 
         /* #4957: one rollup-coverage probe in the background RollupCoverageWarmup.ServiceStartDelay after start, so
            the first MCP or web call finds each rollup's floor already measured and does not wait ~10 s on the
@@ -3818,8 +3820,8 @@ it throws is observed rather than lost with the task. */
             /* Expected on shutdown. */
         }
 
-        /* And the interval-wide BRIN index ensure (#4605), which absorbs its own failures. */
-        await intervalWideBrin;
+        /* And the Query Store read index ensures (#4605, #4952), which absorb their own failures. */
+        await queryStoreIndexes;
 
         /* And the rollup-coverage warm (#4957), which also absorbs its own failures. */
         await rollupCoverageWarm;