Read the interval table below the raw floor, down to the exact bound it can prove, and say how far back it reaches (#4689)

Darling/Darling.Tests/ComposeParameterCoverageTests.cs

@@ -142,18 +142,28 @@ public sealed class ComposeParameterCoverageTests
     /// restated as a function of the author's intent rather than read back off the compiler's own output.
     /// Two for the naive-UTC window, one more when the run names servers, one per filter (every
     /// <c>BuildFilterClause</c> arm binds exactly one value, whether a text array or a scalar), and one for
-    /// <c>topN</c> in the two ranked modes.
+    /// <c>topN</c> in the two ranked modes, and one for the Query Store wide-start bound (#4689) exactly when
+    /// the run is wide-eligible with a start later than the window's and the measure reads the Query Store table.
     ///
     /// <para>An extra bind anywhere raises what the compiler produced and leaves this where it was, which is
     /// what makes it see the spelling coverage cannot. It is deliberately NOT derived from
     /// <c>Parameters.Count</c> — a prediction taken from the thing it is checking agrees with it always.</para>
     /// </summary>
-    internal static int PredictedParameterCount(PanelPlan plan, bool serverScoped) =>
+    internal static int PredictedParameterCount(PanelPlan plan, bool serverScoped, ComposeRunContext context) =>
         2
         + (serverScoped ? 1 : 0)
+        + (PredictsWideStartBind(plan, context) ? 1 : 0)
         + plan.Filters.Count
         + (plan.Mode is PanelMode.Ranked or PanelMode.RankedTimeSeries ? 1 : 0);
 
+    /// <summary>The #4689 rule from intent alone: a wide-eligible run whose common start is later than the
+    /// window start, reading the Query Store table. Not read back from the compiler's parameters.</summary>
+    internal static bool PredictsWideStartBind(PanelPlan plan, ComposeRunContext context) =>
+        context.QueryStoreWideEligible
+        && context.QueryStoreWideStart is DateTime wideStart
+        && wideStart > context.StartUtc
+        && string.Equals(plan.Measure.SourceTable, "query_store_stats", StringComparison.Ordinal);
+
     /// <summary>
     /// The same prediction for an annotation query, whose whole parameter set is the window plus the
     /// optional server scope: the catalog supplies its table, time column and label column as compiler
@@ -234,6 +244,13 @@ same one. Window-only through window + scope + two filters + topN is five distin
 
         Assert.True(corpus.Any(c => c.UsesVariable), "the corpus compiled no variable-resolved filter");
 
+        /* #4689: the wide-start bind must be in the population, both bound and (for a start that is not later)
+           not bound, or the sweep could lose the rule without noticing. */
+        Assert.True(corpus.Any(c => c.WideStartBound), "the corpus compiled no statement carrying the Query Store wide-start bind");
+        Assert.True(
+            corpus.Any(c => !c.WideStartBound && c.Label.Contains(" wide:", StringComparison.Ordinal)),
+            "the corpus compiled no wide-eligible Query Store statement whose start was not later than the window");
+
         /* Parameter-free statements cannot exhibit the defect, so a corpus that had quietly become all
            fleet-wide-scalar would pass the sweep having asserted nothing about binding at all. */
         Assert.True(
@@ -311,11 +328,12 @@ compiler is built on — look like the defect. */
     /// <summary>
     /// <see cref="PredictedParameterCount"/> restates a rule that lives in another file, so it can be
     /// outgrown. This counts the <c>ParamList</c> call sites in <c>ComposeCompiler.cs</c> and pins the
-    /// total: fifteen, which is the three window/scope binds and one <c>topN</c> per ranked arm in
-    /// <c>Compile</c>, the seven <c>BuildFilterClause</c> operator arms, and the three window/scope binds in
+    /// total: sixteen, which is the three window/scope binds and one <c>topN</c> per ranked arm in
+    /// <c>Compile</c>, the Query Store wide-start bind in <c>Compile</c> when the run reads the interval table
+    /// from a later start, the seven <c>BuildFilterClause</c> operator arms, and the three window/scope binds in
     /// <c>CompileAnnotation</c>.
     ///
-    /// <para>A sixteenth is the "next site someone adds" case, and it reds HERE — where the fix is to decide
+    /// <para>A seventeenth is the "next site someone adds" case, and it reds HERE — where the fix is to decide
     /// whether the prediction grows with it — rather than in the sweep, where it would read as a compiler
     /// bug. Comments and string literals are stripped first, because this file's reasoning names
     /// <c>p.AddTextArray</c> in prose.</para>
@@ -334,7 +352,7 @@ public void ThePredictionCoversEveryBindingSite_InTheCompiler()
         var code = CSharpSourceWalker.StripCommentsAndStrings(File.ReadAllText(path));
         var sites = Regex.Matches(code, @"\bp\.Add[A-Za-z]+\s*\(").Count;
 
-        Assert.Equal(15, sites);
+        Assert.Equal(16, sites);
     }
 
     /// <summary>
@@ -389,7 +407,8 @@ private sealed record Statement(
         string? Measure,
         string? Annotation,
         string? FilterOp,
-        bool UsesVariable);
+        bool UsesVariable,
+        bool WideStartBound = false);
 
     /// <summary>
     /// Every measure in every mode its archetype and dimensions allow, fleet-wide and server-scoped; every
@@ -495,6 +514,33 @@ placeholder and nothing about the filter itself looks wrong. */
             }
         }
 
+        /* #4689: Query Store statements compiled wide-eligible. A start LATER than the window binds one extra
+           timestamp; a start that is not later binds nothing extra. */
+        foreach (var measure in MeasureCatalog.Measures.Where(m => string.Equals(m.SourceTable, "query_store_stats", StringComparison.Ordinal)))
+        {
+            var keyword = measure.Kind == MeasureKind.Ratio ? "ratio" : "measure";
+            var aggregate = measure.Kind == MeasureKind.Ratio ? null : MeasureCatalog.WireName(measure.ValidAggs[0]);
+            var aggregateJson = aggregate is null ? "" : $",\"aggregate\":\"{aggregate}\"";
+            var head = $"{{\"source\":\"{measure.SourceTable}\",\"{keyword}\":\"{measure.Key}\"{aggregateJson}";
+            var dimension = measure.AllowedDimensions.Count > 0 ? measure.AllowedDimensions[0] : "server";
+
+            foreach (var wideStart in new DateTime?[] { WindowStart.AddHours(2), WindowStart, WindowStart.AddHours(-1) })
+            {
+                foreach (var servers in new[] { (IReadOnlyList<string>?)null, TwoServers })
+                {
+                    Add(corpus, $"{head},\"timeBucket\":\"hour\",\"viz\":\"line\"}}", servers, measure.Key, wide: (true, wideStart));
+                    Add(corpus, $"{head},\"viz\":\"stat\"}}", servers, measure.Key, wide: (true, wideStart));
+                    Add(corpus, $"{head},\"topN\":5,\"groupBy\":[\"{dimension}\"],\"viz\":\"bar\"}}", servers, measure.Key, wide: (true, wideStart));
+                    Add(
+                        corpus,
+                        $"{head},\"timeBucket\":\"hour\",\"topN\":5,\"groupBy\":[\"{dimension}\"],\"includeOther\":true,\"viz\":\"line\"}}",
+                        servers,
+                        measure.Key,
+                        wide: (true, wideStart));
+                }
+            }
+        }
+
         /* A variable-resolved filter value: the same bind, reached through the $var path rather than a
            literal, so a future divergence between the two resolutions is in the population. */
         Add(
@@ -544,7 +590,8 @@ private static void Add(
         string measureKey,
         string? filterOp = null,
         IReadOnlyDictionary<string, string?>? variables = null,
-        string[]? declaredVariables = null)
+        string[]? declaredVariables = null,
+        (bool Eligible, DateTime? Start)? wide = null)
     {
         var (plan, parseError) = ComposeSpec.TryParsePanel(
             (JsonObject)JsonNode.Parse(json)!,
@@ -557,36 +604,42 @@ private static void Add(
             return;
         }
 
-        var (compiled, compileError) = ComposeCompiler.Compile(plan!, Context(servers, variables));
+        var context = Context(servers, variables, wide);
+        var (compiled, compileError) = ComposeCompiler.Compile(plan!, context);
         if (compileError is not null)
         {
             return;
         }
 
         corpus.Add(new Statement(
             $"{measureKey} {plan!.Mode} ({(servers is null ? "fleet" : "scoped")})"
+            + (wide is null ? "" : $" wide:{wide.Value.Start:HH:mm}")
             + (filterOp is null ? "" : $" filter:{filterOp}"),
             compiled!,
-            PredictedParameterCount(plan, servers is not null),
+            PredictedParameterCount(plan, servers is not null, context),
             plan.Mode,
             servers is not null,
             measureKey,
             Annotation: null,
             filterOp,
-            variables is not null));
+            variables is not null,
+            PredictsWideStartBind(plan, context)));
     }
 
     private static ComposeRunContext Context(
         IReadOnlyList<string>? servers,
-        IReadOnlyDictionary<string, string?>? variables) =>
+        IReadOnlyDictionary<string, string?>? variables,
+        (bool Eligible, DateTime? Start)? wide = null) =>
         new(
             servers,
             WindowStart,
             WindowEnd,
             variables ?? ComposeRunContext.NoVariables,
             RollupAvailability.All,
             WindowEnd,
-            RollupCoverage.Unknown);
+            RollupCoverage.Unknown,
+            QueryStoreWideEligible: wide?.Eligible ?? false,
+            QueryStoreWideStart: wide?.Start);
 
     /// <summary>The repository root, from this file's own compile-time path — the same anchor
     /// <c>StartupCommandTimeoutTests</c> and <c>ServerLocalReadFrameDisciplineTests</c> use, so the source

Darling/Darling.Tests/ComposeQueryStoreWideStartLiveTests.cs

@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json.Nodes;
+using System.Threading;
+using System.Threading.Tasks;
+using Npgsql;
+using PerformanceMonitor.Collectors;
+using PerformanceMonitor.Darling.Service;
+using PerformanceMonitor.Darling.Storage;
+using Xunit;
+
+namespace Darling.Tests;
+
+/// <summary>
+/// #4689: a composed Query Store panel that reads the interval table below raw's chunk floor binds the runner's
+/// exact start (<see cref="ComposeRunContext.QueryStoreWideStart"/>), not the window start. Seeds through the
+/// real write path, forces <c>filled_since</c> to S + 12 h so the seed's backdated table rows (S + 1 h) sit
+/// outside the coverage claim, drops raw's older chunks, then runs the panel the way the endpoint does.
+/// </summary>
+/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. The test reaches DARLING_TEST_PG only to
+   CREATE and DROP its own database through ScratchPostgres and works entirely inside it. */
+public sealed class ComposeQueryStoreWideStartLiveTests
+{
+    private const int ServerId = -4689002;
+    private const string ServerName = "qsiw-compose-start";
+
+    private static readonly DateTime S = DateTime.SpecifyKind(DateTime.UtcNow.Date.AddDays(-6), DateTimeKind.Unspecified);
+
+    private const string PanelJson =
+        "{\"source\":\"query_store_stats\",\"measure\":\"qs_executions\",\"aggregate\":\"sum\",\"timeBucket\":\"hour\",\"viz\":\"line\"}";
+
+    [Fact]
+    public async Task ComposePanel_BindsTheExactWideStart_NotTheWindowStart()
+    {
+        var baseCs = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+        Assert.SkipWhen(string.IsNullOrEmpty(baseCs), "Set DARLING_TEST_PG to a Postgres connection string to run the #4689 Compose live test.");
+        var ct = TestContext.Current.CancellationToken;
+
+        await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct);
+        await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+        await connection.OpenAsync(ct);
+        await PgMigrations.MigrateAsync(connection, ct);
+        Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "TimescaleDB must be enabled on the test cluster");
+        await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct);
+
+        await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString);
+        var runner = new DarlingCollectorRunner(postgres, new CollectorDeltaCalculator());
+        await QueryStoreIntervalWideGridLiveTests.SeedGridAsync(runner, ServerId, S, ct);
+        await ExecAsync(connection,
+            "INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major_version, created_date, modified_date) "
+            + $"VALUES ({ServerId}, '{ServerName}', '{ServerName}', TRUE, 16, now(), now()) ON CONFLICT (server_id) DO UPDATE SET is_enabled = TRUE", ct);
+        await QueryStoreIntervalWideBelowFloorLiveTests.SeedQueryStoreLogAsync(connection, ServerId, S.AddDays(-60), S.AddDays(4), null, null, ct);
+        await ExecAsync(connection,
+            $"UPDATE collect.query_store_interval_wide_coverage SET filled_since = TIMESTAMP '{S.AddHours(12):yyyy-MM-dd HH:mm:ss}' WHERE server_id = {ServerId}", ct);
+
+        var end = (DateTime)(await ScalarAsync(connection,
+            $"SELECT applied_through FROM collect.query_store_interval_wide_coverage WHERE server_id = {ServerId}", ct))!;
+        await ExecAsync(connection,
+            $"SELECT drop_chunks('collect.query_store_stats', older_than => TIMESTAMP '{S.AddDays(1):yyyy-MM-dd HH:mm:ss}')", ct);
+
+        /* The runner's own eligibility step, through the product path. */
+        var resolution = await DarlingWebEndpoints.ResolveQueryStoreWideEligibleAsync(postgres, null, S, end, end, ct);
+        Assert.True(resolution.Eligible);
+        Assert.Equal(S.AddHours(12), resolution.WideStart);
+        Assert.Equal(QueryStoreIntervalWide.WideStartBound.FilledSince, resolution.Bound);
+
+        var rollups = await TimescaleSupport.DetectRollupsAsync(postgres, ct);
+        var coverage = await TimescaleSupport.DetectRollupCoverageAsync(postgres, rollups, ct);
+        var (plan, parseError) = ComposeSpec.TryParsePanel((JsonObject)JsonNode.Parse(PanelJson)!, Array.Empty<string>());
+        Assert.True(parseError is null, parseError);
+
+        var served = await RunBucketsAsync(connection, plan!,
+            new ComposeRunContext(null, S, end, ComposeRunContext.NoVariables, rollups, end, coverage, resolution.Eligible, resolution.WideStart), ct);
+        Assert.NotEmpty(served);
+        Assert.True(served.Min() >= S.AddHours(12), $"the panel must not include intervals below filled_since; earliest bucket {served.Min():o}");
+
+        /* The window-start bind (what the panel did before the exact start was bound) does include the
+           backdated table rows, so the bound above is what excludes them. */
+        var unbound = await RunBucketsAsync(connection, plan!,
+            new ComposeRunContext(null, S, end, ComposeRunContext.NoVariables, rollups, end, coverage, true), ct);
+        Assert.True(unbound.Min() < S.AddHours(12), "expected the unbound read to include the backdated table rows");
+
+        Assert.Equal(S.AddHours(12), resolution.WideStart);
+        Assert.Equal(ServerName, resolution.SettingServer);
+        var note = DarlingWebEndpoints.QueryStoreHistoryNote(resolution.WideStart!.Value, resolution.Bound, resolution.SettingServer);
+        Assert.Contains("interval table began keeping complete history for " + ServerName, note, StringComparison.Ordinal);
+    }
+
+    private static async Task<List<DateTime>> RunBucketsAsync(NpgsqlConnection connection, PanelPlan plan, ComposeRunContext context, CancellationToken ct)
+    {
+        var (compiled, error) = ComposeCompiler.Compile(plan, context);
+        Assert.True(error is null, error);
+        var buckets = new List<DateTime>();
+        await using var command = new NpgsqlCommand(compiled!.Sql, connection);
+        foreach (var p in compiled.Parameters)
+        {
+            command.Parameters.Add(p);
+        }
+
+        await using var reader = await command.ExecuteReaderAsync(ct);
+        while (await reader.ReadAsync(ct))
+        {
+            buckets.Add(reader.GetDateTime(0));
+        }
+
+        return buckets;
+    }
+
+    private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct)
+    {
+        await using var command = new NpgsqlCommand(sql, connection);
+        await command.ExecuteNonQueryAsync(ct);
+    }
+
+    private static async Task<object?> ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken ct)
+    {
+        await using var command = new NpgsqlCommand(sql, connection);
+        return await command.ExecuteScalarAsync(ct);
+    }
+}