Query Store compose rollup: a failing hour backs off to once a day, with a longer timeout after a timeout (#5867)

Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs

@@ -923,8 +923,10 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
            DarlingWorker. 16th exempt since #4750: EvaluateNotificationChannelsAsync's wrapper catch — the
            webhook channels' failure counts are read from the webhook service's memory, so there is no store
            read for the condition to swallow. 19th exempt since #5654: EvaluateExternalAlertsAsync's wrapper
-           catch - the delivery self-alert's report is a parameter, with the count read counted in DarlingWorker. */
-        (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 13, 19),
+           catch - the delivery self-alert's report is a parameter, with the count read counted in DarlingWorker. 20th exempt since
+           #5867: EvaluateComposeHoursStuckAsync's wrapper catch - the compose rollup self-alert's failure record is a parameter,
+           with the read exempt in DarlingWorker (it holds the alert on a failed read). */
+        (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 13, 20),
     };
 
     /// <summary>
@@ -962,6 +964,9 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
            a swallowed read would leave the standing alert unjudged, and the worker holds it (sends no report) rather than
            resolving it. */
         "EvaluateExternalAlertsAsync",
+        /* #5867: the compose rollup self-alert's one store read, the compose builder's failure record. EXEMPT: a failed read
+           sends no report, so the worker holds the standing alert rather than resolving it, and the next hourly tick reads again. */
+        "EvaluateComposeHoursStuckAsync",
         /* #3354: the mute-rule reload. Not an alert pass — a control-plane read — but its swallowed
            failure decides what the engine suppresses on every following sweep, so it belongs to this
            population. It is also the member whose scope this list was designed to admit: an alerting read
@@ -975,8 +980,10 @@ resolving it. */
        not exempt, because a RejectedValue row is judgeable evidence for the alert, not context for its text. */
     private const int WorkerCountedSites = 12;
     /* 13th: the deadlock text repair pass's catch in SweepStoreSelfMetricsAsync, exempt like the re-mask pass beside it.
-       14th: the collection log hourly rollup's catch there (#5810), a maintenance write no alert is judged on. */
-    private const int WorkerExemptSites = 14;
+       14th: the collection log hourly rollup's catch there (#5810), a maintenance write no alert is judged on.
+       15th: EvaluateComposeHoursStuckAsync's catch (#5867), the compose builder's failure record read for its self-alert, which
+       holds the standing alert on a failed read. */
+    private const int WorkerExemptSites = 15;
 
     /// <summary>
     /// Counted sites tree-wide. ONE numeral with several readers rather than the same number written out at
@@ -1055,6 +1062,8 @@ resolving it. */
         ["Store-job cadence self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker",
         ["Retention-held self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker",
         ["Raw-purge-over-horizon self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker",
+        ["Compose-rollup-hours-stuck self-alert failed"] = "handed its evidence as parameters; the read is isolated in DarlingWorker.EvaluateComposeHoursStuckAsync",
+        ["Query Store compose rollup: the failure record could not be read for the self-alert"] = "an unreadable failure record sends no report, so the alert's standing state is held as it was, and the next hourly compose tick reads it again",
         ["Notification-channel self-alert failed"] = "reads the webhook channels' failure counts from the webhook service's memory and performs no store read at all - there is no read for this condition to be the swallowing of",
         ["Stale-mute self-alert failed"] = "handed its evidence (the live MuteRuleService cache) as a parameter and performs no store read at all - there is no read anywhere for this condition to be the swallowing of",
         ["Fleet gate self-alert failed"] = "handed its evidence (the worker's in-memory gate counts) as a parameter and performs no store read at all",
@@ -1269,8 +1278,10 @@ WebTlsCertificateState report the web host publishes - there is no store read to
            its evidence as a parameter. 37th: the deadlock text repair pass, the second stage beside the re-mask pass (the same
            maintenance rewrite, marker-gated, resumed next hour). 38th since #5654: the external-alert delivery self-alert's
            wrapper catch, handed its report as a parameter. 39th since #5810: the collection log hourly rollup's catch, a
-           maintenance write no alert is judged on, redone by the next pass. */
-        Assert.Equal(39, totalExempt);
+           maintenance write no alert is judged on, redone by the next pass. 40th since #5867: the Compose Rollup Hours Stuck
+           self-alert's wrapper catch, handed its failure record as a parameter. 41st since #5867: the worker's read of that
+           record, which holds the standing alert on a failure. */
+        Assert.Equal(41, totalExempt);
 
         /* Every exemption in the table is actually used. An exemption for a message that no longer exists
            is a hole this pin would otherwise keep open indefinitely — the shape that lets a real new catch

Darling/Darling.Tests/AlertResolutionSettingsRungTests.cs

@@ -110,9 +110,10 @@ public async Task OnALiveStore_AFreshAndAnUpgradedStore_EndWithTheColumnsAndThei
         {
             /* Fresh: every column, its type, its nullability and its default. */
             await AssertColumnsAsync(connection, ct);
-            Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) sits above this rung */
+            Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) and V189 (#5867) sit above this rung */
 
             /* A store one rung behind, holding a notification row, a route and an alert-history row from before the rung. */
+            await RungRewindArtifacts.DropV189SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV188SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV187SentinelAsync(connection, ct);
             await ExecAsync(connection, "DELETE FROM darling_schema_version WHERE version >= 187", ct);
@@ -127,7 +128,7 @@ await ExecAsync(connection,
 
             await PgMigrations.MigrateAsync(connection, ct);
 
-            Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) sits above this rung */
+            Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) and V189 (#5867) sit above this rung */
             await AssertColumnsAsync(connection, ct);
             Assert.Equal("f|f|f|f", await ScalarAsync(connection,
                 "SELECT concat_ws('|', slack_send_resolutions, teams_send_resolutions, generic_send_resolutions, email_send_resolutions) FROM config.config_notification WHERE id = 1", ct));

Darling/Darling.Tests/AwsPerServerRoleRungTests.cs

@@ -112,7 +112,7 @@ public void TheProbeMapsAFullyMigratedStoreToThisTopRung()
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 9})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 10})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 11})", viewer, StringComparison.Ordinal);
-        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */
+        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 12})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 13})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 14})", viewer, StringComparison.Ordinal);

Darling/Darling.Tests/CaggGroupIndexDropLiveTests.cs

@@ -318,9 +318,15 @@ the group index still present — the same fixture shape CaggGroupIndexDropUpgra
     /// A future rung that adds a new probe sentinel must add its artifact's removal here.</summary>
     private static async Task DropArtifactsOfLaterRungsAsync(NpgsqlConnection connection, int simulatedVersion, CancellationToken ct)
     {
+        if (simulatedVersion < 189)
+        {
+            /* V189 (#5867) - the compose builder's failure table is the probe's newest sentinel. */
+            await RungRewindArtifacts.DropV189SentinelAsync(connection, ct);
+        }
+
         if (simulatedVersion < 188)
         {
-            /* V188 (#5821) - the investigation link rows' alert values are the probe's newest sentinel. */
+            /* V188 (#5821) - the investigation link rows' alert values. */
             await RungRewindArtifacts.DropV188SentinelAsync(connection, ct);
         }
 

Darling/Darling.Tests/CollectionLogHourlyLiveTests.cs

@@ -89,6 +89,8 @@ public async Task AnEmptyStore_TakesV186_AndAStoreAtV185_TakesItAgain_AndTheProb
         Assert.Equal(StorageVersion.SchemaVersion, await ProbeAsync(scratch.ConnectionString, ct));
 
         /* Back to V185: the table gone and the ladder at 185. The probe must say 185, not 186. */
+        /* V189 (#5867) sits above this rung too, so its sentinel goes first. */
+        await RungRewindArtifacts.DropV189SentinelAsync(connection, ct);
         /* V187 (#5825) sits above this rung, so its sentinel goes too. */
         await RungRewindArtifacts.DropV188SentinelAsync(connection, ct);
         await RungRewindArtifacts.DropV187SentinelAsync(connection, ct);

Darling/Darling.Tests/DarlingSelfAlertTests.cs

@@ -6526,6 +6526,32 @@ await e.ApplyRawPurgeOverHorizonAsync(
         Assert.Equal(DarlingSelfAlertEvaluator.RawPurgeOverHorizonClearedMetric, resolved.MetricName);
     }
 
+    private static QueryStoreComposeStamp.HourFailure ComposeFailure(int failures, bool inBackoff) =>
+        new(new DateTime(2026, 10, 7, 15, 0, 0), failures, "57014", "canceling statement due to statement timeout", 240,
+            new DateTime(2026, 10, 15, 14, 5, 0), new DateTime(2026, 10, 15, 16, 5, 0), new DateTime(2026, 10, 16, 16, 5, 0), inBackoff);
+
+    /// <summary>#5867: an hour in daily backoff fires one WARNING naming the hour, its count and reason; an hour still retried every
+    /// tick does not; and once no hour is in backoff the alert resolves.</summary>
+    [Fact]
+    public async Task ComposeHoursStuck_FiresForAnHourInBackoff_NotForOneStillRetried_AndResolvesWhenNoneIsLeft()
+    {
+        var h = new Harness();
+        var e = h.Build();
+
+        await e.ApplyComposeHoursStuckAsync(new[] { ComposeFailure(1, inBackoff: false) }, Ct);
+        Assert.Empty(h.Deliverer.Outcomes);
+
+        await e.ApplyComposeHoursStuckAsync(new[] { ComposeFailure(2, inBackoff: true), ComposeFailure(1, inBackoff: false) with { Hour = new DateTime(2026, 10, 15, 9, 0, 0) } }, Ct);
+        var fired = Assert.Single(h.Deliverer.Outcomes);
+        Assert.Equal(DarlingSelfAlertEvaluator.ComposeHoursStuckMetric, fired.MetricName);
+        Assert.Contains("1 hour(s) of the Query Store compose rollup", fired.DetailText, StringComparison.Ordinal);
+        Assert.Contains("2026-10-07 15:00 UTC: 2 failure(s), last reason 57014: canceling statement due to statement timeout", fired.DetailText, StringComparison.Ordinal);
+
+        await e.ApplyComposeHoursStuckAsync(Array.Empty<QueryStoreComposeStamp.HourFailure>(), Ct);
+        var resolved = Assert.Single(h.History.Records);
+        Assert.Equal(DarlingSelfAlertEvaluator.ComposeHoursStuckClearedMetric, resolved.MetricName);
+    }
+
     [Fact]
     public async Task RawPurgeOverHorizon_RunFailed_NamesTheSqlState()
     {

Darling/Darling.Tests/ExternalAlertRungTests.cs

@@ -110,7 +110,7 @@ public void TheProbeNamesTheRegistry_AndMapsAFullyMigratedStoreToThisTopRung()
 
         var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs");
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal);
-        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels sit above this one; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */
+        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels sit above this one; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 2})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 3})", viewer, StringComparison.Ordinal);
@@ -131,10 +131,11 @@ public void TheProbeNamesTheRegistry_AndMapsAFullyMigratedStoreToThisTopRung()
         var previousArm = viewer.IndexOf("if (hasClientTimeoutSettings)", StringComparison.Ordinal);
         Assert.True(thisArm >= 0 && previousArm > thisArm, "the V180 arm sits above the next rung down");
 
-        /* Every sentinel present: maps to the ladder's top. V186's (#5810), V185's (#5812), V184's (#5632), V183's (#5793), V182's (#5506) and V181's (#5750) absent: this rung. This sentinel
+        /* Every sentinel present: maps to the ladder's top. V189's (#5867), V188's (#5821), V187's (#5825), V186's (#5810), V185's (#5812), V184's (#5632), V183's (#5793), V182's (#5506) and V181's (#5750) absent: this rung. This sentinel
            absent too: a rung back. */
         var allPresent = Enumerable.Range(0, parameters.Length).Select(_ => (object)true).ToArray();
         Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, allPresent)!);
+        allPresent[Array.FindIndex(parameters, p => p.Name == "hasComposeStampFailures")] = false; /* V189 (#5867) absent: the cases below are stores below it. */
         allPresent[Array.FindIndex(parameters, p => p.Name == "hasInvestigationAlertFacts")] = false; /* V188 (#5821) absent: the cases below are stores below it. */
         allPresent[Array.FindIndex(parameters, p => p.Name == "hasAlertResolutionDelivery")] = false; /* V187 (#5825) absent: the cases below are stores below it. */
 
@@ -241,6 +242,7 @@ await ExecAsync(connection,
             /* An upgraded store: one rung behind (no tables, no column, no version row), then migrated, gets all of it. */
             await ExecAsync(connection, "DROP TABLE config.external_alert_state; DROP TABLE config.external_alert_pending; DROP TABLE config.external_alert_source; ALTER TABLE config.config_mute_rules DROP COLUMN external_source", ct);
             await ExecAsync(connection, "DELETE FROM darling_schema_version WHERE version >= " + ExternalAlertTables.RungVersion, ct);
+            await RungRewindArtifacts.DropV189SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV188SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV187SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV186SentinelAsync(connection, ct);

Darling/Darling.Tests/InvestigationRequestStoreLiveTests.cs

@@ -575,6 +575,7 @@ AND NOT attisdropped
 
             Assert.Equal("false|true|true|true|true", await RunBlocksRolledBackAsync(connection, roles, blocks, privileges, ct));
 
+            await RungRewindArtifacts.DropV189SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV188SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV187SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV186SentinelAsync(connection, ct);

Darling/Darling.Tests/InvestigationRungTests.cs

@@ -122,6 +122,7 @@ public async Task AStoreAtV183WithARoute_TakesV184_AndKeepsTheRoute()
         await connection.OpenAsync(ct);
         await PgMigrations.MigrateAsync(connection, ct);
 
+        await RungRewindArtifacts.DropV189SentinelAsync(connection, ct);
         await RungRewindArtifacts.DropV188SentinelAsync(connection, ct);
         await RungRewindArtifacts.DropV187SentinelAsync(connection, ct);
         await RungRewindArtifacts.DropV186SentinelAsync(connection, ct);

Darling/Darling.Tests/LegacyPinCandidateRungTests.cs

@@ -143,7 +143,7 @@ public void TheProbeCarriesTheTable_AtItsOwnOrdinal()
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 11})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 12})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 13})", viewer, StringComparison.Ordinal);
-        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */
+        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 14})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 15})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 16})", viewer, StringComparison.Ordinal);

Darling/Darling.Tests/NotificationRouteScopeRungTests.cs

@@ -180,6 +180,7 @@ public async Task OnALiveStore_TheChecksHold_ALegacyRowIsUntouched_AndTheTagTabl
         try
         {
             /* A store one rung behind: V181's columns gone, a legacy fleet-wide row and a legacy unscoped '*' row stored. */
+            await RungRewindArtifacts.DropV189SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV188SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV187SentinelAsync(connection, ct);
             await RungRewindArtifacts.DropV186SentinelAsync(connection, ct);

Darling/Darling.Tests/NotificationRoutingTests.cs

@@ -531,6 +531,8 @@ private static HashSet<string> DeliveredMetricNames()
             /* #4299: fired through the constant (FireAsync(..., RawPurgeOverHorizonMetric, ...)), not a quoted
                literal, so the FireAsync regex scan below cannot see it — listed here like StoreSettingsMetric. */
             DarlingSelfAlertEvaluator.RawPurgeOverHorizonMetric,
+            /* #5867: fired through the constant, as above. */
+            DarlingSelfAlertEvaluator.ComposeHoursStuckMetric,
             /* #4750: fired through the constant too, so the FireAsync literal scan below cannot see it. */
             DarlingSelfAlertEvaluator.NotificationChannelFailingMetric,
             /* #4732: fired through the constant, so the FireAsync literal scan below cannot see it. */

Darling/Darling.Tests/PlanRegressionDailyRungTests.cs

@@ -359,7 +359,7 @@ public void TheProbeCarriesTheBuiltTableAtItsOwnOrdinal_AndMapsItsRung()
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 10})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 11})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 12})", viewer, StringComparison.Ordinal);
-        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */
+        /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 13})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 14})", viewer, StringComparison.Ordinal);
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 15})", viewer, StringComparison.Ordinal);