diff --git a/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs b/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs index 4875f61b87..ce0a0e03dd 100644 --- a/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs +++ b/Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs @@ -923,8 +923,10 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop DarlingWorker. 16th exempt since #4750: EvaluateNotificationChannelsAsync's wrapper catch — the webhook channels' failure counts are read from the webhook service's memory, so there is no store read for the condition to swallow. 19th exempt since #5654: EvaluateExternalAlertsAsync's wrapper - catch - the delivery self-alert's report is a parameter, with the count read counted in DarlingWorker. */ - (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 13, 19), + catch - the delivery self-alert's report is a parameter, with the count read counted in DarlingWorker. 20th exempt since + #5867: EvaluateComposeHoursStuckAsync's wrapper catch - the compose rollup self-alert's failure record is a parameter, + with the read exempt in DarlingWorker (it holds the alert on a failed read). */ + (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 13, 20), }; /// @@ -962,6 +964,9 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop a swallowed read would leave the standing alert unjudged, and the worker holds it (sends no report) rather than resolving it. */ "EvaluateExternalAlertsAsync", + /* #5867: the compose rollup self-alert's one store read, the compose builder's failure record. EXEMPT: a failed read + sends no report, so the worker holds the standing alert rather than resolving it, and the next hourly tick reads again. */ + "EvaluateComposeHoursStuckAsync", /* #3354: the mute-rule reload. Not an alert pass — a control-plane read — but its swallowed failure decides what the engine suppresses on every following sweep, so it belongs to this population. It is also the member whose scope this list was designed to admit: an alerting read @@ -975,8 +980,10 @@ resolving it. */ not exempt, because a RejectedValue row is judgeable evidence for the alert, not context for its text. */ private const int WorkerCountedSites = 12; /* 13th: the deadlock text repair pass's catch in SweepStoreSelfMetricsAsync, exempt like the re-mask pass beside it. - 14th: the collection log hourly rollup's catch there (#5810), a maintenance write no alert is judged on. */ - private const int WorkerExemptSites = 14; + 14th: the collection log hourly rollup's catch there (#5810), a maintenance write no alert is judged on. + 15th: EvaluateComposeHoursStuckAsync's catch (#5867), the compose builder's failure record read for its self-alert, which + holds the standing alert on a failed read. */ + private const int WorkerExemptSites = 15; /// /// Counted sites tree-wide. ONE numeral with several readers rather than the same number written out at @@ -1055,6 +1062,8 @@ resolving it. */ ["Store-job cadence self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker", ["Retention-held self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker", ["Raw-purge-over-horizon self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker", + ["Compose-rollup-hours-stuck self-alert failed"] = "handed its evidence as parameters; the read is isolated in DarlingWorker.EvaluateComposeHoursStuckAsync", + ["Query Store compose rollup: the failure record could not be read for the self-alert"] = "an unreadable failure record sends no report, so the alert's standing state is held as it was, and the next hourly compose tick reads it again", ["Notification-channel self-alert failed"] = "reads the webhook channels' failure counts from the webhook service's memory and performs no store read at all - there is no read for this condition to be the swallowing of", ["Stale-mute self-alert failed"] = "handed its evidence (the live MuteRuleService cache) as a parameter and performs no store read at all - there is no read anywhere for this condition to be the swallowing of", ["Fleet gate self-alert failed"] = "handed its evidence (the worker's in-memory gate counts) as a parameter and performs no store read at all", @@ -1269,8 +1278,10 @@ WebTlsCertificateState report the web host publishes - there is no store read to its evidence as a parameter. 37th: the deadlock text repair pass, the second stage beside the re-mask pass (the same maintenance rewrite, marker-gated, resumed next hour). 38th since #5654: the external-alert delivery self-alert's wrapper catch, handed its report as a parameter. 39th since #5810: the collection log hourly rollup's catch, a - maintenance write no alert is judged on, redone by the next pass. */ - Assert.Equal(39, totalExempt); + maintenance write no alert is judged on, redone by the next pass. 40th since #5867: the Compose Rollup Hours Stuck + self-alert's wrapper catch, handed its failure record as a parameter. 41st since #5867: the worker's read of that + record, which holds the standing alert on a failure. */ + Assert.Equal(41, totalExempt); /* Every exemption in the table is actually used. An exemption for a message that no longer exists is a hole this pin would otherwise keep open indefinitely — the shape that lets a real new catch diff --git a/Darling/Darling.Tests/AlertResolutionSettingsRungTests.cs b/Darling/Darling.Tests/AlertResolutionSettingsRungTests.cs index a36b7bc8df..207a85e24d 100644 --- a/Darling/Darling.Tests/AlertResolutionSettingsRungTests.cs +++ b/Darling/Darling.Tests/AlertResolutionSettingsRungTests.cs @@ -110,9 +110,10 @@ public async Task OnALiveStore_AFreshAndAnUpgradedStore_EndWithTheColumnsAndThei { /* Fresh: every column, its type, its nullability and its default. */ await AssertColumnsAsync(connection, ct); - Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) sits above this rung */ + Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) and V189 (#5867) sit above this rung */ /* A store one rung behind, holding a notification row, a route and an alert-history row from before the rung. */ + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await ExecAsync(connection, "DELETE FROM darling_schema_version WHERE version >= 187", ct); @@ -127,7 +128,7 @@ await ExecAsync(connection, await PgMigrations.MigrateAsync(connection, ct); - Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) sits above this rung */ + Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); /* the ladder's top: V188 (#5821) and V189 (#5867) sit above this rung */ await AssertColumnsAsync(connection, ct); Assert.Equal("f|f|f|f", await ScalarAsync(connection, "SELECT concat_ws('|', slack_send_resolutions, teams_send_resolutions, generic_send_resolutions, email_send_resolutions) FROM config.config_notification WHERE id = 1", ct)); diff --git a/Darling/Darling.Tests/AwsPerServerRoleRungTests.cs b/Darling/Darling.Tests/AwsPerServerRoleRungTests.cs index 00bcdf3d3c..a3254372fd 100644 --- a/Darling/Darling.Tests/AwsPerServerRoleRungTests.cs +++ b/Darling/Darling.Tests/AwsPerServerRoleRungTests.cs @@ -112,7 +112,7 @@ public void TheProbeMapsAFullyMigratedStoreToThisTopRung() Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 9})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 10})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 11})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 12})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 13})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 14})", viewer, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/CaggGroupIndexDropLiveTests.cs b/Darling/Darling.Tests/CaggGroupIndexDropLiveTests.cs index aac97ad701..7ec40dce1e 100644 --- a/Darling/Darling.Tests/CaggGroupIndexDropLiveTests.cs +++ b/Darling/Darling.Tests/CaggGroupIndexDropLiveTests.cs @@ -318,9 +318,15 @@ the group index still present — the same fixture shape CaggGroupIndexDropUpgra /// A future rung that adds a new probe sentinel must add its artifact's removal here. private static async Task DropArtifactsOfLaterRungsAsync(NpgsqlConnection connection, int simulatedVersion, CancellationToken ct) { + if (simulatedVersion < 189) + { + /* V189 (#5867) - the compose builder's failure table is the probe's newest sentinel. */ + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); + } + if (simulatedVersion < 188) { - /* V188 (#5821) - the investigation link rows' alert values are the probe's newest sentinel. */ + /* V188 (#5821) - the investigation link rows' alert values. */ await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); } diff --git a/Darling/Darling.Tests/CollectionLogHourlyLiveTests.cs b/Darling/Darling.Tests/CollectionLogHourlyLiveTests.cs index 9cd151aa19..562e1565ea 100644 --- a/Darling/Darling.Tests/CollectionLogHourlyLiveTests.cs +++ b/Darling/Darling.Tests/CollectionLogHourlyLiveTests.cs @@ -89,6 +89,8 @@ public async Task AnEmptyStore_TakesV186_AndAStoreAtV185_TakesItAgain_AndTheProb Assert.Equal(StorageVersion.SchemaVersion, await ProbeAsync(scratch.ConnectionString, ct)); /* Back to V185: the table gone and the ladder at 185. The probe must say 185, not 186. */ + /* V189 (#5867) sits above this rung too, so its sentinel goes first. */ + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); /* V187 (#5825) sits above this rung, so its sentinel goes too. */ await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/DarlingSelfAlertTests.cs b/Darling/Darling.Tests/DarlingSelfAlertTests.cs index c7b56927ff..386ed9ea4d 100644 --- a/Darling/Darling.Tests/DarlingSelfAlertTests.cs +++ b/Darling/Darling.Tests/DarlingSelfAlertTests.cs @@ -6526,6 +6526,32 @@ await e.ApplyRawPurgeOverHorizonAsync( Assert.Equal(DarlingSelfAlertEvaluator.RawPurgeOverHorizonClearedMetric, resolved.MetricName); } + private static QueryStoreComposeStamp.HourFailure ComposeFailure(int failures, bool inBackoff) => + new(new DateTime(2026, 10, 7, 15, 0, 0), failures, "57014", "canceling statement due to statement timeout", 240, + new DateTime(2026, 10, 15, 14, 5, 0), new DateTime(2026, 10, 15, 16, 5, 0), new DateTime(2026, 10, 16, 16, 5, 0), inBackoff); + + /// #5867: an hour in daily backoff fires one WARNING naming the hour, its count and reason; an hour still retried every + /// tick does not; and once no hour is in backoff the alert resolves. + [Fact] + public async Task ComposeHoursStuck_FiresForAnHourInBackoff_NotForOneStillRetried_AndResolvesWhenNoneIsLeft() + { + var h = new Harness(); + var e = h.Build(); + + await e.ApplyComposeHoursStuckAsync(new[] { ComposeFailure(1, inBackoff: false) }, Ct); + Assert.Empty(h.Deliverer.Outcomes); + + await e.ApplyComposeHoursStuckAsync(new[] { ComposeFailure(2, inBackoff: true), ComposeFailure(1, inBackoff: false) with { Hour = new DateTime(2026, 10, 15, 9, 0, 0) } }, Ct); + var fired = Assert.Single(h.Deliverer.Outcomes); + Assert.Equal(DarlingSelfAlertEvaluator.ComposeHoursStuckMetric, fired.MetricName); + Assert.Contains("1 hour(s) of the Query Store compose rollup", fired.DetailText, StringComparison.Ordinal); + Assert.Contains("2026-10-07 15:00 UTC: 2 failure(s), last reason 57014: canceling statement due to statement timeout", fired.DetailText, StringComparison.Ordinal); + + await e.ApplyComposeHoursStuckAsync(Array.Empty(), Ct); + var resolved = Assert.Single(h.History.Records); + Assert.Equal(DarlingSelfAlertEvaluator.ComposeHoursStuckClearedMetric, resolved.MetricName); + } + [Fact] public async Task RawPurgeOverHorizon_RunFailed_NamesTheSqlState() { diff --git a/Darling/Darling.Tests/ExternalAlertRungTests.cs b/Darling/Darling.Tests/ExternalAlertRungTests.cs index 9b3065593d..a66912f400 100644 --- a/Darling/Darling.Tests/ExternalAlertRungTests.cs +++ b/Darling/Darling.Tests/ExternalAlertRungTests.cs @@ -110,7 +110,7 @@ public void TheProbeNamesTheRegistry_AndMapsAFullyMigratedStoreToThisTopRung() var viewer = RepoFile.ReadRepoFile("Darling", "PerformanceMonitor.Darling.Viewer", "ViewerDataService.cs"); Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels sit above this one; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels sit above this one; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 2})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 3})", viewer, StringComparison.Ordinal); @@ -131,10 +131,11 @@ public void TheProbeNamesTheRegistry_AndMapsAFullyMigratedStoreToThisTopRung() var previousArm = viewer.IndexOf("if (hasClientTimeoutSettings)", StringComparison.Ordinal); Assert.True(thisArm >= 0 && previousArm > thisArm, "the V180 arm sits above the next rung down"); - /* Every sentinel present: maps to the ladder's top. V186's (#5810), V185's (#5812), V184's (#5632), V183's (#5793), V182's (#5506) and V181's (#5750) absent: this rung. This sentinel + /* Every sentinel present: maps to the ladder's top. V189's (#5867), V188's (#5821), V187's (#5825), V186's (#5810), V185's (#5812), V184's (#5632), V183's (#5793), V182's (#5506) and V181's (#5750) absent: this rung. This sentinel absent too: a rung back. */ var allPresent = Enumerable.Range(0, parameters.Length).Select(_ => (object)true).ToArray(); Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, allPresent)!); + allPresent[Array.FindIndex(parameters, p => p.Name == "hasComposeStampFailures")] = false; /* V189 (#5867) absent: the cases below are stores below it. */ allPresent[Array.FindIndex(parameters, p => p.Name == "hasInvestigationAlertFacts")] = false; /* V188 (#5821) absent: the cases below are stores below it. */ allPresent[Array.FindIndex(parameters, p => p.Name == "hasAlertResolutionDelivery")] = false; /* V187 (#5825) absent: the cases below are stores below it. */ @@ -241,6 +242,7 @@ await ExecAsync(connection, /* An upgraded store: one rung behind (no tables, no column, no version row), then migrated, gets all of it. */ await ExecAsync(connection, "DROP TABLE config.external_alert_state; DROP TABLE config.external_alert_pending; DROP TABLE config.external_alert_source; ALTER TABLE config.config_mute_rules DROP COLUMN external_source", ct); await ExecAsync(connection, "DELETE FROM darling_schema_version WHERE version >= " + ExternalAlertTables.RungVersion, ct); + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/InvestigationRequestStoreLiveTests.cs b/Darling/Darling.Tests/InvestigationRequestStoreLiveTests.cs index f162b2195d..d6cee03329 100644 --- a/Darling/Darling.Tests/InvestigationRequestStoreLiveTests.cs +++ b/Darling/Darling.Tests/InvestigationRequestStoreLiveTests.cs @@ -575,6 +575,7 @@ AND NOT attisdropped Assert.Equal("false|true|true|true|true", await RunBlocksRolledBackAsync(connection, roles, blocks, privileges, ct)); + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/InvestigationRungTests.cs b/Darling/Darling.Tests/InvestigationRungTests.cs index 495f6edef8..bf8aebdff0 100644 --- a/Darling/Darling.Tests/InvestigationRungTests.cs +++ b/Darling/Darling.Tests/InvestigationRungTests.cs @@ -122,6 +122,7 @@ public async Task AStoreAtV183WithARoute_TakesV184_AndKeepsTheRoute() await connection.OpenAsync(ct); await PgMigrations.MigrateAsync(connection, ct); + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/LegacyPinCandidateRungTests.cs b/Darling/Darling.Tests/LegacyPinCandidateRungTests.cs index b584e20417..138291960c 100644 --- a/Darling/Darling.Tests/LegacyPinCandidateRungTests.cs +++ b/Darling/Darling.Tests/LegacyPinCandidateRungTests.cs @@ -143,7 +143,7 @@ public void TheProbeCarriesTheTable_AtItsOwnOrdinal() Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 11})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 12})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 13})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 14})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 15})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 16})", viewer, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/NotificationRouteScopeRungTests.cs b/Darling/Darling.Tests/NotificationRouteScopeRungTests.cs index b4b36aa8a0..1d637fbf94 100644 --- a/Darling/Darling.Tests/NotificationRouteScopeRungTests.cs +++ b/Darling/Darling.Tests/NotificationRouteScopeRungTests.cs @@ -180,6 +180,7 @@ public async Task OnALiveStore_TheChecksHold_ALegacyRowIsUntouched_AndTheTagTabl try { /* A store one rung behind: V181's columns gone, a legacy fleet-wide row and a legacy unscoped '*' row stored. */ + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/NotificationRoutingTests.cs b/Darling/Darling.Tests/NotificationRoutingTests.cs index f243936b90..eb354eb2a1 100644 --- a/Darling/Darling.Tests/NotificationRoutingTests.cs +++ b/Darling/Darling.Tests/NotificationRoutingTests.cs @@ -531,6 +531,8 @@ private static HashSet DeliveredMetricNames() /* #4299: fired through the constant (FireAsync(..., RawPurgeOverHorizonMetric, ...)), not a quoted literal, so the FireAsync regex scan below cannot see it — listed here like StoreSettingsMetric. */ DarlingSelfAlertEvaluator.RawPurgeOverHorizonMetric, + /* #5867: fired through the constant, as above. */ + DarlingSelfAlertEvaluator.ComposeHoursStuckMetric, /* #4750: fired through the constant too, so the FireAsync literal scan below cannot see it. */ DarlingSelfAlertEvaluator.NotificationChannelFailingMetric, /* #4732: fired through the constant, so the FireAsync literal scan below cannot see it. */ diff --git a/Darling/Darling.Tests/PlanRegressionDailyRungTests.cs b/Darling/Darling.Tests/PlanRegressionDailyRungTests.cs index c5ca8929c3..ce96d58de6 100644 --- a/Darling/Darling.Tests/PlanRegressionDailyRungTests.cs +++ b/Darling/Darling.Tests/PlanRegressionDailyRungTests.cs @@ -359,7 +359,7 @@ public void TheProbeCarriesTheBuiltTableAtItsOwnOrdinal_AndMapsItsRung() Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 10})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 11})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 12})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 13})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 14})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 15})", viewer, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/PlanRegressionRowcountRungTests.cs b/Darling/Darling.Tests/PlanRegressionRowcountRungTests.cs index f826a15b3e..c119a88df0 100644 --- a/Darling/Darling.Tests/PlanRegressionRowcountRungTests.cs +++ b/Darling/Darling.Tests/PlanRegressionRowcountRungTests.cs @@ -126,7 +126,7 @@ public void TheProbeNamesTheColumn_AndMapsToThisRungWhenNoLaterSentinelIsPresent Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 3})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 4})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 5})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 6})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 7})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 8})", viewer, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/QueryStoreComposeStampFailureLiveTests.cs b/Darling/Darling.Tests/QueryStoreComposeStampFailureLiveTests.cs new file mode 100644 index 0000000000..7a321431df --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreComposeStampFailureLiveTests.cs @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Npgsql; +using PerformanceMonitor.Darling.Storage; +using Xunit; +using static Darling.Tests.ComposeStampLiveSupport; + +namespace Darling.Tests; + +/// +/// #5867: a Query Store compose build that throws is recorded per hour (collect.query_store_compose_stamp_failures), retried next +/// tick (under the extended timeout after a statement timeout), and after +/// failures in a row, or a timeout under the extended timeout, retried once a day instead of every tick. A success clears the record. The +/// failing build is planted with a row trigger on the rollup that raises for one hour only; a query_canceled raise carries SQLSTATE +/// 57014, the code a statement_timeout cancel carries, so the timeout path runs without waiting a minute. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Every test reaches DARLING_TEST_PG only to CREATE and DROP its + own database through ScratchPostgres and works entirely inside it, so it never touches the shared database and cannot race the + live collection. */ +public sealed class QueryStoreComposeStampFailureLiveTests +{ + /// + /// Plants a failing build of : a row trigger on the rollup that raises for that hour's rows. With + /// it raises only while the transaction's statement_timeout is the normal + /// one (60 s, shown as 1min), so the same build succeeds under the extended timeout: an hour that needs longer than 60 s. + /// + private static Task PlantFailureAsync(NpgsqlConnection connection, DateTime hour, string errcode, bool onlyUnderTheNormalTimeout, CancellationToken ct) => + ExecAsync(connection, $@" +CREATE OR REPLACE FUNCTION collect.fail_5867() RETURNS trigger LANGUAGE plpgsql AS $f$ +BEGIN + IF {(onlyUnderTheNormalTimeout ? "current_setting('statement_timeout') = '1min'" : "true")} THEN + RAISE EXCEPTION 'planted build failure 5867 (statement_timeout %)', current_setting('statement_timeout') USING ERRCODE = '{errcode}'; + END IF; + RETURN NEW; +END +$f$; +DROP TRIGGER IF EXISTS trg_fail_5867 ON collect.query_store_compose_stamp; +CREATE TRIGGER trg_fail_5867 BEFORE INSERT ON collect.query_store_compose_stamp +FOR EACH ROW WHEN (NEW.collection_time >= TIMESTAMP '{At(hour)}' AND NEW.collection_time < TIMESTAMP '{At(hour)}' + interval '1 hour') +EXECUTE FUNCTION collect.fail_5867();", ct); + + private static Task DropFailureAsync(NpgsqlConnection connection, CancellationToken ct) => + ExecAsync(connection, "DROP TRIGGER IF EXISTS trg_fail_5867 ON collect.query_store_compose_stamp", ct); + + /// Makes a never-built hour again, so the next plan picks it up. + private static Task UnbuildAsync(NpgsqlConnection connection, DateTime hour, CancellationToken ct) => + ExecAsync(connection, $"DELETE FROM collect.query_store_compose_stamp_hours WHERE hour = TIMESTAMP '{At(hour)}'", ct); + + private static Task HoursBuiltAsync(NpgsqlConnection connection, DateTime hour, CancellationToken ct) => CountAsync(connection, + $"SELECT count(*) FROM collect.query_store_compose_stamp_hours WHERE hour = TIMESTAMP '{At(hour)}'", ct); + + /// A day passes for the hours in backoff: their next attempt moves back by . + private static Task ADayPassesAsync(NpgsqlConnection connection, CancellationToken ct) => + ExecAsync(connection, "UPDATE collect.query_store_compose_stamp_failures SET retry_at = retry_at - interval '1 day', last_failed_at = last_failed_at - interval '1 day'", ct); + + private static Task TickAsync(NpgsqlDataSource source, ILogger logger, CancellationToken ct) => + QueryStoreComposeStamp.RunTickAsync(source, DateTime.UtcNow, RetentionDays, logger, () => TimeSpan.Zero, ct, + new QueryStoreComposeStamp.TickOptions(new QueryStoreComposeStamp.DeferralWatch(() => DateTime.UtcNow))); + + [Fact] + public async Task ABuildThatFailsForAnotherReason_IsRetriedEachTickThreeTimes_ThenOnceADay_IsReadable_AndASuccessClearsIt() + { + Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), SkipReason); + var ct = TestContext.Current.CancellationToken; + var (scratch, connection, source, hourNow) = await ArrangeAsync(ct); + var bodySucceeded = false; + try + { + var failing = hourNow.AddHours(-5); + var normal = hourNow.AddHours(-6); + await PlantFailureAsync(connection, failing, "40P01", onlyUnderTheNormalTimeout: false, ct); + await UnbuildAsync(connection, failing, ct); + await UnbuildAsync(connection, normal, ct); + var logger = new CapturingTestLogger(); + + /* Tick 1: the normal hour builds as before and leaves no record; the failing hour is recorded once and retried next tick. */ + var first = await TickAsync(source, logger, ct); + Assert.Equal(1, first.Failed); + Assert.Equal(1, first.BuiltMissing); + Assert.Equal(0, first.BackedOff); + Assert.Equal(1, await HoursBuiltAsync(connection, normal, ct)); + var record = Assert.Single(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)); + Assert.Equal(failing, record.Hour); + Assert.Equal(1, record.Failures); + Assert.Equal("40P01", record.LastSqlState); + Assert.Contains("planted build failure 5867", record.LastReason, StringComparison.Ordinal); + Assert.Equal(QueryStoreComposeStamp.BuildStatementTimeoutSeconds, record.LastTimeoutSeconds); + Assert.False(record.InBackoff); + Assert.Contains(logger.Lines, l => l.StartsWith("Warning:", StringComparison.Ordinal) && l.Contains("is retried next tick:", StringComparison.Ordinal)); + + /* Not a timeout, so the retry runs under the normal timeout. */ + var planned = Assert.Single(await QueryStoreComposeStamp.PlanBuildsAsync(connection, DateTime.UtcNow, RetentionDays, ct)); + Assert.Equal(new QueryStoreComposeStamp.Build(failing, false, 1, false), planned); + + /* Ticks 2 and 3: the third failure in a row moves the hour to its daily attempt. */ + Assert.Equal(1, (await TickAsync(source, logger, ct)).Failed); + var third = await TickAsync(source, logger, ct); + Assert.Equal(1, third.Failed); + Assert.Equal(1, third.BackedOff); + record = Assert.Single(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)); + Assert.Equal(QueryStoreComposeStamp.FailuresBeforeBackoff, record.Failures); + Assert.True(record.InBackoff); + Assert.Equal(record.LastFailedAt + QueryStoreComposeStamp.BackoffInterval, record.NextAttemptAt); + var backoffLine = logger.Lines.Last(l => l.StartsWith("Warning:", StringComparison.Ordinal)); + Assert.Contains("has failed 3 time(s) in a row; it is retried once a day", backoffLine, StringComparison.Ordinal); + + /* Tick 4, the next hour: the hour in backoff is not planned, not built, and logs nothing. */ + var warnings = logger.CountAtLevel(LogLevel.Warning); + Assert.Empty(await QueryStoreComposeStamp.PlanBuildsAsync(connection, DateTime.UtcNow, RetentionDays, ct)); + var fourth = await TickAsync(source, logger, ct); + Assert.Equal(0, fourth.Failed); + Assert.Equal(0, fourth.Built); + Assert.Equal(warnings, logger.CountAtLevel(LogLevel.Warning)); + Assert.Equal(QueryStoreComposeStamp.FailuresBeforeBackoff, Assert.Single(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)).Failures); + + /* A day later it is tried once, fails once more, and goes back to waiting a day. */ + await ADayPassesAsync(connection, ct); + var daily = await TickAsync(source, logger, ct); + Assert.Equal(1, daily.Failed); + Assert.Equal(1, daily.BackedOff); + Assert.Equal(QueryStoreComposeStamp.FailuresBeforeBackoff + 1, Assert.Single(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)).Failures); + Assert.Equal(0, (await TickAsync(source, logger, ct)).Failed); + + /* The cause goes away: the next daily attempt builds the hour and clears its record. */ + await DropFailureAsync(connection, ct); + await ADayPassesAsync(connection, ct); + var healed = await TickAsync(source, logger, ct); + Assert.Equal(0, healed.Failed); + Assert.Equal(1, healed.BuiltMissing); + Assert.Equal(1, await HoursBuiltAsync(connection, failing, ct)); + Assert.Empty(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)); + bodySucceeded = true; + } + finally + { + await CleanupAsync(scratch, connection, source, bodySucceeded); + } + } + + [Fact] + public async Task ABuildThatTimesOut_IsRetriedOnceUnderTheExtendedTimeout_AndBuildsThere_WhichClearsItsRecord() + { + Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), SkipReason); + var ct = TestContext.Current.CancellationToken; + var (scratch, connection, source, hourNow) = await ArrangeAsync(ct); + var bodySucceeded = false; + try + { + var failing = hourNow.AddHours(-5); + await PlantFailureAsync(connection, failing, "query_canceled", onlyUnderTheNormalTimeout: true, ct); + await UnbuildAsync(connection, failing, ct); + var logger = new CapturingTestLogger(); + + var first = await TickAsync(source, logger, ct); + Assert.Equal(1, first.Failed); + var record = Assert.Single(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)); + Assert.Equal(QueryStoreComposeStamp.StatementTimeoutSqlState, record.LastSqlState); + Assert.False(record.InBackoff); + Assert.Contains(logger.Lines, l => l.Contains($"is retried next tick under a {QueryStoreComposeStamp.ExtendedBuildStatementTimeoutSeconds} s statement timeout", StringComparison.Ordinal)); + Assert.Equal(new QueryStoreComposeStamp.Build(failing, false, 1, true), + Assert.Single(await QueryStoreComposeStamp.PlanBuildsAsync(connection, DateTime.UtcNow, RetentionDays, ct))); + + /* The next tick runs the build under the extended timeout, where the planted check passes: built, and the record is gone. */ + var second = await TickAsync(source, logger, ct); + Assert.Equal(0, second.Failed); + Assert.Equal(1, second.BuiltMissing); + Assert.Equal(1, await HoursBuiltAsync(connection, failing, ct)); + Assert.Empty(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)); + bodySucceeded = true; + } + finally + { + await CleanupAsync(scratch, connection, source, bodySucceeded); + } + } + + [Fact] + public async Task ABuildThatTimesOutUnderTheExtendedTimeoutToo_BacksOffAtOnce_AndIsNotPlannedOnTheNextTick() + { + Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), SkipReason); + var ct = TestContext.Current.CancellationToken; + var (scratch, connection, source, hourNow) = await ArrangeAsync(ct); + var bodySucceeded = false; + try + { + var failing = hourNow.AddHours(-5); + await PlantFailureAsync(connection, failing, "query_canceled", onlyUnderTheNormalTimeout: false, ct); + await UnbuildAsync(connection, failing, ct); + + Assert.Equal(0, (await TickAsync(source, NullLogger.Instance, ct)).BackedOff); + var second = await TickAsync(source, NullLogger.Instance, ct); + Assert.Equal(1, second.Failed); + Assert.Equal(1, second.BackedOff); + var record = Assert.Single(await QueryStoreComposeStamp.ReadFailuresAsync(connection, ct)); + Assert.Equal(2, record.Failures); + Assert.Equal(QueryStoreComposeStamp.ExtendedBuildStatementTimeoutSeconds, record.LastTimeoutSeconds); + Assert.True(record.InBackoff); + + var third = await TickAsync(source, NullLogger.Instance, ct); + Assert.Equal(0, third.Failed); + Assert.Equal(0, third.Built); + Assert.Equal(0, await HoursBuiltAsync(connection, failing, ct)); + bodySucceeded = true; + } + finally + { + await CleanupAsync(scratch, connection, source, bodySucceeded); + } + } + + /// + /// #5871: an attempt's client timeout is the same for EVERY command in it, because the server grants each statement the whole + /// statement_timeout: a command left at the short client timeout would be cancelled by the client first, with no 57014 to + /// key the failure record on. Seen through commandCreated, which lists each command's timeout as it is made. + /// + [Fact] + public async Task EveryCommandOfABuildAttempt_CarriesTheAttemptsClientTimeout_ExtendedOrNot() + { + Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), SkipReason); + var ct = TestContext.Current.CancellationToken; + var (scratch, connection, source, hourNow) = await ArrangeAsync(ct); + var bodySucceeded = false; + try + { + var hour = hourNow.AddHours(-5); + await UnbuildAsync(connection, hour, ct); + + var normal = new System.Collections.Generic.List(); + Assert.NotNull(await QueryStoreComposeStamp.BuildHourAsync(connection, hour, DateTime.UtcNow, ct, commandCreated: c => normal.Add(c.CommandTimeout))); + + await UnbuildAsync(connection, hour, ct); + var extended = new System.Collections.Generic.List(); + Assert.NotNull(await QueryStoreComposeStamp.BuildHourAsync(connection, hour, DateTime.UtcNow, ct, + statementTimeoutSeconds: QueryStoreComposeStamp.ExtendedBuildStatementTimeoutSeconds, commandCreated: c => extended.Add(c.CommandTimeout))); + + /* The two attempts must differ, or the check below could pass with one timeout for everything. */ + var normalTimeout = Assert.Single(normal.Distinct()); + var extendedTimeout = Assert.Single(extended.Distinct()); + Assert.True(extendedTimeout > normalTimeout); + Assert.True(extendedTimeout >= QueryStoreComposeStamp.ExtendedBuildStatementTimeoutSeconds); + Assert.Equal(normal.Count, extended.Count); + Assert.True(extended.Count > 1); + bodySucceeded = true; + } + finally + { + await CleanupAsync(scratch, connection, source, bodySucceeded); + } + } +} diff --git a/Darling/Darling.Tests/QueryStoreComposeStampLiveTests.cs b/Darling/Darling.Tests/QueryStoreComposeStampLiveTests.cs index 79deb31f79..58c342dbd9 100644 --- a/Darling/Darling.Tests/QueryStoreComposeStampLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreComposeStampLiveTests.cs @@ -407,7 +407,8 @@ SELECT md5( } Assert.Equal(before, await TextAsync(connection, Fingerprint, ct)); - Assert.Equal(3, await CountAsync(connection, "SELECT count(*) FROM pg_class WHERE relnamespace = 'collect'::regnamespace AND relname LIKE 'query_store_compose_stamp%' AND relkind = 'r'", ct)); + /* Four since V189 (#5867) added the failure record. */ + Assert.Equal(4, await CountAsync(connection, "SELECT count(*) FROM pg_class WHERE relnamespace = 'collect'::regnamespace AND relname LIKE 'query_store_compose_stamp%' AND relkind = 'r'", ct)); bodySucceeded = true; } finally diff --git a/Darling/Darling.Tests/QueryStoreComposeStampTests.cs b/Darling/Darling.Tests/QueryStoreComposeStampTests.cs index cc8ea13364..186ee2e861 100644 --- a/Darling/Darling.Tests/QueryStoreComposeStampTests.cs +++ b/Darling/Darling.Tests/QueryStoreComposeStampTests.cs @@ -32,6 +32,15 @@ public sealed class QueryStoreComposeStampTests /* ---- the column map ------------------------------------------------------------------------------- */ + /// #5871: is read by no code, so a renumber at merge could miss it and still + /// compile. The rung that creates the failure table is the migration that carries the table's DDL. + [Fact] + public void TheFailuresRungVersion_IsTheVersionOfTheMigrationThatCreatesTheFailureTable() + { + var rung = Assert.Single(PgMigrations.Scripts, s => s.Sql == QueryStoreComposeStamp.FailuresCreateSql); + Assert.Equal(QueryStoreComposeStamp.FailuresRungVersion, rung.Version); + } + [Fact] public void TheMap_HasUniqueColumnsAndExpressions_OverTheFactAlias_NeverDoublePrecision() { @@ -168,7 +177,9 @@ public void TheBuildOrder_IsStaleHoursFirst_ThenMissingHoursNewestFirst_AfterThe Assert.Contains("WHERE p.rn <= $2", plan, StringComparison.Ordinal); Assert.Contains("b.built_seq IS DISTINCT FROM b.late_seq", plan, StringComparison.Ordinal); /* The build lag is read off the STORE's clock, the one the row trigger's WHEN uses; the service's clock is only the retention floor. */ - Assert.Contains("date_trunc('hour', coalesce($3::timestamp, now() AT TIME ZONE 'UTC') - interval '3 hours')", plan, StringComparison.Ordinal); + /* #5867: the store clock moved into the plan's clock CTE, which the failure filter reads too. */ + Assert.Contains("SELECT coalesce($3::timestamp, now() AT TIME ZONE 'UTC') AS at", plan, StringComparison.Ordinal); + Assert.Contains("date_trunc('hour', clock.at - interval '3 hours')", plan, StringComparison.Ordinal); Assert.DoesNotContain("$1::timestamp - interval", plan, StringComparison.Ordinal); Assert.Contains("greatest($1::timestamp,", plan, StringComparison.Ordinal); Assert.Contains("filled_since", plan, StringComparison.Ordinal); @@ -178,7 +189,9 @@ public void TheBuildOrder_IsStaleHoursFirst_ThenMissingHoursNewestFirst_AfterThe Assert.Equal(6, QueryStoreComposeStamp.MaxBuildsPerTick); Assert.Equal(60, QueryStoreComposeStamp.BuildStatementTimeoutSeconds); Assert.Equal(TimeSpan.FromMinutes(10), QueryStoreComposeStamp.MaxTickDuration); - Assert.Contains("60s", typeof(QueryStoreComposeStamp).GetField("StatementTimeoutSql", BindingFlags.NonPublic | BindingFlags.Static)!.GetValue(null)!.ToString(), StringComparison.Ordinal); + /* #5867: the timeout text is built per attempt (the normal 60 s, or the extended timeout after a statement timeout). */ + Assert.Contains("60s", typeof(QueryStoreComposeStamp).GetMethod("StatementTimeoutSql", BindingFlags.NonPublic | BindingFlags.Static)!.Invoke(null, new object[] { 60 })!.ToString(), StringComparison.Ordinal); + Assert.Equal(240, QueryStoreComposeStamp.ExtendedBuildStatementTimeoutSeconds); } [Fact] @@ -284,6 +297,21 @@ public void ChooseBuilds_WithNeverBuiltHoursWaiting_GivesStaleHoursAtMostHalfThe Assert.Equal(new[] { B(3, true), B(4, true), B(5, true), B(20, false), B(21, false), B(22, false) }, chosen); } + /// #5867: an hour that has failed before runs after the hours that have not, whatever its kind and age, so a build that may + /// run to its (extended) timeout again cannot spend the tick's wall-clock budget before the other hours get their turn. Which hours + /// are chosen does not change. + [Fact] + public void ChooseBuilds_RunsTheHoursThatFailedBefore_AfterTheOthers_AndChoosesTheSameHours() + { + var failedStale = B(3, true) with { Failures = 1, Extended = true }; + var failedMissing = B(20, false) with { Failures = 2 }; + var candidates = new[] { failedStale, B(4, true), B(5, true), failedMissing, B(21, false), B(22, false), B(23, false) }; + + var chosen = QueryStoreComposeStamp.ChooseBuilds(candidates, QueryStoreComposeStamp.MaxBuildsPerTick); + + Assert.Equal(new[] { B(4, true), B(5, true), B(21, false), B(22, false), failedStale, failedMissing }, chosen); + } + [Fact] public void ChooseBuilds_WithFewNeverBuiltHours_TakesThemAll_AndWithNoneWaiting_LetsStaleHoursTakeTheWholeTick() { @@ -438,7 +466,7 @@ public void TheProbe_HasThisRungsSentinelAndMapsItsArmToV173() Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 5})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 6})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 7})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 8})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 9})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 10})", viewer, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs b/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs index 1b4dbe400a..d47623038b 100644 --- a/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs +++ b/Darling/Darling.Tests/QueryStoreIntervalPartitionRungTests.cs @@ -69,7 +69,7 @@ public void TheRungsAreRegistered_DenseAndTheTop_AndShareOneBuilder() Assert.Equal(Enumerable.Range(171, StorageVersion.SchemaVersion - 171 + 1).ToArray(), versions.OrderBy(v => v).Where(v => v >= 171).ToArray()); Assert.Equal("query-store-interval-wide-partitioned", WideRung.Name); Assert.Equal("query-store-interval-latest-partitioned", LatestRung.Name); - /* No longer the top rung: the Query Store compose rollup rung (V173, #5582), the service version rung (V174, #5655), the plan regression rowcount rung (V175, #5631), the external alert rung (V180, #5654), the read latency capacity rung (V182, #5506) the read caller rung (V183, #5793), the investigation rung (V184, #5632) and the plan regression partial-day rung (V185, #5812) landed above it. */ + /* No longer the top rung: the Query Store compose rollup rung (V173, #5582), the service version rung (V174, #5655), the plan regression rowcount rung (V175, #5631), the external alert rung (V180, #5654), the read latency capacity rung (V182, #5506) the read caller rung (V183, #5793), the investigation rung (V184, #5632), the plan regression partial-day rung (V185, #5812), the collection log hourly rung (V186, #5810), the alert resolution rung (V187, #5825), the investigation alert facts rung (V188, #5821) and the compose failure rung (V189, #5867) landed above it. */ Assert.True(LatestRungVersion < StorageVersion.SchemaVersion); /* Registered once each, found by version: the distance from the top moved with every new rung (#5825). */ Assert.Single(PgMigrations.Scripts, m => m.Version == LatestRungVersion); @@ -660,6 +660,7 @@ private static async Task RewindToV170Async(NpgsqlConnection connection, Cancell } /* V185 (#5812), V184 (#5632), V183 (#5793), V182 (#5506) and V181 (#5750): the plan regression watermark column, the investigation tables, the read caller columns, the read latency capacity columns and the route scope columns are above V170 too (V185's is the probe's newest sentinel). */ + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/QueryStoreWatermarksRungTests.cs b/Darling/Darling.Tests/QueryStoreWatermarksRungTests.cs index 81d2b9afb7..f8c0d30190 100644 --- a/Darling/Darling.Tests/QueryStoreWatermarksRungTests.cs +++ b/Darling/Darling.Tests/QueryStoreWatermarksRungTests.cs @@ -97,7 +97,7 @@ public void TheProbeNamesTheTable_AndMapsAFullyMigratedStoreToTheTopRung_AndThis /* V179's (#5640) and V180's (#5654) sentinels are appended after this one, so this rung is no longer the top. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 1})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 2})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, and V187's (#5825) is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next; V184's (#5632) and V185's (#5812) are next, V186's (#5810) next, V187's (#5825) and V188's (#5821) next, and V189's (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 3})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 4})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 5})", viewer, StringComparison.Ordinal); @@ -120,11 +120,12 @@ public void TheProbeNamesTheTable_AndMapsAFullyMigratedStoreToTheTopRung_AndThis absent too this rung is the answer; with this sentinel absent as well, one rung back. */ var all = Enumerable.Repeat((object)true, parameters.Length).ToArray(); Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!); + all[Array.FindIndex(parameters, p => p.Name == "hasComposeStampFailures")] = false; /* V189 (#5867) absent: the cases below are stores below it. */ all[Array.FindIndex(parameters, p => p.Name == "hasInvestigationAlertFacts")] = false; /* V188 (#5821) absent: the cases below are stores below it. */ all[Array.FindIndex(parameters, p => p.Name == "hasAlertResolutionDelivery")] = false; /* V187 (#5825) absent: the cases below are stores below it. */ /* With V182's sentinel (#5506) absent too the answer is V181's. */ - /* With V186's (#5810) and V185's (#5812) sentinels absent the answer is V184's (#5632); with V184's absent too, V183's. */ + /* With V189's (#5867), V188's (#5821), V187's (#5825), V186's (#5810) and V185's (#5812) sentinels absent the answer is V184's (#5632); with V184's absent too, V183's. */ var withoutV185 = (object[])all.Clone(); withoutV185[ProbeOrdinal + 8] = false; withoutV185[ProbeOrdinal + 7] = false; @@ -182,6 +183,7 @@ public async Task OnAFreshStore_TheProbeMapsToTheNewestRung_AndWithoutTheTableTo Assert.Equal(StorageVersion.SchemaVersion, Convert.ToInt32(await ScalarAsync(connection, "SELECT MAX(version) FROM darling_schema_version", ct))); Assert.Equal(StorageVersion.SchemaVersion, await ProbedVersionAsync(connection, ct)); + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/ReadCallerRungTests.cs b/Darling/Darling.Tests/ReadCallerRungTests.cs index a2710063da..b961a97e5b 100644 --- a/Darling/Darling.Tests/ReadCallerRungTests.cs +++ b/Darling/Darling.Tests/ReadCallerRungTests.cs @@ -112,6 +112,7 @@ public async Task AStoreAtV182WithRows_TakesV183_AndKeepsEveryRow() await PgMigrations.MigrateAsync(connection, ct); /* Put the two tables back to their V182 shape and the ladder back to 182. */ + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/ReadLatencyCapacityRungTests.cs b/Darling/Darling.Tests/ReadLatencyCapacityRungTests.cs index 7d3cd2dc7c..98e416ea64 100644 --- a/Darling/Darling.Tests/ReadLatencyCapacityRungTests.cs +++ b/Darling/Darling.Tests/ReadLatencyCapacityRungTests.cs @@ -127,6 +127,7 @@ public async Task AStoreAtV181WithRows_TakesV182_AndKeepsEveryRow() await PgMigrations.MigrateAsync(connection, ct); /* Put the two tables back to their V181 shape and the ladder back to 181. */ + await RungRewindArtifacts.DropV189SentinelAsync(connection, ct); await RungRewindArtifacts.DropV188SentinelAsync(connection, ct); await RungRewindArtifacts.DropV187SentinelAsync(connection, ct); await RungRewindArtifacts.DropV186SentinelAsync(connection, ct); diff --git a/Darling/Darling.Tests/RungRewindArtifacts.cs b/Darling/Darling.Tests/RungRewindArtifacts.cs index 6005bdc199..cc7787ed34 100644 --- a/Darling/Darling.Tests/RungRewindArtifacts.cs +++ b/Darling/Darling.Tests/RungRewindArtifacts.cs @@ -214,4 +214,14 @@ ALTER TABLE IF EXISTS config.investigation_request_alerts DROP COLUMN IF EXISTS context_json;", connection); await command.ExecuteNonQueryAsync(cancellationToken); } + + /// + /// V189 (#5867): collect.query_store_compose_stamp_failures (the probe's newest sentinel), the compose builder's + /// failure record. Safe on a store that never had it. Nothing before V189 reads it. + /// + internal static async Task DropV189SentinelAsync(NpgsqlConnection connection, CancellationToken cancellationToken) + { + await using var command = new NpgsqlCommand("DROP TABLE IF EXISTS collect.query_store_compose_stamp_failures;", connection); + await command.ExecuteNonQueryAsync(cancellationToken); + } } diff --git a/Darling/Darling.Tests/RunnableQueueSettingsRungTests.cs b/Darling/Darling.Tests/RunnableQueueSettingsRungTests.cs index 46ede9b672..f020656d34 100644 --- a/Darling/Darling.Tests/RunnableQueueSettingsRungTests.cs +++ b/Darling/Darling.Tests/RunnableQueueSettingsRungTests.cs @@ -118,9 +118,9 @@ is the probe's last ordinal. */ /* With V180's sentinel (the top) absent the answer is V179's, 179; with V179's absent too, V178's, 178; with V178's absent too, this rung is the answer. */ /* With V182's sentinel (#5506) absent too the answer is V181's. */ - /* With V186's (#5810) and V185's (#5812) sentinels absent the answer is V184's (#5632); with V184's absent too, V183's. */ + /* With V189's (#5867), V188's (#5821), V187's (#5825), V186's (#5810) and V185's (#5812) sentinels absent the answer is V184's (#5632); with V184's absent too, V183's. */ var withoutV185 = (object[])all.Clone(); - /* V187 (#5825) and any later rung's sentinel are absent in every case below. */ + /* V187 (#5825), V188 (#5821), V189 (#5867) and any later rung's sentinel are absent in every case below. */ for (var i = ProbeOrdinal + 9; i < withoutV185.Length; i++) { withoutV185[i] = false; diff --git a/Darling/Darling.Tests/ServiceVersionRungTests.cs b/Darling/Darling.Tests/ServiceVersionRungTests.cs index 3f7513ea92..a46cc24012 100644 --- a/Darling/Darling.Tests/ServiceVersionRungTests.cs +++ b/Darling/Darling.Tests/ServiceVersionRungTests.cs @@ -90,8 +90,8 @@ public void TheProbeNamesTheTable_AndMapsAFullyMigratedStoreToThisTopRung() Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 4})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 5})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 6})", viewer, StringComparison.Ordinal); - /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next, then V184's (#5632) and V185's (#5812); V186's (#5810) - is the probe's last ordinal. */ + /* V181's (#5750), V182's (#5506) and V183's (#5793) sentinels are next, then V184's (#5632) and V185's (#5812), then V186's (#5810), V187's (#5825) and V188's (#5821); V189's + (#5867) is the probe's last ordinal. */ Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 7})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 8})", viewer, StringComparison.Ordinal); Assert.Contains($"reader.GetBoolean({ProbeOrdinal + 9})", viewer, StringComparison.Ordinal); diff --git a/Darling/Darling.Tests/StoreMetricsSummaryFirstTests.cs b/Darling/Darling.Tests/StoreMetricsSummaryFirstTests.cs index 346dd844e5..39ed699232 100644 --- a/Darling/Darling.Tests/StoreMetricsSummaryFirstTests.cs +++ b/Darling/Darling.Tests/StoreMetricsSummaryFirstTests.cs @@ -308,7 +308,7 @@ public void TheDescription_StatesTheSummaryFirstContract_AndTheCodeAgrees() var method = ToolMethod(); var description = method.GetCustomAttribute()!.Description; - Assert.Matches(@"SUMMARY FIRST: by default[^.]*three ranked lists, each bounded by limit[^.]*truncated", description); + Assert.Matches(@"SUMMARY FIRST: by default[^.]*four ranked lists, each bounded by limit[^.]*truncated", description); foreach (var key in new[] { "objects", "fastest_growing", "background_jobs", "delta_since" }) { Assert.Contains(key, description, StringComparison.Ordinal); @@ -342,7 +342,8 @@ public void TheTool_NoLongerProjectsADailySeriesForEveryObject() new Regex(@"view\s*==\s*DarlingStoreMetricsReader\.StoreMetricsView\.Object\s*\?\s*DarlingStoreMetricsReader\.SeriesFor\(daily,\s*selection\.Matched\[0\]\)"), code); /* Every list the payload publishes is bounded through the shared page helper. */ - Assert.Equal(3, Regex.Matches(code, @"McpHelpers\.BoundPage\(").Count); + /* Four since #5867 added the Query Store compose failure list. */ + Assert.Equal(4, Regex.Matches(code, @"McpHelpers\.BoundPage\(").Count); } /// The web mirror advertises and binds the three new parameters, so /api/read and the triage page diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs index 5934c76999..aa29f9a319 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingSelfAlertEvaluator.cs @@ -987,6 +987,24 @@ state does not collide with the generic hold's — a raw relation reads BOTH con /// Prefixes the fleet-level raw-purge-over-horizon alert serverKey so it never parses as a server_id. private const string RawPurgeOverHorizonKeyPrefix = "rawpurgehorizon:"; + /* Query Store Compose Hours Stuck edge state (#5867). FLEET-level, ONE key: the store has hours of the Query Store compose + rollup whose build has failed often enough to be retried only once a day (QueryStoreComposeStamp.FailuresBeforeBackoff, or a + statement timeout under the extended timeout). STANDING, the Raw Purge Over Horizon idiom: fire once, re-fire on the state + cooldown while it persists, one clearing resolution when no hour is in backoff. */ + private readonly ConcurrentDictionary _activeComposeStuck = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _lastComposeStuckAlert = new(StringComparer.Ordinal); + + /// The #5867 alert metric: hours of the Query Store compose rollup in daily backoff. + internal const string ComposeHoursStuckMetric = "Compose Rollup Hours Stuck"; + + /// The resolution title clears with. + internal const string ComposeHoursStuckClearedMetric = "Compose Rollup Hours Built"; + + /// Prefixes the fleet-level compose alert serverKey so it never parses as a server_id. + private const string ComposeHoursStuckKeyPrefix = "composestuck:"; + + private const string ComposeHoursStuckKey = "querystore"; + /* Notification Channel Failing edge state (#4750). FLEET-level, MULTI-keyed by channel name (Teams, Slack, Generic, PagerDuty). A webhook channel counts its own failures in a row and logs the first few, but nothing else showed it, so a channel could fail for weeks while another one delivered every alert. @@ -7094,6 +7112,86 @@ await RecordResolutionAsync(new AlertResolution( } } + /// + /// The isolating entry point for the #5867 Compose Rollup Hours Stuck check: the worker calls it after each compose tick with the + /// failure record it read (). Same failure isolation as + /// . + /// + public async Task EvaluateComposeHoursStuckAsync( + IReadOnlyList failures, CancellationToken cancellationToken) + { + try + { + await ApplyComposeHoursStuckAsync(failures, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + /* NOT counted by #3013's swallowed-read counter: the failure record is a parameter; the read that produces it is + isolated in DarlingWorker.BuildQueryStoreComposeStampAsync. */ + _logger?.LogError("Compose-rollup-hours-stuck self-alert failed: {Message}", ex.Message); + } + } + + /// + /// Applies the fleet-level Compose Rollup Hours Stuck condition (#5867): one WARNING while any hour of the Query Store compose + /// rollup is in backoff (: its build has failed + /// times in a row, or timed out under the extended timeout, and is now + /// tried once a day). The alert names how many hours, the oldest, its count and its last reason; get_store_metrics lists + /// them all. Hours that failed but are still retried every tick do not fire it. A STANDING condition, the Raw Purge Over Horizon + /// idiom: fire once, re-fire on the cooldown while it persists, one clearing resolution once no hour is in backoff. Gated on the + /// master alerts switch. Internal so it pins directly with a recording deliverer and a controllable clock. + /// + internal async Task ApplyComposeHoursStuckAsync( + IReadOnlyList failures, CancellationToken cancellationToken) + { + if (!_settings.AlertsEnabled) + { + return; + } + + var now = _utcNow(); + var key = ComposeHoursStuckKey; + var stuck = failures.Where(f => f.InBackoff).OrderBy(f => f.Hour).ToList(); + if (stuck.Count > 0) + { + _activeComposeStuck[key] = true; + var decision = DecideStateAlert(_lastComposeStuckAlert, key, ComposeHoursStuckMetric, key, now, out var refire); + if (decision is ConnectionAlertDecision.Lost or ConnectionAlertDecision.StillDown) + { + _lastComposeStuckAlert[key] = now; + var oldest = stuck[0]; + var reason = string.IsNullOrEmpty(oldest.LastSqlState) ? oldest.LastReason : $"{oldest.LastSqlState}: {oldest.LastReason}"; + var delivery = await FireAsync( + StoreKey(ComposeHoursStuckKeyPrefix + key), _storeLabel, ComposeHoursStuckMetric, + $"{stuck.Count} hour(s)", "0 hours", + detail: StateRepeatNote(decision, refire) + + $"{stuck.Count} hour(s) of the Query Store compose rollup failed to build {QueryStoreComposeStamp.FailuresBeforeBackoff} times in a row " + + "(or timed out under the longer timeout) and are now retried once a day. " + + $"The oldest is {oldest.Hour:yyyy-MM-dd HH}:00 UTC: {oldest.Failures} failure(s), last reason {reason}, next attempt {oldest.NextAttemptAt:yyyy-MM-dd HH:mm} UTC. " + + "get_store_metrics lists every such hour under query_store_compose; the service log's 'Query Store compose rollup: building hour' " + + "lines carry each failure.", + severity: AlertSeverityLevel.Warning, + shortMessage: $"{stuck.Count} Query Store compose hour(s) retried once a day - {reason}", + numericCurrentValue: stuck.Count, + numericThresholdValue: 0, + cancellationToken); + NoteStateSend(ComposeHoursStuckMetric, key, delivery); + } + } + else if (_activeComposeStuck.TryRemove(key, out var was) && was) + { + EndStateAlert(_lastComposeStuckAlert, key, ComposeHoursStuckMetric, key); + await RecordResolutionAsync(new AlertResolution( + StoreKey(ComposeHoursStuckKeyPrefix + key), _storeLabel, ComposeHoursStuckMetric, + ComposeHoursStuckClearedMetric, + $"{_storeLabel}: no Query Store compose hour is in daily backoff any more"), cancellationToken); + } + } + /// Plain-words rendering of a 's outcome: the alert names the /// reason, one of repair pending, a hole, not covered, or a failed run (with SqlState). /// null (never recorded yet) reads as "not covered" — the trigger has not recorded a run for this diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs index 156b538849..8380f84697 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingTriageEndpoint.cs @@ -143,6 +143,7 @@ to grow (a NEW store family needs its alias here, but no section mapping). */ (DarlingSelfAlertEvaluator.CheckpointerPressureRecoveredMetric, DarlingSelfAlertEvaluator.CheckpointerPressureMetric), (DarlingSelfAlertEvaluator.StoreSettingsResolvedMetric, DarlingSelfAlertEvaluator.StoreSettingsMetric), (DarlingSelfAlertEvaluator.RawPurgeOverHorizonClearedMetric, DarlingSelfAlertEvaluator.RawPurgeOverHorizonMetric), + (DarlingSelfAlertEvaluator.ComposeHoursStuckClearedMetric, DarlingSelfAlertEvaluator.ComposeHoursStuckMetric), (DarlingSelfAlertEvaluator.NotificationChannelRecoveredMetric, DarlingSelfAlertEvaluator.NotificationChannelFailingMetric), }; diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs index 0b9745abc7..cb19c12481 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingWorker.cs @@ -12837,6 +12837,39 @@ await QueryStoreComposeStamp.RunTickAsync( { _logger.LogWarning("Query Store compose rollup could not run; the next hourly tick retries: {Message}", ex.Message); } + + /* #5867: the hours whose build now waits a day between attempts raise the Compose Rollup Hours Stuck self-alert, from its own + read after the tick. */ + if (_selfAlerts is not null) + { + await EvaluateComposeHoursStuckAsync(stoppingToken); + } + } + + /// + /// #5867: reads the Query Store compose builder's failure record and hands it to + /// . A failed read sends no report, so the standing alert + /// is held, never resolved on uncertainty, and the next hourly compose tick reads again. Exempt from #3013's swallowed-read + /// counter rather than counted (unlike #5654's): counting it would also add a read to the fleet-scoped inventory that both SKUs' + /// get_collection_health descriptions carry. + /// + private async Task EvaluateComposeHoursStuckAsync(CancellationToken cancellationToken) + { + IReadOnlyList failures; + try + { + await using var connection = await _postgres!.OpenConnectionAsync(cancellationToken); + failures = await QueryStoreComposeStamp.ReadFailuresAsync(connection, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning( + "Query Store compose rollup: the failure record could not be read for the self-alert; the standing alert is held this tick: {Message}", + ex.Message); + return; + } + + await _selfAlerts!.EvaluateComposeHoursStuckAsync(failures, cancellationToken); } /// diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs index 47d50404a9..293b625f06 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpStoreMetricsTools.cs @@ -37,7 +37,7 @@ namespace PerformanceMonitor.Darling.Service.Mcp; /// Summary first (#3903). It used to return every object's latest row AND its daily series on /// every call. The object count is fixed by the schema (about 250: 72 hypertables, 25 aggregates, 146 jobs), /// not by the fleet, so that was 1.9 MB on every production store, 91% of it the per-object series, and more -/// than any MCP client's context holds. The default is now the store-level blocks plus three ranked lists bounded +/// than any MCP client's context holds. The default is now the store-level blocks plus four ranked lists bounded /// by limit, each row carrying its change over the window in place of its series; object_kind /// lists one kind and an exact object_name returns one object's series. /// @@ -74,12 +74,12 @@ public sealed class DarlingMcpStoreMetricsTools [McpServerTool(Name = "get_store_metrics"), Description( "Gets the monitoring STORE's size/growth/health, not a monitored server's; no server_name. Default is a " - + "SUMMARY: store blocks + 3 ranked lists bounded by limit; object_kind/object_name narrow it. Daily " + + "SUMMARY: store blocks + 4 ranked lists bounded by limit; object_kind/object_name narrow it. Daily " + "points are each day's LAST snapshot, never a max/mean; a longest-run-that-day question can't be " + "answered. job_history is ownership-filtered: empty may mean no visibility, not no jobs; check " + "execution_logging/owner_evidence. retention.held policies are PAUSED by the rollup gate; arming by " + "hand drops history's only copy. inventory.reconciled=false flags bytes no row explains. <> " - + "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot per store object: each hypertable, and each continuous aggregate (object_kind continuous_aggregate, sized through its materialization, because timescaledb_information.hypertables never lists a materialization), with total and pre/post-compression bytes and chunk count; the query-text and query-plan payload dimensions (object_kind dimension) with estimated row counts (the planner reltuples estimate, null before the first vacuum or analyze); the named plain tables (object_kind table: collect.query_store_text, collect.query_store_plan_map, config.config_alert_log); two catch-alls with byte total and relation count, object_kind other (user-schema relations no named row accounts for) and object_kind system (catalogs and TimescaleDB bookkeeping); the whole store with the enabled-server count; and each TimescaleDB background job (object_kind background_job) with its last run duration, schedule interval, duration_vs_cadence_percent and run/failure totals. SUMMARY FIRST: by default it returns the store-level blocks (inventory, retention, job_history, checkpointer; below), the whole-store daily growth with the per-server ingest rate (daily growth / enabled servers, the number onboarding N servers multiplies; each daily_growth point carries metric_time = that day's last snapshot, span_days = 1 (a pair is left out unless its days are consecutive and its two snapshots are between 12 and 36 hours apart, except that today's partial point skips the 12-hour floor but not the 36-hour ceiling, so a hole in the list is missing snapshots, not zero growth) and partial = true for today, whose day is not over), and three ranked lists, each bounded by limit with its match count and truncated: objects (the largest byte-bearing objects), fastest_growing (by growth_bytes over the window) and background_jobs (jobs whose failures grew in the window first, then by duration_vs_cadence_percent). Every date and time in the response is UTC: day and delta_since are UTC dates, and each timestamp (metric_time, as_of, sweep_at and the others) is a UTC time, so a day is a UTC day, not the caller's or the server's local one. Each row carries its change over the window from delta_since instead of a per-object series. object_kind lists every object of one kind; object_name drills in: an exact name returns that object's daily series over days_back, a partial name lists the objects whose names contain it. The inventory block RECONCILES the newest sweep against its own pg_database_size: enumerated_percent is the share under named objects, attributed_percent the share any row accounts for, residual_bytes what no row explains, and reconciled is false, a finding, when that residual exceeds the larger of 1% and 64 MiB. bytes_by_kind answers 'what is driving growth' in one glance, and largest_unenumerated names, live, the biggest relations inside other. Each continuous_aggregate row carries, live from the catalog, compression_enabled and its refresh, compression and retention policy job ids. The retention block lists, live, every retention policy the rollup-coverage gate holds PAUSED, with the tier's actual span and how many times its drop_after horizon it is holding: a held policy looks healthy in the stored job telemetry and is a common cause of unexplained growth. Each daily point is that day's LAST snapshot, never its maximum or its mean, so a MAXIMUM question (a job's longest run that day) cannot be answered from it: the day's peak is DROPPED rather than smoothed, and the hourly sample misses a run longer than its offset altogether. The per-run route is timescaledb_information.job_history, which records a SUCCESSFUL run only while timescaledb.enable_job_execution_logging is on, and that setting defaults OFF (a FAILED run's row is written regardless of it), so on a store that never turned it on a maximum over it covers zero rows of successful runs. The job_history block in every response reports that setting's EFFECTIVE value and source: read it before treating an empty job_history as an answer. Whether YOU see rows is a second question, because job_history is ownership-filtered — its rows are visible only to members of the job's owner role or of the database owner, while the jobs and job_stats views show every role every job. So the block reports which role it read as (reader_role), that role's standing under the view's own predicate (visibility: All, Partial or None), the rows it observed over a fixed 24-hour window (rows_observed, newest_row_at) beside the jobs job_stats says started a run in it (jobs_run_in_window), and contradiction, true only when recording is on, a reader admitted to every job's history saw no rows, and jobs ran. In managed mode this tool reads as the least-privilege mcp role, which the view filters out. The role that can see is the service's owner, and the service's hourly self-metrics sweep runs as it and persists the owner's own count over the same 24-hour window, reported as owner_evidence (Observed, Stale, Filtered or Absent), owner_role, owner_observed_at, owner_rows_observed, owner_newest_row_at and owner_jobs_run_in_window: a measurement taken by the service's sweep rather than by this connection. TOAST UTILISATION (V137): each dimension row also carries toast_bytes, toast_live_bytes, toast_utilisation_pct and toast_utilisation_note, because plan XML and statement text do not live in the dimension's heap — they live in its TOAST file, and pg_total_relation_size says how big that file is and nothing about how FULL it is. Row churn leaves free space in that file that ordinary VACUUM returns to the table and never to the operating system. toast_utilisation_pct is toast_live_bytes / toast_bytes (one decimal), null when either is null or the file is empty; toast_live_bytes needs the pg_freespacemap extension, which the bundled store image ships and does not install, and without it the note says 'not measured' and NOTHING is computed from total_bytes or a tuple share. Under 50 % on a file over 10 GiB the note carries the reclaim path: --recompress-plan-dim --vacuum-full, at the MAINTAINER's word in a maintenance window, because VACUUM FULL takes an ACCESS EXCLUSIVE lock on the dimension and needs free disk for a full copy of the live data; the informational Store TOAST Slack self-alert fires daily on the same condition. This tool never reclaims anything by itself. Hypertable, aggregate, table and catch-all objects carry the four TOAST fields as null. CHECKPOINTER (V137): the checkpointer block is the store's OWN checkpointer over the last self-metrics interval: write_ms and sync_ms (milliseconds in its write and sync/fsync phases), requested (checkpoints started by WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement rather than by checkpoint_timeout; the counter does not say which), interval_seconds (MEASURED between the two sweeps that bound it, not the assumed cadence), and the cumulative counters beside them. status says whether there IS an interval: Observed, NoPrevious (one row so far), Reset (a counter went backwards — a stats reset or restart between sweeps, so no delta is stated rather than a negative one), Restarted (the store's postmaster restarted between the two sweeps, postmaster_restarted true: PostgreSQL counts the shutdown checkpoint as requested and keeps the count across the restart, and that checkpoint's own write and sync phases land in the same counters, so no delta is stated and the self-alert judges neither arm; the next sweep's interval is judged normally), or Absent; postmaster_start_time is when the store's postmaster that produced the newest row started (null on a row older than V139). longest_sync_ms and longest_sync_at (V156) are the interval's longest single sync, found by a once-a-minute sample of the counter and stored on the newer row, and the UTC time of the sample that saw it; both are null on a row from before V156, in an hour with no sample, and whenever status is not Observed. timed (the checkpoints started by checkpoint_timeout inside the interval; cumulative_timed is the counter behind it) and checkpoint_count (timed plus requested, the checkpoints the interval held) are null unless status is Observed and both rows are from V140 or later, average_sync_ms_per_checkpoint is sync_ms divided by checkpoint_count (null when checkpoint_count is null or zero), and sync_bar_ms is the per-checkpoint bar in milliseconds that average is judged against, as longest_sync_ms is. The informational Store Checkpointer Pressure self-alert fires when the interval's average sync per checkpoint exceeds 10,000 ms, when its longest single sync does, or when requested exceeds 0, naming WAL sizing and refresh slicing as the levers. The checkpointer row is never an object row: it carries no bytes.")] + + "Gets the monitoring store's OWN size and growth metrics — not a monitored SQL Server's. The service records an hourly self-metrics snapshot per store object: each hypertable, and each continuous aggregate (object_kind continuous_aggregate, sized through its materialization, because timescaledb_information.hypertables never lists a materialization), with total and pre/post-compression bytes and chunk count; the query-text and query-plan payload dimensions (object_kind dimension) with estimated row counts (the planner reltuples estimate, null before the first vacuum or analyze); the named plain tables (object_kind table: collect.query_store_text, collect.query_store_plan_map, config.config_alert_log); two catch-alls with byte total and relation count, object_kind other (user-schema relations no named row accounts for) and object_kind system (catalogs and TimescaleDB bookkeeping); the whole store with the enabled-server count; and each TimescaleDB background job (object_kind background_job) with its last run duration, schedule interval, duration_vs_cadence_percent and run/failure totals. SUMMARY FIRST: by default it returns the store-level blocks (inventory, retention, job_history, checkpointer; below), the whole-store daily growth with the per-server ingest rate (daily growth / enabled servers, the number onboarding N servers multiplies; each daily_growth point carries metric_time = that day's last snapshot, span_days = 1 (a pair is left out unless its days are consecutive and its two snapshots are between 12 and 36 hours apart, except that today's partial point skips the 12-hour floor but not the 36-hour ceiling, so a hole in the list is missing snapshots, not zero growth) and partial = true for today, whose day is not over), and four ranked lists, each bounded by limit with its match count and truncated: objects (the largest byte-bearing objects), fastest_growing (by growth_bytes over the window), background_jobs (jobs whose failures grew in the window first, then by duration_vs_cadence_percent) and query_store_compose.hours (the Query Store compose rollup hours whose build failed, newest first). Every date and time in the response is UTC: day and delta_since are UTC dates, and each timestamp (metric_time, as_of, sweep_at and the others) is a UTC time, so a day is a UTC day, not the caller's or the server's local one. Each row carries its change over the window from delta_since instead of a per-object series. object_kind lists every object of one kind; object_name drills in: an exact name returns that object's daily series over days_back, a partial name lists the objects whose names contain it. The inventory block RECONCILES the newest sweep against its own pg_database_size: enumerated_percent is the share under named objects, attributed_percent the share any row accounts for, residual_bytes what no row explains, and reconciled is false, a finding, when that residual exceeds the larger of 1% and 64 MiB. bytes_by_kind answers 'what is driving growth' in one glance, and largest_unenumerated names, live, the biggest relations inside other. Each continuous_aggregate row carries, live from the catalog, compression_enabled and its refresh, compression and retention policy job ids. The retention block lists, live, every retention policy the rollup-coverage gate holds PAUSED, with the tier's actual span and how many times its drop_after horizon it is holding: a held policy looks healthy in the stored job telemetry and is a common cause of unexplained growth. Each daily point is that day's LAST snapshot, never its maximum or its mean, so a MAXIMUM question (a job's longest run that day) cannot be answered from it: the day's peak is DROPPED rather than smoothed, and the hourly sample misses a run longer than its offset altogether. The per-run route is timescaledb_information.job_history, which records a SUCCESSFUL run only while timescaledb.enable_job_execution_logging is on, and that setting defaults OFF (a FAILED run's row is written regardless of it), so on a store that never turned it on a maximum over it covers zero rows of successful runs. The job_history block in every response reports that setting's EFFECTIVE value and source: read it before treating an empty job_history as an answer. Whether YOU see rows is a second question, because job_history is ownership-filtered — its rows are visible only to members of the job's owner role or of the database owner, while the jobs and job_stats views show every role every job. So the block reports which role it read as (reader_role), that role's standing under the view's own predicate (visibility: All, Partial or None), the rows it observed over a fixed 24-hour window (rows_observed, newest_row_at) beside the jobs job_stats says started a run in it (jobs_run_in_window), and contradiction, true only when recording is on, a reader admitted to every job's history saw no rows, and jobs ran. In managed mode this tool reads as the least-privilege mcp role, which the view filters out. The role that can see is the service's owner, and the service's hourly self-metrics sweep runs as it and persists the owner's own count over the same 24-hour window, reported as owner_evidence (Observed, Stale, Filtered or Absent), owner_role, owner_observed_at, owner_rows_observed, owner_newest_row_at and owner_jobs_run_in_window: a measurement taken by the service's sweep rather than by this connection. TOAST UTILISATION (V137): each dimension row also carries toast_bytes, toast_live_bytes, toast_utilisation_pct and toast_utilisation_note, because plan XML and statement text do not live in the dimension's heap — they live in its TOAST file, and pg_total_relation_size says how big that file is and nothing about how FULL it is. Row churn leaves free space in that file that ordinary VACUUM returns to the table and never to the operating system. toast_utilisation_pct is toast_live_bytes / toast_bytes (one decimal), null when either is null or the file is empty; toast_live_bytes needs the pg_freespacemap extension, which the bundled store image ships and does not install, and without it the note says 'not measured' and NOTHING is computed from total_bytes or a tuple share. Under 50 % on a file over 10 GiB the note carries the reclaim path: --recompress-plan-dim --vacuum-full, at the MAINTAINER's word in a maintenance window, because VACUUM FULL takes an ACCESS EXCLUSIVE lock on the dimension and needs free disk for a full copy of the live data; the informational Store TOAST Slack self-alert fires daily on the same condition. This tool never reclaims anything by itself. Hypertable, aggregate, table and catch-all objects carry the four TOAST fields as null. CHECKPOINTER (V137): the checkpointer block is the store's OWN checkpointer over the last self-metrics interval: write_ms and sync_ms (milliseconds in its write and sync/fsync phases), requested (checkpoints started by WAL volume reaching max_wal_size, a base backup, or a CHECKPOINT statement rather than by checkpoint_timeout; the counter does not say which), interval_seconds (MEASURED between the two sweeps that bound it, not the assumed cadence), and the cumulative counters beside them. status says whether there IS an interval: Observed, NoPrevious (one row so far), Reset (a counter went backwards — a stats reset or restart between sweeps, so no delta is stated rather than a negative one), Restarted (the store's postmaster restarted between the two sweeps, postmaster_restarted true: PostgreSQL counts the shutdown checkpoint as requested and keeps the count across the restart, and that checkpoint's own write and sync phases land in the same counters, so no delta is stated and the self-alert judges neither arm; the next sweep's interval is judged normally), or Absent; postmaster_start_time is when the store's postmaster that produced the newest row started (null on a row older than V139). longest_sync_ms and longest_sync_at (V156) are the interval's longest single sync, found by a once-a-minute sample of the counter and stored on the newer row, and the UTC time of the sample that saw it; both are null on a row from before V156, in an hour with no sample, and whenever status is not Observed. timed (the checkpoints started by checkpoint_timeout inside the interval; cumulative_timed is the counter behind it) and checkpoint_count (timed plus requested, the checkpoints the interval held) are null unless status is Observed and both rows are from V140 or later, average_sync_ms_per_checkpoint is sync_ms divided by checkpoint_count (null when checkpoint_count is null or zero), and sync_bar_ms is the per-checkpoint bar in milliseconds that average is judged against, as longest_sync_ms is. The informational Store Checkpointer Pressure self-alert fires when the interval's average sync per checkpoint exceeds 10,000 ms, when its longest single sync does, or when requested exceeds 0, naming WAL sizing and refresh slicing as the levers. The checkpointer row is never an object row: it carries no bytes.")] public static async Task GetStoreMetrics( NpgsqlDataSource postgres, [Description("Days of history the window deltas and series cover. Default 30; max 400 (the series' own retention).")] int days_back = 30, @@ -160,7 +160,7 @@ last run — indistinguishable from a healthy job in every stored column. On the This checks out its OWN connection (review catch — an earlier comment here claimed it reused one, which was not true of any code path in this method: the two readers above go through - postgres.CreateCommand and leave nothing open). That is a third pooled checkout per call, + postgres.CreateCommand and leave nothing open). That is a third pooled checkout per call, and the Query Store compose read below is a fourth, which is cheap but is not free, and saying so is the point — a comment that overstates what the code does is worse than none. */ List holds; @@ -170,6 +170,21 @@ the code does is worse than none. */ .ToList(); } + /* #5867: the Query Store compose hours whose build failed, and which of them wait a day between attempts. Its own + checkout and failure-isolated to null ("Unreadable"), never to an empty list, which would read as "no hour failing". */ + IReadOnlyList? composeFailures; + try + { + await using var composeConnection = await postgres.OpenConnectionAsync(cancellationToken); + composeFailures = await QueryStoreComposeStamp.ReadFailuresAsync(composeConnection, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + composeFailures = null; + } + + var (composePage, composeTruncated) = McpHelpers.BoundPage(composeFailures ?? Array.Empty(), limit); + var heldPolicies = holds .Where(h => !h.Armed) .OrderByDescending(h => h.OverHorizonRatio ?? 0) @@ -362,6 +377,38 @@ a store and let a growth investigation trust it. Null fields where the newest sw over_horizon_ratio = h.OverHorizonRatio is { } r ? Math.Round(r, 2) : (double?)null, }), }, + /* #5867. Present on EVERY response, for the #2813 reason above: "no compose hour is failing" and an absent block + must not look alike. */ + query_store_compose = new + { + state = composeFailures is null ? "Unreadable" : "Observed", + failing_hours = composeFailures?.Count, + hours_in_backoff = composeFailures?.Count(f => f.InBackoff), + hours_returned = composePage.Count, + truncated = composeTruncated, + note = composeFailures is null + ? "The failure record of the Query Store compose rollup could not be read." + : composeFailures.Count == 0 + ? "No Query Store compose hour is failing to build." + : "These hours of the Query Store compose rollup failed to build. An hour is retried next tick (after a statement timeout, under a " + + QueryStoreComposeStamp.ExtendedBuildStatementTimeoutSeconds.ToString(CultureInfo.InvariantCulture) + + " s timeout) until it has failed " + + QueryStoreComposeStamp.FailuresBeforeBackoff.ToString(CultureInfo.InvariantCulture) + + " times in a row or timed out under the longer timeout; then in_backoff is true and it is tried once a " + + "day, at retry_at. A successful build removes the hour from this list. Times are UTC.", + hours = composePage.Select(f => new + { + hour = f.Hour.ToString("yyyy-MM-dd HH:mm", CultureInfo.InvariantCulture), + failures = f.Failures, + in_backoff = f.InBackoff, + last_sqlstate = f.LastSqlState, + last_reason = f.LastReason, + last_timeout_seconds = f.LastTimeoutSeconds, + first_failed_at = f.FirstFailedAt.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture), + last_failed_at = f.LastFailedAt.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture), + retry_at = f.NextAttemptAt.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture), + }), + }, /* #3175. Present on EVERY response, for the #2813 reason one line up and one step further: here the absence being guarded is the absence of ROWS in the instrument this description sends a maximum question to, so an absent block would leave the redirect unqualified diff --git a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs index e3d56023ca..ccc92ecb47 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/PgMigrations.cs @@ -1667,8 +1667,19 @@ holds the ordering. */ new Migration(186, "collection-log-hourly", V186Sql), new Migration(187, "alert-resolution-delivery", V187Sql), new Migration(188, "investigation-alert-facts", V188Sql), + new Migration(189, "query-store-compose-failures", V189Sql), }; + /// + /// V189 (#5867): collect.query_store_compose_stamp_failures, the failure record of the Query Store compose builder: one row + /// per hour whose last build threw, with its consecutive failures, last error and next attempt, so a build that cannot finish backs + /// off to one attempt a day instead of running every tick, and get_store_metrics and the self-alert can show it + /// (). A plain empty table, no GRANT (the collect schema's blanket + /// SELECT covers it). No Lite twin: Lite has no PostgreSQL store. A const, so the list above reads it whatever the + /// order of the static initializers. + /// + private const string V189Sql = QueryStoreComposeStamp.FailuresCreateSql; + /// /// V114 — the statistics ESTIMATE columns on collect.pg_index_bloat, and the retirement of the /// rotation cursor that the exact census needed (#3234). diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreComposeStamp.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreComposeStamp.cs index f8cf67db30..fbd620a085 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreComposeStamp.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreComposeStamp.cs @@ -91,7 +91,38 @@ public static class QueryStoreComposeStamp private const int CommandTimeoutSeconds = BuildStatementTimeoutSeconds + 30; - private static readonly string StatementTimeoutSql = $"SET LOCAL statement_timeout = '{BuildStatementTimeoutSeconds}s'"; + /// The store schema version of the rung that creates (V189, #5867). The number is the next + /// free rung when this was written; it is renumbered at merge if another rung took it first. + public const int FailuresRungVersion = 189; + + /// The failure record of each hour whose last build threw (#5867): one row per hour, removed when the hour builds. + public const string FailuresTable = "collect.query_store_compose_stamp_failures"; + + /// + /// The consecutive failures after which an hour is retried once a day instead of every tick (#5867). Three, because the failures a + /// retry cures (a deadlock, a lock or serialization failure, a dropped connection, a restart of the store) clear within a tick or + /// two, and at the service's hourly tick three failures in a row are three hours of the same error: past that, an hourly retry + /// only repeats it, and spends the build's timeout every hour. + /// + public const int FailuresBeforeBackoff = 3; + + /// How long an hour in backoff waits between attempts (#5867): one attempt, and one WARN, a day. + public static readonly TimeSpan BackoffInterval = TimeSpan.FromDays(1); + + /// + /// The statement_timeout of the attempt that follows a statement timeout (#5867): four times , + /// 240 s. The normal timeout is sized for the normal hour (about 2.6 s cold); an hour that runs past 60 s is over 20 times that, + /// and four times the timeout gives it room up to about 90 times the normal hour while staying well under the tick's + /// of 10 minutes, so even this attempt, which the tick runs after the other hours, leaves most of the + /// budget. A build that times out under it too backs off at once: the next tick's attempt would be the same build under the same timeout. + /// + public const int ExtendedBuildStatementTimeoutSeconds = BuildStatementTimeoutSeconds * 4; + + /// PostgreSQL's SQLSTATE for a statement cancelled by statement_timeout (query_canceled). + public const string StatementTimeoutSqlState = "57014"; + + private static string StatementTimeoutSql(int seconds) => + "SET LOCAL statement_timeout = '" + seconds.ToString(System.Globalization.CultureInfo.InvariantCulture) + "s'"; /// The alias the compiler gives the fact relation in a partial's text (ComposeCompiler.FactAlias). The /// map's is written over it, verbatim the text TryBuildPartialValueExpr @@ -283,6 +314,29 @@ FOR EACH ROW """; /// The insert column list: collection_time, the partials in map order, then the dimensions. + /// + /// V189 (#5867): the failure record of the hours whose build threw. One row per hour, written by + /// after the failed build rolled back, and deleted by the build that succeeds (in its own transaction) or by the cleanup once the + /// hour falls below the floor. retry_at is on the store's clock, as the plan's "now" is; the plan skips an hour + /// until it is due. last_timeout_seconds is the statement_timeout the failed attempt ran under. Naive UTC. + /// + public const string FailuresCreateSql = """ +/* V189 (#5867): one row per Query Store compose hour whose last build failed: how many times in a row, the last error, and when + the builder tries it again. The builder deletes the row when the hour builds. Naive UTC throughout. */ +CREATE TABLE IF NOT EXISTS collect.query_store_compose_stamp_failures +( + hour timestamp NOT NULL, + failures integer NOT NULL, + last_sqlstate text, + last_reason text NOT NULL, + last_timeout_seconds integer NOT NULL, + first_failed_at timestamp NOT NULL, + last_failed_at timestamp NOT NULL, + retry_at timestamp NOT NULL, + CONSTRAINT pk_query_store_compose_stamp_failures PRIMARY KEY (hour) +); +"""; + private static string InsertColumns() => "collection_time, " + string.Join(", ", PartialColumnMap.Select(c => c.Column)) + ", " + string.Join(", ", DimensionColumns); @@ -470,6 +524,11 @@ INSERT INTO collect.query_store_compose_stamp_hours (hour, built_at) /// per blocked, so hours that cannot build do not take the build slots of the hours that can: an old, never-built hour still builds /// while a writer pins the newest ones (#5582 review round 2). The blocked rows are only counted, for the log; the per-build check stays, /// because a writer can start between the plan and the build. + /// + /// An hour with a failure record (, #5867) is planned only once its retry_at is due by + /// the same clock, so an hour in backoff is not planned at all between its daily attempts. The fourth column is its consecutive + /// failures (0 for none), which uses to run it after the other hours; the fifth is true when its last + /// failure was a statement timeout (its SQLSTATE is $4, ), so its next attempt runs under . /// public const string PlanSql = @" WITH bounds AS @@ -480,6 +539,10 @@ FROM collect.query_store_interval_wide_coverage AS c WHERE sv.is_enabled HAVING min(c.filled_since) IS NOT NULL ), +clock AS +( + SELECT coalesce($3::timestamp, now() AT TIME ZONE 'UTC') AS at +), cutoff AS ( SELECT coalesce(min(coalesce(w.xact_start AT TIME ZONE 'UTC', '-infinity'::timestamp)), 'infinity'::timestamp) AS since @@ -501,18 +564,72 @@ missing AS SELECT g AS hour, false AS stale, (g + interval '2 hours' > cutoff.since) AS blocked FROM bounds CROSS JOIN cutoff - CROSS JOIN LATERAL generate_series(bounds.floor_hour, date_trunc('hour', coalesce($3::timestamp, now() AT TIME ZONE 'UTC') - interval '3 hours'), interval '1 hour') AS g + CROSS JOIN clock + CROSS JOIN LATERAL generate_series(bounds.floor_hour, date_trunc('hour', clock.at - interval '3 hours'), interval '1 hour') AS g WHERE NOT EXISTS (SELECT 1 FROM collect.query_store_compose_stamp_hours AS h WHERE h.hour = g) ) -SELECT p.hour, p.stale, p.blocked +SELECT p.hour, p.stale, p.blocked, p.failures, p.extended FROM ( - SELECT u.hour, u.stale, u.blocked, row_number() OVER (PARTITION BY u.stale, u.blocked ORDER BY u.hour DESC) AS rn + SELECT u.hour, u.stale, u.blocked, coalesce(f.failures, 0) AS failures, + coalesce(f.last_sqlstate = $4::text, false) AS extended, + row_number() OVER (PARTITION BY u.stale, u.blocked ORDER BY u.hour DESC) AS rn FROM (SELECT * FROM stale UNION ALL SELECT * FROM missing) AS u + LEFT JOIN collect.query_store_compose_stamp_failures AS f ON f.hour = u.hour + CROSS JOIN clock + WHERE f.hour IS NULL + OR f.retry_at <= clock.at ) AS p WHERE p.rn <= $2 ORDER BY p.stale DESC, p.blocked, p.hour DESC;"; + /// + /// Records a failed build of an hour (#5867) and returns its consecutive failures, its next attempt, and whether that is a backoff: + /// $1 the hour, $2 the SQLSTATE (NULL when the error had none), $3 the error message, $4 the statement_timeout the attempt ran + /// under, $5 the store clock a test passes (NULL in the service: the store's own, as 's), $6 + /// , $7 , $8 , $9 . The + /// next attempt is the next tick (retry_at = now) until the hour has failed times in + /// a row, or until a build under the extended timeout has timed out too; from then on it is one attempt a day. Runs after the + /// build's transaction rolled back. + /// + public const string RecordFailureSql = @" +INSERT INTO collect.query_store_compose_stamp_failures AS f + (hour, failures, last_sqlstate, last_reason, last_timeout_seconds, first_failed_at, last_failed_at, retry_at) +SELECT $1, 1, $2::text, $3::text, $4::integer, c.at, c.at, + c.at + CASE WHEN 1 >= $6::integer OR ($2::text = $9::text AND $4::integer >= $7::integer) THEN $8::interval ELSE interval '0' END +FROM (SELECT coalesce($5::timestamp, now() AT TIME ZONE 'UTC') AS at) AS c +ON CONFLICT (hour) DO UPDATE +SET failures = f.failures + 1, + last_sqlstate = EXCLUDED.last_sqlstate, + last_reason = EXCLUDED.last_reason, + last_timeout_seconds = EXCLUDED.last_timeout_seconds, + last_failed_at = EXCLUDED.last_failed_at, + retry_at = EXCLUDED.last_failed_at + + CASE WHEN f.failures + 1 >= $6::integer OR (EXCLUDED.last_sqlstate = $9::text AND EXCLUDED.last_timeout_seconds >= $7::integer) + THEN $8::interval ELSE interval '0' END +RETURNING f.failures, f.retry_at, f.retry_at > f.last_failed_at AS backed_off;"; + + /// A successful build clears the hour's failure record (#5867): $1 the hour. In the build's own transaction. + public const string ClearFailureSql = @" +DELETE FROM collect.query_store_compose_stamp_failures +WHERE hour = $1;"; + + /// Removes the failure records below the floor (#5867): $1 the floor. An hour the rollup no longer holds has nothing to retry. + public const string GcFailuresSql = @" +DELETE FROM collect.query_store_compose_stamp_failures +WHERE hour < $1;"; + + /// + /// Every hour with a failure record (#5867), newest first: what get_store_metrics lists and the self-alert reads. An hour is + /// in backoff when its next attempt is later than its last failure (it has failed times in a row, + /// or timed out under the extended timeout); the others are retried next tick. + /// + public const string ReadFailuresSql = @" +SELECT f.hour, f.failures, f.last_sqlstate, f.last_reason, f.last_timeout_seconds, f.first_failed_at, f.last_failed_at, f.retry_at, + f.retry_at > f.last_failed_at AS in_backoff +FROM collect.query_store_compose_stamp_failures AS f +ORDER BY f.hour DESC;"; + /// /// The first step of the cleanup: removes the hours below the floor from the hours table and returns them: $1 the floor. /// The second step removes each returned hour's rollup rows by the (collection_time, server_id) index @@ -533,11 +650,21 @@ DELETE FROM collect.query_store_compose_stamp_built /// Removes one expired hour of rollup rows: $1 the hour. public const string GcRollupSql = DeleteHourSql; - /// One planned build. - public readonly record struct Build(DateTime Hour, bool Stale); + /// One planned build: is the hour's consecutive failed builds (#5867, 0 for none), and + /// is true when the last one was a statement timeout, so this attempt runs under + /// . + public readonly record struct Build(DateTime Hour, bool Stale, int Failures = 0, bool Extended = false); + + /// One hour's failure record (, #5867). Times are naive UTC on the store's clock. + public readonly record struct HourFailure( + DateTime Hour, int Failures, string? LastSqlState, string LastReason, int LastTimeoutSeconds, + DateTime FirstFailedAt, DateTime LastFailedAt, DateTime NextAttemptAt, bool InBackoff); + + /// What wrote: the consecutive failures, the next attempt, and whether that is a backoff. + public readonly record struct FailureOutcome(int Failures, DateTime NextAttemptAt, bool BackedOff); /// What one tick did. - public readonly record struct TickResult(int BuiltStale, int BuiltMissing, int Failed, long HoursRemoved, int Deferred = 0, int DeferredForWriters = 0, int HeldBack = 0) + public readonly record struct TickResult(int BuiltStale, int BuiltMissing, int Failed, long HoursRemoved, int Deferred = 0, int DeferredForWriters = 0, int HeldBack = 0, int BackedOff = 0) { public int Built => BuiltStale + BuiltMissing; } @@ -568,6 +695,10 @@ public static DateTime FloorHour(DateTime nowUtc, int retentionDays) /// tick's builds, so a storm of late writes (a backfill, a pending replay) that keeps marking built hours stale cannot starve the /// never-built backlog (a new store's first ~190 hours); with none waiting, stale hours may take every build. The reserved /// half goes to the never-built hours and no further (#5582 review round 1). + /// + /// The hours that have failed before ( above 0, #5867) run after the others, in the same order among + /// themselves: such a build may run up to its timeout again ( after a timeout), and + /// the tick's wall-clock budget () goes first to the hours that build. /// public static IReadOnlyList ChooseBuilds(IReadOnlyList candidates, int max) { @@ -578,7 +709,8 @@ public static IReadOnlyList ChooseBuilds(IReadOnlyList candidates, var staleCap = missing.Count > 0 ? max / 2 : max; var chosen = stale.Take(staleCap).ToList(); chosen.AddRange(missing.Take(Math.Max(0, max - chosen.Count))); - return chosen; + /* OrderBy is stable: the order above holds within each group. */ + return chosen.OrderBy(b => b.Failures > 0).ToList(); } /// One plan: the hours to build, and how many hours the plan held back because an open writer of the wide table would defer them. @@ -602,6 +734,7 @@ public static async Task PlanAsync( command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = FloorHour(nowUtc, retentionDays) }); command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = MaxBuildsPerTick }); command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = storeNow is { } at ? Unspecified(at) : DBNull.Value }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = StatementTimeoutSqlState }); await using var reader = await command.ExecuteReaderAsync(cancellationToken); while (await reader.ReadAsync(cancellationToken)) { @@ -611,7 +744,7 @@ public static async Task PlanAsync( continue; } - builds.Add(new Build(reader.GetDateTime(0), reader.GetBoolean(1))); + builds.Add(new Build(reader.GetDateTime(0), reader.GetBoolean(1), reader.GetInt32(3), reader.GetBoolean(4))); } return new Plan(ChooseBuilds(builds, MaxBuildsPerTick), heldBack); @@ -733,6 +866,12 @@ public static async Task GcAsync( await built.ExecuteNonQueryAsync(cancellationToken); } + await using (var failures = new NpgsqlCommand(GcFailuresSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + { + failures.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = floor }); + await failures.ExecuteNonQueryAsync(cancellationToken); + } + foreach (var hour in expired) { await using var rollup = new NpgsqlCommand(GcRollupSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }; @@ -748,27 +887,44 @@ public static async Task GcAsync( /// Builds one hour in one transaction (the five steps in the type's summary). Returns the wide rows read, or null when the hour /// was DEFERRED because a writing transaction that could hold an unmarked row of it is still open (); /// a deferred hour changes nothing and is planned again next tick. An hour with no wide rows still gets its row in the hours table. + /// is the transaction's statement_timeout: , + /// or for the attempt after a timeout (#5867). A build that commits clears the + /// hour's failure record in the same transaction. /// public static async Task BuildHourAsync( - NpgsqlConnection connection, DateTime hour, DateTime nowUtc, CancellationToken cancellationToken, Func? beforeCommit = null) + NpgsqlConnection connection, DateTime hour, DateTime nowUtc, CancellationToken cancellationToken, Func? beforeCommit = null, + int statementTimeoutSeconds = BuildStatementTimeoutSeconds, Action? commandCreated = null) { ArgumentNullException.ThrowIfNull(connection); + ArgumentOutOfRangeException.ThrowIfLessThan(statementTimeoutSeconds, 1); var hourValue = Unspecified(hour); + /* The client waits 30 s past the server's timeout, so the server's cancel (57014, which the failure record keys on) arrives first. */ + var commandTimeout = Math.Max(CommandTimeoutSeconds, statementTimeoutSeconds + 30); await using var transaction = await connection.BeginTransactionAsync(cancellationToken); - await using (var timeout = new NpgsqlCommand(StatementTimeoutSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + /* The attempt's client timeout is one value for every command in it (#5871): the server grants each statement the whole + statement_timeout, so a command left at the short client timeout would be cancelled client-side first, as a timeout + that carries no 57014. commandCreated is a test seam that sees each command as it is made. */ + NpgsqlCommand NewCommand(string sql) + { + var created = new NpgsqlCommand(sql, connection, transaction) { CommandTimeout = commandTimeout }; + commandCreated?.Invoke(created); + return created; + } + + await using (var timeout = NewCommand(StatementTimeoutSql(statementTimeoutSeconds))) { await timeout.ExecuteNonQueryAsync(cancellationToken); } - await using (var advisory = new NpgsqlCommand(HourLockSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var advisory = NewCommand(HourLockSql)) { advisory.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = HourNumber(hourValue) }); await advisory.ExecuteNonQueryAsync(cancellationToken); } - await using (var writers = new NpgsqlCommand(OpenWriterSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var writers = NewCommand(OpenWriterSql)) { writers.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); /* One row (the oldest such writer) when an hour would defer, none when no writer of the wide table could hold an unmarked row. */ @@ -781,7 +937,7 @@ public static async Task GcAsync( var servers = new List(); var lateSeqs = new List(); - await using (var readPairs = new NpgsqlCommand(ReadPairsSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var readPairs = NewCommand(ReadPairsSql)) { readPairs.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); await using var reader = await readPairs.ExecuteReaderAsync(cancellationToken); @@ -792,26 +948,26 @@ public static async Task GcAsync( } } - await using (var delete = new NpgsqlCommand(DeleteHourSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var delete = NewCommand(DeleteHourSql)) { delete.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); await delete.ExecuteNonQueryAsync(cancellationToken); } - await using (var insert = new NpgsqlCommand(BuildHourInsertSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var insert = NewCommand(BuildHourInsertSql)) { insert.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); await insert.ExecuteNonQueryAsync(cancellationToken); } long sourceRows; - await using (var count = new NpgsqlCommand(SourceRowsSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var count = NewCommand(SourceRowsSql)) { count.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); sourceRows = Convert.ToInt64(await count.ExecuteScalarAsync(cancellationToken)); } - await using (var built = new NpgsqlCommand(UpsertBuiltSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var built = NewCommand(UpsertBuiltSql)) { built.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); built.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = Unspecified(nowUtc) }); @@ -820,13 +976,19 @@ public static async Task GcAsync( await built.ExecuteNonQueryAsync(cancellationToken); } - await using (var hours = new NpgsqlCommand(UpsertHourSql, connection, transaction) { CommandTimeout = CommandTimeoutSeconds }) + await using (var hours = NewCommand(UpsertHourSql)) { hours.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); hours.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = Unspecified(nowUtc) }); await hours.ExecuteNonQueryAsync(cancellationToken); } + await using (var clear = NewCommand(ClearFailureSql)) + { + clear.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = hourValue }); + await clear.ExecuteNonQueryAsync(cancellationToken); + } + /* Test seam: runs with the build's row locks still held and nothing committed (a late write that bumps a pair the build has already upserted waits here for the commit). */ if (beforeCommit is not null) @@ -838,9 +1000,82 @@ public static async Task GcAsync( return sourceRows; } + /// Records a failed build (, #5867) on , outside any transaction. + public static async Task RecordFailureAsync( + NpgsqlConnection connection, DateTime hour, string? sqlState, string reason, int timeoutSeconds, DateTime? storeNow, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(connection); + + await using var command = new NpgsqlCommand(RecordFailureSql, connection) { CommandTimeout = CommandTimeoutSeconds }; + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = Unspecified(hour) }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = (object?)sqlState ?? DBNull.Value }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = reason ?? string.Empty }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = timeoutSeconds }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = storeNow is { } at ? Unspecified(at) : DBNull.Value }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = FailuresBeforeBackoff }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = ExtendedBuildStatementTimeoutSeconds }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Interval, Value = BackoffInterval }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = StatementTimeoutSqlState }); + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + await reader.ReadAsync(cancellationToken); + return new FailureOutcome(reader.GetInt32(0), reader.GetDateTime(1), reader.GetBoolean(2)); + } + + /// Every hour with a failure record (, #5867), newest first. Empty on a store below + /// (the table is absent there). + public static async Task> ReadFailuresAsync(NpgsqlConnection connection, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(connection); + + var rows = new List(); + await using (var probe = new NpgsqlCommand("SELECT to_regclass('" + FailuresTable + "') IS NOT NULL;", connection) { CommandTimeout = CommandTimeoutSeconds }) + { + if (await probe.ExecuteScalarAsync(cancellationToken) is not true) + { + return rows; + } + } + + await using var command = new NpgsqlCommand(ReadFailuresSql, connection) { CommandTimeout = CommandTimeoutSeconds }; + await using var reader = await command.ExecuteReaderAsync(cancellationToken); + while (await reader.ReadAsync(cancellationToken)) + { + rows.Add(new HourFailure( + reader.GetDateTime(0), reader.GetInt32(1), reader.IsDBNull(2) ? null : reader.GetString(2), reader.GetString(3), reader.GetInt32(4), + reader.GetDateTime(5), reader.GetDateTime(6), reader.GetDateTime(7), reader.GetBoolean(8))); + } + + return rows; + } + + /// + /// The WARN for a failed build (#5867). Before the backoff: one line per attempt, saying the hour is retried next tick (and under the + /// extended timeout after a statement timeout). In backoff, the build runs once a day, so this is one line a day, naming the + /// consecutive failures, the last reason and the next attempt. + /// + private static void LogFailure(ILogger logger, Exception ex, Build item, string? sqlState, FailureOutcome? outcome) + { + if (outcome is { BackedOff: true } backoff) + { + logger.LogWarning(ex, + "Query Store compose rollup: building hour {Hour:yyyy-MM-dd HH}:00 (stale {Stale}) has failed {Failures} time(s) in a row; it is retried once a day, next at {Next:yyyy-MM-dd HH:mm} UTC, and get_store_metrics lists it: {Message}", + item.Hour, item.Stale, backoff.Failures, backoff.NextAttemptAt, ex.Message); + return; + } + + var extendedNext = string.Equals(sqlState, StatementTimeoutSqlState, StringComparison.Ordinal); + logger.LogWarning(ex, + "Query Store compose rollup: building hour {Hour:yyyy-MM-dd HH}:00 (stale {Stale}) failed ({Failures} time(s) in a row) and is retried next tick{Timeout}: {Message}", + item.Hour, item.Stale, outcome?.Failures ?? item.Failures + 1, + extendedNext ? string.Create(System.Globalization.CultureInfo.InvariantCulture, $" under a {ExtendedBuildStatementTimeoutSeconds} s statement timeout") : string.Empty, + ex.Message); + } + /// /// One maintenance tick: remove the expired hours, plan, then build each hour in its own transaction. A build that fails is - /// logged (naming the hour) and the loop goes on; the next tick plans it again. Once has passed + /// logged (naming the hour), recorded (, #5867) and the loop goes on: the next tick retries it, under + /// after a statement timeout, until it has failed + /// times in a row (or timed out under the extended timeout), and once a day after that. Once has passed /// no further build starts, and the planned hours left are reported as deferred. /// public static Task RunTickAsync( @@ -860,7 +1095,7 @@ public static async Task RunTickAsync( ArgumentNullException.ThrowIfNull(logger); ArgumentNullException.ThrowIfNull(elapsed); - int builtStale = 0, builtMissing = 0, failed = 0, started = 0, deferredForWriters = 0; + int builtStale = 0, builtMissing = 0, failed = 0, started = 0, deferredForWriters = 0, backedOff = 0; var watch = options?.Watch ?? DeferralWatch.Shared; await using var connection = await dataSource.OpenConnectionAsync(cancellationToken); @@ -876,9 +1111,10 @@ public static async Task RunTickAsync( } started++; + var timeoutSeconds = item.Extended ? ExtendedBuildStatementTimeoutSeconds : BuildStatementTimeoutSeconds; try { - if (await BuildHourAsync(connection, item.Hour, nowUtc, cancellationToken) is null) + if (await BuildHourAsync(connection, item.Hour, nowUtc, cancellationToken, statementTimeoutSeconds: timeoutSeconds) is null) { deferredForWriters++; continue; @@ -896,8 +1132,23 @@ public static async Task RunTickAsync( catch (Exception ex) when (ex is not OperationCanceledException) { failed++; - logger.LogWarning(ex, "Query Store compose rollup: building hour {Hour:yyyy-MM-dd HH}:00 (stale {Stale}) failed and is retried next tick: {Message}", - item.Hour, item.Stale, ex.Message); + var sqlState = (ex as PostgresException)?.SqlState; + FailureOutcome? outcome = null; + try + { + outcome = await RecordFailureAsync(connection, item.Hour, sqlState, ex.Message, timeoutSeconds, options?.StoreNow, cancellationToken); + } + catch (Exception recordEx) when (recordEx is not OperationCanceledException) + { + /* A broken connection fails the record too; the hour has no new record and is planned again next tick, as before #5867. */ + logger.LogDebug("Query Store compose rollup: recording the failure of hour {Hour:yyyy-MM-dd HH}:00 failed: {Message}", item.Hour, recordEx.Message); + } + + LogFailure(logger, ex, item, sqlState, outcome); + if (outcome is { BackedOff: true }) + { + backedOff++; + } } } @@ -934,7 +1185,7 @@ public static async Task RunTickAsync( } var deferred = plan.Count - started + deferredForWriters; - var result = new TickResult(builtStale, builtMissing, failed, removed, deferred, deferredForWriters, planned.HeldBack); + var result = new TickResult(builtStale, builtMissing, failed, removed, deferred, deferredForWriters, planned.HeldBack, backedOff); logger.LogInformation("Query Store compose rollup: built {Built} hour(s) (stale {Stale}, missing {Missing}), failed {Failed}, deferred {Deferred}, removed {Removed} hour(s), {ElapsedMs} ms", result.Built, builtStale, builtMissing, failed, deferred, removed, (long)elapsed().TotalMilliseconds); return result; diff --git a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs index 2c990e3293..84f93edb07 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/StorageVersion.cs @@ -16,5 +16,5 @@ namespace PerformanceMonitor.Darling.Storage; /// public static class StorageVersion { - public const int SchemaVersion = 188; + public const int SchemaVersion = 189; } diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs index fe8d14c597..bcdc7f1fe1 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.cs @@ -1227,7 +1227,11 @@ per the V71 finding. */ EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.config_alert_log') AND attname = 'resolution_outcome' AND NOT attisdropped), /* V188 (#5821) adds the alert's values to the investigation link rows. Found by catalog so the answer does not depend on the connecting role's privileges. Named only in this probe line, never in prose, per the V71 finding. */ - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.investigation_request_alerts') AND attname = 'context_json' AND NOT attisdropped)"; + EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('config.investigation_request_alerts') AND attname = 'context_json' AND NOT attisdropped), + /* V189 (#5867) adds the failure record of the Query Store compose builder. That table is the sentinel, found by name in the + catalog so the answer does not depend on the connecting role's privileges on it. Named only in this probe line, never in + prose, per the V71 finding. */ + EXISTS (SELECT 1 WHERE to_regclass('collect.query_store_compose_stamp_failures') IS NOT NULL)"; /// The store schema version this viewer build requires — the highest migration it knows /// (). The connect-time gate blocks a store below this. @@ -1249,7 +1253,7 @@ per the V71 finding. */ await using var reader = await command.ExecuteReaderAsync(cancellationToken); if (await reader.ReadAsync(cancellationToken)) { - return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163)); + return MapProbedSchemaVersion(reader.GetBoolean(0), reader.GetBoolean(1), reader.GetBoolean(2), reader.GetBoolean(3), reader.GetBoolean(4), reader.GetBoolean(5), reader.GetBoolean(6), reader.GetBoolean(7), reader.GetBoolean(8), reader.GetBoolean(9), reader.GetBoolean(10), reader.GetBoolean(11), reader.GetBoolean(12), reader.GetBoolean(13), reader.GetBoolean(14), reader.GetBoolean(15), reader.GetBoolean(16), reader.GetBoolean(17), reader.GetBoolean(18), reader.GetBoolean(19), reader.GetBoolean(20), reader.GetBoolean(21), reader.GetBoolean(22), reader.GetBoolean(23), reader.GetBoolean(24), reader.GetBoolean(25), reader.GetBoolean(26), reader.GetBoolean(27), reader.GetBoolean(28), reader.GetBoolean(29), reader.GetBoolean(30), reader.GetBoolean(31), reader.GetBoolean(32), reader.GetBoolean(33), reader.GetBoolean(34), reader.GetBoolean(35), reader.GetBoolean(36), reader.GetBoolean(37), reader.GetBoolean(38), reader.GetBoolean(39), reader.GetBoolean(40), reader.GetBoolean(41), reader.GetBoolean(42), reader.GetBoolean(43), reader.GetBoolean(44), reader.GetBoolean(45), reader.GetBoolean(46), reader.GetBoolean(47), reader.GetBoolean(48), reader.GetBoolean(49), reader.GetBoolean(50), reader.GetBoolean(51), reader.GetBoolean(52), reader.GetBoolean(53), reader.GetBoolean(54), reader.GetBoolean(55), reader.GetBoolean(56), reader.GetBoolean(57), reader.GetBoolean(58), reader.GetBoolean(59), reader.GetBoolean(60), reader.GetBoolean(61), reader.GetBoolean(62), reader.GetBoolean(63), reader.GetBoolean(64), reader.GetBoolean(65), reader.GetBoolean(66), reader.GetBoolean(67), reader.GetBoolean(68), reader.GetBoolean(69), reader.GetBoolean(70), reader.GetBoolean(71), reader.GetBoolean(72), reader.GetBoolean(73), reader.GetBoolean(74), reader.GetBoolean(75), reader.GetBoolean(76), reader.GetBoolean(77), reader.GetBoolean(78), reader.GetBoolean(79), reader.GetBoolean(80), reader.GetBoolean(81), reader.GetBoolean(82), reader.GetBoolean(83), reader.GetBoolean(84), reader.GetBoolean(85), reader.GetBoolean(86), reader.GetBoolean(87), reader.GetBoolean(88), reader.GetBoolean(89), reader.GetBoolean(90), reader.GetBoolean(91), reader.GetBoolean(92), reader.GetBoolean(93), reader.GetBoolean(94), reader.GetBoolean(95), reader.GetBoolean(96), reader.GetBoolean(97), reader.GetBoolean(98), reader.GetBoolean(99), reader.GetBoolean(100), reader.GetBoolean(101), reader.GetBoolean(102), reader.GetBoolean(103), reader.GetBoolean(104), reader.GetBoolean(105), reader.GetBoolean(106), reader.GetBoolean(107), reader.GetBoolean(108), reader.GetBoolean(109), reader.GetBoolean(110), reader.GetBoolean(111), reader.GetBoolean(112), reader.GetBoolean(113), reader.GetBoolean(114), reader.GetBoolean(115), reader.GetBoolean(116), reader.GetBoolean(117), reader.GetBoolean(118), reader.GetBoolean(119), reader.GetBoolean(120), reader.GetBoolean(121), reader.GetBoolean(122), reader.GetBoolean(123), reader.GetBoolean(124), reader.GetBoolean(125), reader.GetBoolean(126), reader.GetBoolean(127), reader.GetBoolean(128), reader.GetBoolean(129), reader.GetBoolean(130), reader.GetBoolean(131), reader.GetBoolean(132), reader.GetBoolean(133), reader.GetBoolean(134), reader.GetBoolean(135), reader.GetBoolean(136), reader.GetBoolean(137), reader.GetBoolean(138), reader.GetBoolean(139), reader.GetBoolean(140), reader.GetBoolean(141), reader.GetBoolean(142), reader.GetBoolean(143), reader.GetBoolean(144), reader.GetBoolean(145), reader.GetBoolean(146), reader.GetBoolean(147), reader.GetBoolean(148), reader.GetBoolean(149), reader.GetBoolean(150), reader.GetBoolean(151), reader.GetBoolean(152), reader.GetBoolean(153), reader.GetBoolean(154), reader.GetBoolean(155), reader.GetBoolean(156), reader.GetBoolean(157), reader.GetBoolean(158), reader.GetBoolean(159), reader.GetBoolean(160), reader.GetBoolean(161), reader.GetBoolean(162), reader.GetBoolean(163), reader.GetBoolean(164)); } return null; @@ -1274,7 +1278,7 @@ per the V71 finding. */ /// is unit-tested without a live store; any schema bump past the newest arm trips the pinning test that keeps /// this in step with . /// - internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false) + internal static int MapProbedSchemaVersion(bool hasConfigControlPlane, bool hasAlertDeliveryOverride, bool hasAnalysisState, bool hasAlertTuningKnobs, bool hasDefaultTraceEvents, bool hasIndexObjectStatsLatestIndex, bool hasCollectionLogHypertableOrPlainPg, bool hasJobHistory, bool hasAgentStatus, bool hasGenericWebhook, bool hasDeadlocksDatabaseName, bool hasQueryStoreReplicaRole, bool hasLongQueryCompletions, bool hasWebDashboardConfig, bool hasCustomViews, bool hasServerTags, bool hasConnectionRefireKnobs = false, bool hasAgCollectors = false, bool hasAgAlertKnobs = false, bool hasAgLatencyColumns = false, bool hasAgDisconnectRefire = false, bool hasPayloadDimensions = false, bool hasDimFloorIndexes = false, bool hasBlockingWaitThreshold = false, bool hasQueryStoreIntervalIdentity = false, bool hasPagerDutyWebhook = false, bool hasPagerDutyProxy = false, bool hasCollectorState = false, bool hasPlanCorrection = false, bool hasPvsStats = false, bool hasPvsPressureKnobs = false, bool hasDatabaseStateAlert = false, bool hasServerTagColour = false, bool hasQueryStatsHostObject = false, bool hasFindingDrillDown = false, bool hasStoreMetrics = false, bool hasPlanDimGzip = false, bool hasSelfAlertKnobs = false, bool hasJobMetricsColumns = false, bool hasJobCadenceKnob = false, bool hasBackfillSwitch = false, bool hasCollectorMemoryKnobs = false, bool hasDatabaseStateEdgeMemory = false, bool hasIncidentOccurrences = false, bool hasPlanXmlCompressionKnob = false, bool hasMonitoredServerEngine = false, bool hasPgBlockingEdges = false, bool hasQueryStorePlanMap = false, bool hasPgStatementText = false, bool hasQueryStoreText = false, bool hasPlanContentRetentionKnob = false, bool hasQueryStoreHealth = false, bool hasQueryStoreTextHash = false, bool hasComposeTimeoutKnob = false, bool hasFileGrowthAlert = false, bool hasCollectionLogFanoutRollup = false, bool hasTempDbMaxSize = false, bool hasServerEngineKind = false, bool hasPgDatabaseStats = false, bool hasPgIndexUsageStats = false, bool hasPgTableBloatStats = false, bool hasPgSessionStates = false, bool hasPgPlanCaptureReadiness = false, bool hasPgWriteStats = false, bool hasPgExtensionAvailability = false, bool hasPgLockStats = false, bool hasPgColumnStats = false, bool hasPgReplicationStats = false, bool hasPgBufferUsage = false, bool hasPgIndexBloat = false, bool hasPgPerDatabaseAttribution = false, bool hasPgWaitSampling = false, bool hasPgKernelStats = false, bool hasPgPredicateStats = false, bool hasPgPlanCapture = false, bool hasPgMajorVersion = false, bool hasPg18IoBytes = false, bool hasPgServerConfig = false, bool hasPgDeadlocks = false, bool hasPgDeadlockIdentity = false, bool hasCollectorCost = false, bool hasPgCpuUtilization = false, bool hasPlanForceActions = false, bool hasCollectionLogPhaseSplit = false, bool hasCollectionLogDrainForensics = false, bool hasCollectionLogFetchPhaseSums = false, bool hasStoreLogSelfMonitoring = false, bool hasCollectorStallProbes = false, bool hasRemediationCredentialAndActor = false, bool hasPgIndexBloatEstimate = false, bool hasPgCpuCapacityHeadroom = false, bool hasCustomAlertCore = false, bool hasMuteRuleReloadBeacon = false, bool hasBuiltinAlertPersistence = false, bool hasRetentionHoldRatioKnobs = false, bool hasDeadlockRateBandKnobs = false, bool hasOversizedPlanBacklog = false, bool hasPgAlertCountKnobs = false, bool hasFleetSweepState = false, bool hasFleetSweepCadenceKnobs = false, bool hasCollectorScheduleDatabases = false, bool hasSelfDiskWarnGbFloor = false, bool hasDeltaFamilyIntervalColumns = false, bool hasDeltaFamilyIntervalCompletion = false, bool hasPgLogEvents = false, bool hasPgLogEventMetrics = false, bool hasNotificationRoutes = false, bool hasPerfmonCounterType = false, bool hasPgNumbackendsAndSampledMs = false, bool hasTimeHonesty = false, bool hasLrqExclusionKnob = false, bool hasPgDatabaseSizeStatsAndHostMemory = false, bool hasQsCaptureModeRouteKnobToast = false, bool hasPgServerConfigDatabaseRoleOverrides = false, bool hasPostmasterStartTime = false, bool hasCheckpointsTimed = false, bool hasCollectionCaveats = false, bool hasIndexObjectStatsServerTimeIndex = false, bool hasQueryStoreIntervalLatest = false, bool hasRawChunkIntervalRungHistory = false, bool hasQueryStoreIntervalWide = false, bool hasManagedConfVerdicts = false, bool hasComposeTimeoutSixty = false, bool hasReadLatency = false, bool hasHotLivenessTouch = false, bool hasCollectionLogWatermarkAndJobHistoryIndexes = false, bool hasAgGroupId = false, bool hasCaggGroupIndexDrop = false, bool hasIntervalFirstExecIndexes = false, bool hasIntervalWideFirstExecIndex = false, bool hasQueryStoreIntervalEnd = false, bool hasCheckpointLongestSync = false, bool hasMuteRuleServerId = false, bool hasInstallId = false, bool hasInstallIdTableOid = false, bool hasCollectorRunAt = false, bool hasQueryStoreTopDaily = false, bool hasSlowReads = false, bool hasStoreStatementHistory = false, bool hasQueryStatsHourLedger = false, bool hasPasswordKey = false, bool hasPagerDutyAutoResolve = false, bool hasLegacyPinCandidates = false, bool hasPlanRegressionDaily = false, bool hasAwsRole = false, bool hasPgIoStatsHourly = false, bool hasQueryStoreIntervalWidePartitioned = false, bool hasQueryStoreIntervalLatestPartitioned = false, bool hasQueryStoreComposeStamp = false, bool hasServiceVersion = false, bool hasPlanRegressionDailyRowcount = false, bool hasProcedureStatsTotals = false, bool hasRunnableQueueSettings = false, bool hasQueryStoreWatermarks = false, bool hasClientTimeoutSettings = false, bool hasExternalAlerts = false, bool hasNotificationRouteScope = false, bool hasReadLatencyCapacityFields = false, bool hasReadCaller = false, bool hasInvestigations = false, bool hasPlanRegressionPartialDays = false, bool hasCollectionLogHourly = false, bool hasAlertResolutionDelivery = false, bool hasInvestigationAlertFacts = false, bool hasComposeStampFailures = false) { /* V71 (the PostgreSQL blocking-edges rung): a table-existence sentinel and now the newest-first arm. A collector table would ordinarily get no arm at all — see the V63-V69 note below — but the TOP @@ -1474,6 +1478,14 @@ the V71 finding. */ newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 181 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the V71 finding. */ + /* V189 (#5867): the failure record of the Query Store compose builder. Table-existence sentinel, newest-first, and the TOP + rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 188 and + showing a spurious upgrade banner, or the connect-time gate refusing a store that is current. Named only in the probe + line, not this prose, per the V71 finding. */ + if (hasComposeStampFailures) + { + return 189; + } /* V188 (#5821): the alert values on the investigation link rows. Column-existence sentinel, newest-first, and the TOP rung, so a fully-migrated store maps to EXACTLY StorageVersion.SchemaVersion rather than falling through to 186 and showing a spurious upgrade banner on a store that is current. Named only in the probe line, not this prose, per the diff --git a/PerformanceMonitor.Notifications/AlertFamily.cs b/PerformanceMonitor.Notifications/AlertFamily.cs index 514ba010f0..d35d4165af 100644 --- a/PerformanceMonitor.Notifications/AlertFamily.cs +++ b/PerformanceMonitor.Notifications/AlertFamily.cs @@ -156,6 +156,8 @@ one never covers the other. */ ["Collector Cost Regression"] = SelfMonitor, /* #4299: the raw-retention over-horizon self-alert. */ ["Raw Purge Over Horizon"] = SelfMonitor, + /* #5867: hours of the Query Store compose rollup whose build is retried only once a day. */ + ["Compose Rollup Hours Stuck"] = SelfMonitor, /* #4750: a webhook channel that has failed three times in a row. */ ["Notification Channel Failing"] = SelfMonitor, /* #3783: the store's own TOAST slack and checkpointer pressure — informational conditions about the