diff --git a/Darling/Darling.Tests/InvestigationToolContract.cs b/Darling/Darling.Tests/InvestigationToolContract.cs index 7f51fc84b5..1ed0d8e36e 100644 --- a/Darling/Darling.Tests/InvestigationToolContract.cs +++ b/Darling/Darling.Tests/InvestigationToolContract.cs @@ -72,6 +72,7 @@ private static ToolParameter P(string name, string type, string defaultValue, st P("links", "string?", "null", "Optional JSON array of at most 10 {kind, url, key}; https only. See the guide."), P("family_of", "string?", "null", "Optional. The request_id of the first report in this family; it must exist."), P("queries_run", "string?", "null", "Optional JSON array of statements you ran, " + InvestigationReportFields.QueriesRunRule + "."), + P("follow_up_at", "string?", "null", "Optional. When to open a follow-up request: " + InvestigationReportFields.FollowUpRule + "."), }), ["get_investigation_reports"] = new( new[] { "ok", "empty" }, @@ -84,16 +85,15 @@ private static ToolParameter P(string name, string type, string defaultValue, st }; /// - /// The parameters only Darling's tools take, after the shared ones: Darling's submit takes follow_up_at and - /// follow_up_condition (#6008, #6009); Lite's does not (#6012). links, family_of and queries_run - /// (#5914, #6004) and the basis on evidence are shared since #6011. Darling's test passes this table to + /// The parameters only Darling's tools take, after the shared ones: Darling's submit takes follow_up_condition + /// (#6009); Lite's does not (Lite opens a time follow-up only, #6012). links, family_of and queries_run + /// (#5914, #6004) and the basis on evidence are shared since #6011, and follow_up_at (#6008) since #6012. Darling's test passes this table to /// ; Lite's passes none. /// internal static readonly IReadOnlyDictionary DarlingOnlyParameters = new Dictionary(StringComparer.Ordinal) { ["submit_investigation_report"] = new[] { - P("follow_up_at", "string?", "null", "Optional. When to open a follow-up request: " + InvestigationReportFields.FollowUpRule + "."), P("follow_up_condition", "string?", "null", "Optional. Open the follow-up when a module is deployed: " + InvestigationReportFields.FollowUpConditionRule + "."), }, }; diff --git a/Lite.Tests/LiteInvestigationEvidenceTests.cs b/Lite.Tests/LiteInvestigationEvidenceTests.cs index 442d46f236..858613eeea 100644 --- a/Lite.Tests/LiteInvestigationEvidenceTests.cs +++ b/Lite.Tests/LiteInvestigationEvidenceTests.cs @@ -328,12 +328,12 @@ string Shape(DuckDbInitializer db) => var first = Shape(upgraded); Assert.Contains("investigation_evidence.section", first); Assert.Contains("investigation_requests.evidence_state", first); - Assert.Equal("74", Scalar2(upgraded, "SELECT max(version) FROM schema_version")); + Assert.Equal("75", Scalar2(upgraded, "SELECT max(version) FROM schema_version")); using var again = new DuckDbInitializer(path); await again.InitializeAsync(); Assert.Equal(first, Shape(again)); - Assert.Equal("74", Scalar2(again, "SELECT max(version) FROM schema_version")); + Assert.Equal("75", Scalar2(again, "SELECT max(version) FROM schema_version")); } private static void Exec2(DuckDbInitializer db, string sql) diff --git a/Lite.Tests/LiteInvestigationFollowUpTests.cs b/Lite.Tests/LiteInvestigationFollowUpTests.cs new file mode 100644 index 0000000000..d869755ce9 --- /dev/null +++ b/Lite.Tests/LiteInvestigationFollowUpTests.cs @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Common; +using PerformanceMonitorLite.Database; +using PerformanceMonitorLite.Investigations; +using PerformanceMonitorLite.Mcp; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// #6012 part A, Lite's twins of Darling's #6008 pins over DuckDB: a report's follow_up_at opens one follow-up request +/// when its time comes (never before, never twice), whose brief carries the earlier report; a route holding 20 pending +/// follow-ups refuses the 21st and stores nothing of that report; a time past 30 days is refused through the tool; and the +/// v75 step adds the two columns to a new and an existing database. Lite has no event delivery, so the follow-up is found in the +/// request list. +/// +public sealed class LiteInvestigationFollowUpTests : IDisposable +{ + private const string Evidence = "[{\"tool\":\"get_blocking\",\"arguments\":{},\"key_numbers\":\"head blocker 412 s\",\"basis\":\"verified\"}]"; + private const string RootCause = "a nightly index rebuild held the lock follow-up-canary"; + private static readonly DateTime T0 = new(2026, 10, 14, 12, 0, 0, DateTimeKind.Utc); + + private readonly string _tempDir; + private readonly DuckDbInitializer _db; + private readonly DuckDbInvestigationStore _store; + private DateTime _now = T0; + + public LiteInvestigationFollowUpTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), "LiteInvestigationFollowUpTests_" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(_tempDir); + _db = new DuckDbInitializer(Path.Combine(_tempDir, "lite.duckdb")); + _db.InitializeAsync().GetAwaiter().GetResult(); + _store = new DuckDbInvestigationStore(_db, () => _now, DuckDbInvestigationStoreTests.TestLockBudget); + } + + public void Dispose() + { + _db.Dispose(); + try { if (Directory.Exists(_tempDir)) Directory.Delete(_tempDir, recursive: true); } + catch { /* best effort */ } + } + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private InvestigationAlert Alert(int serverId = 7, int routeId = 1) => + new(routeId, serverId, "srv-" + serverId.ToString(CultureInfo.InvariantCulture), "blocking", "Blocking", _now, + new[] { "key-" + serverId.ToString(CultureInfo.InvariantCulture) }, null, 100, 1000, LeaseMinutes: 15, ExpiryMinutes: 240); + + private InvestigationBriefComposer Composer() => new( + new DuckDbInvestigationBriefSourceReader(_db), new McpDataDetailSetting(McpDataDetail.Full), LiteInvestigationBriefTools.Tools); + + private static JsonElement Root(string json) => JsonDocument.Parse(json).RootElement; + + private string? Scalar(string sql, DuckDbInitializer? db = null) + { + using var connection = (db ?? _db).CreateConnection(); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = sql; + var value = command.ExecuteScalar(); + return value is null or DBNull ? null : Convert.ToString(value, CultureInfo.InvariantCulture); + } + + private static void Exec(string sql, DuckDbInitializer db) + { + using var connection = db.CreateConnection(); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + /// Opens a request for a server on a route, claims it and reports it through the tool with . + private async Task<(Guid Id, JsonElement Answer)> ReportAsync(int serverId, int routeId, string? followUpAt) + { + var id = (await _store.OpenOrJoinAsync(Alert(serverId, routeId), Ct)).RequestId!.Value; + var claim = Root(await McpInvestigationTools.ClaimAsync(_store, Composer(), id.ToString(), "agent-a", null, null, Ct)); + var answer = Root(await McpInvestigationTools.SubmitAsync(_store, id.ToString(), claim.GetProperty("claim_id").GetString()!, 1, + "real", RootCause, Evidence, "summary", null, null, Ct, follow_up_at: followUpAt)); + return (id, answer); + } + + private static string Iso(DateTime value) => value.ToString("yyyy-MM-dd'T'HH:mm:ss'Z'", CultureInfo.InvariantCulture); + + [Fact] + public async Task AFollowUp_OpensOnceAtItsTime_NamesTheEarlierRequest_AndItsBriefCarriesTheEarlierReport() + { + _now = DateTime.UtcNow; + var due = _now.AddHours(2); + var (earlier, answer) = await ReportAsync(7, 1, Iso(due)); + Assert.Equal("reported", answer.GetProperty("status").GetString()); + + /* Before the time: nothing opens. */ + _now = due.AddMinutes(-1); + Assert.Equal(0, (await _store.SweepAsync(Ct)).FollowUps); + Assert.Equal("1", Scalar("SELECT count(*) FROM investigation_requests")); + + /* At the time: one request opens, naming the earlier one, with one follow_up journal row of trigger time. */ + _now = due.AddMinutes(1); + Assert.Equal(1, (await _store.SweepAsync(Ct)).FollowUps); + var followUp = Scalar($"SELECT request_id FROM investigation_requests WHERE follow_up_of = '{earlier}'"); + Assert.NotNull(followUp); + Assert.Equal("open", Scalar($"SELECT state FROM investigation_requests WHERE request_id = '{followUp}'")); + Assert.Equal("1", Scalar($"SELECT count(*) FROM investigation_journal WHERE request_id = '{followUp}' AND event = 'follow_up' AND detail LIKE '%\"trigger\":\"time\"%' AND detail LIKE '%{earlier}%'")); + + /* A second sweep opens no second one. */ + _now = due.AddMinutes(5); + Assert.Equal(0, (await _store.SweepAsync(Ct)).FollowUps); + Assert.Equal("2", Scalar("SELECT count(*) FROM investigation_requests")); + + /* Its brief carries the earlier report, through the same read as any earlier report of the server and family. */ + var brief = await Composer().ComposeAsync(Guid.Parse(followUp!), Ct); + Assert.NotNull(brief); + Assert.Contains("follow-up-canary", brief, StringComparison.Ordinal); + } + + [Fact] + public async Task ARouteHolding20PendingFollowUps_RefusesThe21st_AndStoresNothingFromThatReport() + { + var future = Iso(DateTime.UtcNow.AddDays(3)); + _now = DateTime.UtcNow; + for (var server = 1; server <= InvestigationReportFields.MaxPendingFollowUpsPerRoute; server++) + { + var (_, ok) = await ReportAsync(server, 1, future); + Assert.Equal("reported", ok.GetProperty("status").GetString()); + } + + var (id, refused) = await ReportAsync(99, 1, future); + Assert.Equal("refused", refused.GetProperty("status").GetString()); + Assert.Equal("follow_up_at", refused.GetProperty("hints").GetProperty("parameter").GetString()); + Assert.Contains("20 pending follow-ups", refused.GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.Equal("0", Scalar($"SELECT count(*) FROM investigation_reports WHERE request_id = '{id}'")); + Assert.Equal("claimed", Scalar($"SELECT state FROM investigation_requests WHERE request_id = '{id}'")); + Assert.Null(Scalar($"SELECT follow_up_at FROM investigation_requests WHERE request_id = '{id}'")); + + /* Another route has its own count, and the refused report is accepted without a time. */ + var (_, otherRoute) = await ReportAsync(98, 2, future); + Assert.Equal("reported", otherRoute.GetProperty("status").GetString()); + } + + [Fact] + public async Task AFollowUpTime31DaysAhead_IsRefusedThroughTheTool_AndNothingIsStored() + { + _now = DateTime.UtcNow; + var (id, answer) = await ReportAsync(7, 1, Iso(DateTime.UtcNow.AddDays(31))); + + Assert.Equal("invalid", answer.GetProperty("status").GetString()); + Assert.Contains("follow_up_at", answer.GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.Equal("0", Scalar($"SELECT count(*) FROM investigation_reports WHERE request_id = '{id}'")); + + var (_, past) = await ReportAsync(8, 1, Iso(DateTime.UtcNow.AddHours(-1))); + Assert.Equal("invalid", past.GetProperty("status").GetString()); + } + + [Fact] + public async Task APurge_ClearsAFollowUpOfThatNamesARequestItDeletes() + { + var gone = Guid.NewGuid(); + var kept = Guid.NewGuid(); + var oldTime = _now.AddDays(-400); + foreach (var (id, followUpOf, opened, state) in new (Guid, Guid?, DateTime, string)[] + { (gone, null, oldTime, "reported"), (kept, gone, _now.AddDays(-1), "reported") }) + { + Exec($@"INSERT INTO investigation_requests + (request_id, route_id, server_id, server_name, family, first_metric, first_alert_at, last_alert_at, window_until, state, + opened_at, expires_at, follow_up_of) +VALUES ('{id}', 1, 7, 'srv-7', 'blocking', 'Blocking', TIMESTAMP '{opened:yyyy-MM-dd HH:mm:ss}', TIMESTAMP '{opened:yyyy-MM-dd HH:mm:ss}', + TIMESTAMP '{opened:yyyy-MM-dd HH:mm:ss}', '{state}', TIMESTAMP '{opened:yyyy-MM-dd HH:mm:ss}', TIMESTAMP '{opened.AddDays(1):yyyy-MM-dd HH:mm:ss}', + {(followUpOf is null ? "NULL" : "'" + followUpOf + "'")})", _db); + } + + Assert.Equal(1, await _store.PurgeAsync(Ct)); + Assert.Equal("1", Scalar("SELECT count(*) FROM investigation_requests")); + Assert.Null(Scalar($"SELECT follow_up_of FROM investigation_requests WHERE request_id = '{kept}'")); + } + + [Fact] + public async Task SchemaStepV75_AppliesToANewAndAnExistingDatabase_AndASecondRunChangesNothing() + { + const string ColumnsSql = "SELECT string_agg(column_name, ',' ORDER BY column_name) FROM information_schema.columns WHERE table_name = 'investigation_requests' AND column_name IN ('follow_up_of', 'follow_up_at')"; + Assert.Equal("follow_up_at,follow_up_of", Scalar(ColumnsSql)); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version")); + + var path = Path.Combine(_tempDir, "old.duckdb"); + using (var old = new DuckDbInitializer(path)) + { + await old.InitializeAsync(); + Exec("ALTER TABLE investigation_requests DROP COLUMN follow_up_of", old); + Exec("ALTER TABLE investigation_requests DROP COLUMN follow_up_at", old); + Exec("UPDATE schema_version SET version = 74", old); + } + + using var upgraded = new DuckDbInitializer(path); + await upgraded.InitializeAsync(); + Assert.Equal("follow_up_at,follow_up_of", Scalar(ColumnsSql, upgraded)); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version", upgraded)); + + using var again = new DuckDbInitializer(path); + await again.InitializeAsync(); + Assert.Equal("follow_up_at,follow_up_of", Scalar(ColumnsSql, again)); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version", again)); + } +} diff --git a/Lite.Tests/LiteInvestigationGoldenTests.cs b/Lite.Tests/LiteInvestigationGoldenTests.cs index abf2574cf3..ad19a63442 100644 --- a/Lite.Tests/LiteInvestigationGoldenTests.cs +++ b/Lite.Tests/LiteInvestigationGoldenTests.cs @@ -86,6 +86,6 @@ public void LitesLevel1ToolTexts_AreUnchanged() text.Append(InvestigationFormats.UnsupportedReportFormatMessage(2, "Lite")); var all = text.ToString(); - Assert.True(Sha(all) == "061953075B45E959B8FE504B1E03D07FC2E122E02C7A839713FC7E542B4448BA", $"Lite's Level 1 tool texts moved; their SHA-256 is {Sha(all)}:\n{all}"); + Assert.True(Sha(all) == "62A75C554816D5CB1A61C4F8449774A13D4B0C0747AF487E44A689CCA8F2FC37", $"Lite's Level 1 tool texts moved; their SHA-256 is {Sha(all)}:\n{all}"); } } diff --git a/Lite.Tests/LiteInvestigationL2Tests.cs b/Lite.Tests/LiteInvestigationL2Tests.cs index f759e83c35..7469acdf79 100644 --- a/Lite.Tests/LiteInvestigationL2Tests.cs +++ b/Lite.Tests/LiteInvestigationL2Tests.cs @@ -321,7 +321,7 @@ public async Task SchemaStepV74_AppliesToANewAndAnExistingDatabase_AndASecondRun const string ColumnsSql = "SELECT string_agg(column_name, ',' ORDER BY column_name) FROM information_schema.columns " + "WHERE table_name = 'investigation_reports' AND column_name IN ('family_of', 'queries_run')"; Assert.Equal("family_of,queries_run", Scalar(ColumnsSql)); - Assert.Equal("74", Scalar("SELECT max(version) FROM schema_version")); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version")); var path = Path.Combine(_tempDir, "old.duckdb"); using (var old = new DuckDbInitializer(path)) @@ -335,7 +335,7 @@ public async Task SchemaStepV74_AppliesToANewAndAnExistingDatabase_AndASecondRun using var upgraded = new DuckDbInitializer(path); await upgraded.InitializeAsync(); Assert.Equal("family_of,queries_run", Scalar(ColumnsSql, upgraded)); - Assert.Equal("74", Scalar("SELECT max(version) FROM schema_version", upgraded)); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version", upgraded)); const string ShapeSql = "SELECT string_agg(column_name || ':' || data_type, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_name = 'investigation_reports'"; var shape = Scalar(ShapeSql, upgraded); Assert.Contains("family_of:UUID", shape, StringComparison.Ordinal); @@ -343,6 +343,6 @@ public async Task SchemaStepV74_AppliesToANewAndAnExistingDatabase_AndASecondRun using var again = new DuckDbInitializer(path); await again.InitializeAsync(); Assert.Equal(shape, Scalar(ShapeSql, again)); - Assert.Equal("74", Scalar("SELECT max(version) FROM schema_version", again)); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version", again)); } } diff --git a/Lite.Tests/LiteInvestigationRequestEventsTests.cs b/Lite.Tests/LiteInvestigationRequestEventsTests.cs index 7113c51d0d..98c34d2f89 100644 --- a/Lite.Tests/LiteInvestigationRequestEventsTests.cs +++ b/Lite.Tests/LiteInvestigationRequestEventsTests.cs @@ -155,7 +155,7 @@ public async Task SchemaStepV73_AppliesToANewAndAnExistingDatabase_AndASecondRun { const string ColumnSql = "SELECT count(*) FROM information_schema.columns WHERE table_name = 'investigation_requests' AND column_name = 'max_severity'"; Assert.Equal("1", Scalar(ColumnSql)); - Assert.Equal("74", Scalar("SELECT max(version) FROM schema_version")); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version")); var path = Path.Combine(_tempDir, "old.duckdb"); using (var old = new DuckDbInitializer(path)) @@ -168,12 +168,12 @@ public async Task SchemaStepV73_AppliesToANewAndAnExistingDatabase_AndASecondRun using var upgraded = new DuckDbInitializer(path); await upgraded.InitializeAsync(); Assert.Equal("1", Scalar(ColumnSql, upgraded)); - Assert.Equal("74", Scalar("SELECT max(version) FROM schema_version", upgraded)); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version", upgraded)); var shape = Scalar("SELECT string_agg(column_name, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_name = 'investigation_requests'", upgraded); using var again = new DuckDbInitializer(path); await again.InitializeAsync(); Assert.Equal(shape, Scalar("SELECT string_agg(column_name, ',' ORDER BY ordinal_position) FROM information_schema.columns WHERE table_name = 'investigation_requests'", again)); - Assert.Equal("74", Scalar("SELECT max(version) FROM schema_version", again)); + Assert.Equal("75", Scalar("SELECT max(version) FROM schema_version", again)); } } diff --git a/Lite.Tests/LiteSignedWebhookTests.cs b/Lite.Tests/LiteSignedWebhookTests.cs new file mode 100644 index 0000000000..89e809b855 --- /dev/null +++ b/Lite.Tests/LiteSignedWebhookTests.cs @@ -0,0 +1,548 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using PerformanceMonitor.Notifications; +using PerformanceMonitorLite.Database; +using PerformanceMonitorLite.Services; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// #6012 part B, Lite's twins of Darling's #5998 pins. Lite is the sender, so a generic webhook to a destination with a key +/// carries the three Standard Webhooks headers over the exact bytes sent, and a receiver that recomputes the HMAC accepts it +/// (and refuses one changed byte); during a rotation both keys verify. The key is kept in the secret store (an in-memory +/// pair here, never the user's Credential Manager) and in no table, settings text or status line; and a saved key that will +/// not open sends nothing to that destination. In the app-alert-statics collection: the settings row's destination is +/// the App statics, and the route cache is static. +/// +[Collection("app-alert-statics")] +public sealed class LiteSignedWebhookTests : IDisposable +{ + private readonly string _tempDir; + private readonly DuckDbInitializer _db; + private readonly FakeRouteSecrets _secrets = new(); + private readonly DuckDbNotificationRouteStore _store; + private readonly bool _enabled = App.GenericWebhookEnabled; + private readonly string _url = App.GenericWebhookUrl; + private readonly string _headers = App.GenericWebhookHeadersJson; + private readonly bool _signingSet = App.GenericSigningSet; + private readonly DateTime? _signingUntil = App.GenericSigningUntilUtc; + + public LiteSignedWebhookTests() + { + _tempDir = Path.Combine(Path.GetTempPath(), "LiteSignedWebhookTests_" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(_tempDir); + _db = new DuckDbInitializer(Path.Combine(_tempDir, "lite.duckdb")); + _db.InitializeAsync().GetAwaiter().GetResult(); + _store = new DuckDbNotificationRouteStore(_db, () => new[] { 7101 }, _secrets.Access); + App.GenericWebhookEnabled = false; + App.GenericWebhookUrl = ""; + App.GenericWebhookHeadersJson = ""; + App.GenericSigningSet = false; + App.GenericSigningUntilUtc = null; + } + + public void Dispose() + { + NotificationRouteCache.Configure(null); + App.GenericWebhookEnabled = _enabled; + App.GenericWebhookUrl = _url; + App.GenericWebhookHeadersJson = _headers; + App.GenericSigningSet = _signingSet; + App.GenericSigningUntilUtc = _signingUntil; + _db.Dispose(); + try { if (Directory.Exists(_tempDir)) Directory.Delete(_tempDir, recursive: true); } + catch (IOException) { /* best-effort cleanup */ } + catch (UnauthorizedAccessException) { /* best-effort cleanup */ } + } + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + private static int s_metric; + + /// One alert through the real sender, the settings Lite really builds; a fresh metric name each time so no cooldown holds it. + private async Task FireAsync() + { + var service = new WebhookAlertService(new AppAlertSettings(_secrets.Access), EmailAlertService.Branding, new AppLoggerAdapter()); + var metric = "High CPU " + Interlocked.Increment(ref s_metric).ToString(CultureInfo.InvariantCulture); + await service.TrySendWebhookAlertsAsync(metric, "example-server", "97%", "90%", serverId: "7101", cancellationToken: Ct); + } + + /// The receiver check of Darling's docs/signed-webhooks.md, reduced to the HMAC: true when any signature in the header + /// equals the HMAC-SHA256 of id.timestamp.body under . + private static bool Verifies(string key, Captured request) + { + var id = request.Header(WebhookSigning.IdHeader); + var timestamp = request.Header(WebhookSigning.TimestampHeader); + var header = request.Header(WebhookSigning.SignatureHeader); + if (id is null || timestamp is null || header is null) + { + return false; + } + + var secret = Convert.FromBase64String(key[WebhookSigning.KeyPrefix.Length..]); + var signed = Encoding.UTF8.GetBytes(id + "." + timestamp + ".").Concat(request.Body).ToArray(); + var expected = "v1," + Convert.ToBase64String(HMACSHA256.HashData(secret, signed)); + return header.Split(' ').Contains(expected, StringComparer.Ordinal); + } + + private static async Task SaveRouteAsync(DuckDbNotificationRouteStore store, string url) => + await store.InsertAsync(new NotificationRoute(0, NotificationRoute.WildcardMatch, "", "", url, "", "", true) { ScopeServerId = 7101 }); + + private void LoadRoutesIntoTheCache() => + NotificationRouteCache.Configure(() => Task.Run(() => _store.ListAsync()).GetAwaiter().GetResult()); + + // ---- pin 1: the send is signed --------------------------------------------------------------- + + [Fact] + public async Task TheSettingsRowsGenericSend_CarriesTheThreeHeaders_AReceiverAccepts_AndRefusesOneChangedByte() + { + using var endpoint = new RawEndpoint(); + var made = LiteWebhookSigningKeys.Generate(_secrets.Access, LiteWebhookSigningKeys.SettingsName(false), LiteWebhookSigningKeys.SettingsName(true), LiteSigningState.None, DateTime.UtcNow); + Assert.NotNull(made.NewKey); + App.GenericWebhookEnabled = true; + App.GenericWebhookUrl = endpoint.Url; + App.GenericSigningSet = made.State.HasKey; + App.GenericSigningUntilUtc = made.State.PreviousUntilUtc; + + await FireAsync(); + + var request = Assert.Single(endpoint.Requests); + Assert.NotNull(request.Header(WebhookSigning.IdHeader)); + Assert.NotNull(request.Header(WebhookSigning.TimestampHeader)); + Assert.StartsWith("v1,", request.Header(WebhookSigning.SignatureHeader), StringComparison.Ordinal); + Assert.True(Verifies(made.NewKey!, request)); + var changed = (byte[])request.Body.Clone(); + changed[3] ^= 0x01; + Assert.False(Verifies(made.NewKey!, request with { Body = changed })); + } + + [Fact] + public async Task ARoutesOwnUrl_IsSignedWithTheRoutesKey_AndDuringARotationBothKeysVerify() + { + using var endpoint = new RawEndpoint(); + var id = await SaveRouteAsync(_store, endpoint.Url); + var (first, firstError) = await _store.GenerateSigningKeyAsync(id); + Assert.Null(firstError); + var (second, secondError) = await _store.GenerateSigningKeyAsync(id); + Assert.Null(secondError); + Assert.NotEqual(first, second); + LoadRoutesIntoTheCache(); + + await FireAsync(); + + var request = Assert.Single(endpoint.Requests); + Assert.True(Verifies(second!, request), "the new key signs"); + Assert.True(Verifies(first!, request), "the previous key signs during the 7 days"); + Assert.Equal(2, request.Header(WebhookSigning.SignatureHeader)!.Split(' ').Length); + } + + // ---- pin 2: where the key is kept ------------------------------------------------------------ + + [Fact] + public async Task TheKey_IsInTheSecretStore_AndInNoTableStatusTextOrErrorAfterItIsMade() + { + using var endpoint = new RawEndpoint(); + var id = await SaveRouteAsync(_store, endpoint.Url); + + var (key, error) = await _store.GenerateSigningKeyAsync(id); + + Assert.Null(error); + Assert.Equal(key, _secrets.Read(LiteWebhookSigningKeys.RouteName(id, false))); + var bare = key![WebhookSigning.KeyPrefix.Length..]; + + /* Every column of the route's row, as text: the flag and the rotation end are there, the key is not. */ + using (var connection = _db.CreateConnection()) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT CAST(r AS VARCHAR) FROM notification_routes AS r"; + var row = Convert.ToString(command.ExecuteScalar(), CultureInfo.InvariantCulture)!; + Assert.Contains("true", row, StringComparison.Ordinal); + Assert.DoesNotContain(bare, row, StringComparison.Ordinal); + } + + /* The route as the alert path reads it prints no key, and the text the screens show names none. */ + var route = Assert.Single(await _store.ListAsync(Ct)); + Assert.True(route.GenericSigning!.HasKey); + Assert.DoesNotContain(bare, route.GenericSigning.ToString(), StringComparison.Ordinal); + var state = new LiteSigningState(true, DateTime.UtcNow.AddDays(7)); + Assert.DoesNotContain(bare, LiteWebhookSigningKeys.StatusText(state, DateTime.UtcNow), StringComparison.Ordinal); + Assert.DoesNotContain(bare, LiteWebhookSigningKeys.StatusText(state, DateTime.UtcNow, unreadable: true), StringComparison.Ordinal); + + /* A write Credential Manager refuses reports why and never carries the key. */ + _secrets.WriteGate = (_, _) => false; + var refused = LiteWebhookSigningKeys.Generate(_secrets.Access, "a", "b", LiteSigningState.None, DateTime.UtcNow); + Assert.Null(refused.NewKey); + Assert.NotNull(refused.Error); + Assert.DoesNotContain(bare, refused.Error + refused, StringComparison.Ordinal); + } + + [Fact] + public void TheSettingsRowsTwoPlainFacts_LoadFromSettingsJson_AndNothingElseAboutTheKeyDoes() + { + var configDirectory = Path.Combine(_tempDir, "config"); + Directory.CreateDirectory(configDirectory); + var until = new DateTime(2026, 11, 1, 12, 0, 0, DateTimeKind.Utc); + File.WriteAllText(Path.Combine(configDirectory, "settings.json"), + "{\"generic_signing_set\":true,\"generic_signing_until\":\"" + until.ToString("o", CultureInfo.InvariantCulture) + "\"}"); + + App.LoadAlertSettings(configDirectory, _ => "", (key, _) => Assert.Fail($"LoadAlertSettings wrote credential '{key}'.")); + + Assert.True(App.GenericSigningSet); + Assert.Equal(until, App.GenericSigningUntilUtc); + } + + // ---- pin 3: a saved key that will not open sends nothing ------------------------------------- + + [Fact] + public async Task ASettingsRowWithAKeyFlagAndNoReadableKey_SendsNothing() + { + using var endpoint = new RawEndpoint(); + App.GenericWebhookEnabled = true; + App.GenericWebhookUrl = endpoint.Url; + App.GenericSigningSet = true; + + await FireAsync(); + + Assert.Empty(endpoint.Requests); + Assert.True(new AppAlertSettings(_secrets.Access).GenericSigningKeyUnreadable); + } + + [Fact] + public async Task ARouteWithAKeyFlagAndNoReadableKey_SendsNothingToItsUrl_AndNothingFallsToTheAppWideUrl() + { + using var routed = new RawEndpoint(); + using var appWide = new RawEndpoint(); + var id = await SaveRouteAsync(_store, routed.Url); + var (key, _) = await _store.GenerateSigningKeyAsync(id); + Assert.NotNull(key); + App.GenericWebhookEnabled = true; + App.GenericWebhookUrl = appWide.Url; + Assert.True(_secrets.Delete(LiteWebhookSigningKeys.RouteName(id, false))); + LoadRoutesIntoTheCache(); + + await FireAsync(); + + Assert.Empty(routed.Requests); + Assert.Empty(appWide.Requests); + Assert.True(Assert.Single(await _store.ListAsync(Ct)).IsOffFor(NotificationRouter.GenericChannel)); + } + + [Fact] + public async Task ADestinationWithNoKey_IsSentUnsigned_AsItAlwaysWas() + { + using var endpoint = new RawEndpoint(); + App.GenericWebhookEnabled = true; + App.GenericWebhookUrl = endpoint.Url; + + await FireAsync(); + + var request = Assert.Single(endpoint.Requests); + Assert.Null(request.Header(WebhookSigning.SignatureHeader)); + } + + // ---- round 2: a key is shown only after the flag's home took it ------------------------------ + + private static readonly string s_current = LiteWebhookSigningKeys.SettingsName(false); + private static readonly string s_previous = LiteWebhookSigningKeys.SettingsName(true); + + [Fact] + public void AFirstKeyWhoseFlagWillNotSave_ReturnsNoKey_AnError_AndLeavesCredentialManagerEmpty() + { + var result = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, LiteSigningState.None, DateTime.UtcNow, _ => "the disk is full"); + + Assert.Null(result.NewKey); + Assert.NotNull(result.Error); + Assert.Contains("the disk is full", result.Error, StringComparison.Ordinal); + Assert.False(result.State.HasKey); + Assert.Equal("", _secrets.Read(s_current)); + Assert.Equal("", _secrets.Read(s_previous)); + Assert.Equal(0, _secrets.Count); + } + + [Fact] + public void ARotationWhoseFlagWillNotSave_ReturnsNoKey_AndCredentialManagerHoldsTheOldCurrentAndPreviousKeys() + { + var saved = LiteSigningState.None; + var first = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, saved, DateTime.UtcNow, s => { saved = s; return null; }); + var second = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, saved, DateTime.UtcNow, s => { saved = s; return null; }); + Assert.NotNull(first.NewKey); + Assert.NotNull(second.NewKey); + Assert.Equal(second.NewKey, _secrets.Read(s_current)); + Assert.Equal(first.NewKey, _secrets.Read(s_previous)); + var before = saved; + + var failed = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, saved, DateTime.UtcNow, _ => "settings.json could not be written"); + + Assert.Null(failed.NewKey); + Assert.NotNull(failed.Error); + Assert.Equal(before, failed.State); + Assert.Equal(second.NewKey, _secrets.Read(s_current)); + Assert.Equal(first.NewKey, _secrets.Read(s_previous)); + } + + [Fact] + public void ARotationWithNoPreviousKeyWhoseFlagWillNotSave_LeavesTheCurrentKeyAndNoPreviousKey() + { + var saved = LiteSigningState.None; + var first = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, saved, DateTime.UtcNow, s => { saved = s; return null; }); + Assert.NotNull(first.NewKey); + + var failed = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, saved, DateTime.UtcNow, _ => throw new InvalidOperationException("boom")); + + Assert.Null(failed.NewKey); + Assert.NotNull(failed.Error); + Assert.Equal(first.NewKey, _secrets.Read(s_current)); + Assert.Equal("", _secrets.Read(s_previous)); + Assert.DoesNotContain(s_previous, _secrets.Keys); + } + + [Fact] + public void WhenTheFlagWillNotSaveAndCredentialManagerWillNotPutTheKeysBack_TheErrorSaysSo_AndShowsNoKey() + { + var saved = LiteSigningState.None; + _ = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, saved, DateTime.UtcNow, s => { saved = s; return null; }); + + var failed = LiteWebhookSigningKeys.Generate(_secrets.Access, s_current, s_previous, saved, DateTime.UtcNow, _ => + { + _secrets.WriteGate = (_, _) => false; + return "settings.json could not be written"; + }); + + Assert.Null(failed.NewKey); + Assert.Contains("could not be put back", failed.Error, StringComparison.Ordinal); + Assert.DoesNotContain(_secrets.Read(s_current)[WebhookSigning.KeyPrefix.Length..], failed.Error, StringComparison.Ordinal); + } + + /// Makes the next change of a route's flag to true fail the way a real write error would: a unique index on the flag + /// column, with one route already at true and another still at false. + private void MakeASecondTrueFlagFail(bool otherRoutesAreFalse = false) + { + using var connection = _db.CreateConnection(); + connection.Open(); + using var command = connection.CreateCommand(); + /* A new route's flag is NULL, and a unique index lets NULLs repeat; a failing REMOVE needs the other route at false, so + that the first route going from true to false is the duplicate. */ + command.CommandText = (otherRoutesAreFalse ? "UPDATE notification_routes SET generic_signing_set = false WHERE generic_signing_set IS NULL; " : "") + + "CREATE UNIQUE INDEX ux_signing_flag_test ON notification_routes (generic_signing_set)"; + command.ExecuteNonQuery(); + } + + [Fact] + public async Task ARouteWhoseFlagUpdateFails_ReturnsNoKeyAndAnError_AndLeavesCredentialManagerAsItWas() + { + using var endpoint = new RawEndpoint(); + var first = await SaveRouteAsync(_store, endpoint.Url); + var second = await SaveRouteAsync(_store, endpoint.Url + "2"); + var (firstKey, firstError) = await _store.GenerateSigningKeyAsync(first); + Assert.NotNull(firstKey); + Assert.Null(firstError); + MakeASecondTrueFlagFail(); + var before = _secrets.Keys.OrderBy(k => k, StringComparer.Ordinal).ToList(); + + var (key, error) = await _store.GenerateSigningKeyAsync(second); + + Assert.Null(key); + Assert.NotNull(error); + Assert.Equal(before, _secrets.Keys.OrderBy(k => k, StringComparer.Ordinal).ToList()); + Assert.Equal("", _secrets.Read(LiteWebhookSigningKeys.RouteName(second, false))); + Assert.Equal(firstKey, _secrets.Read(LiteWebhookSigningKeys.RouteName(first, false))); + var state = await _store.SigningStateAsync(second); + Assert.False(state!.Value.State.HasKey); + } + + [Fact] + public async Task ARouteWhoseFlagUpdateFailsOnRemove_SaysSettingsWereNotSaved_AndTheRouteIsHeldBack() + { + using var endpoint = new RawEndpoint(); + var first = await SaveRouteAsync(_store, endpoint.Url); + var second = await SaveRouteAsync(_store, endpoint.Url + "2"); + Assert.NotNull((await _store.GenerateSigningKeyAsync(first)).Key); + MakeASecondTrueFlagFail(otherRoutesAreFalse: true); + + var error = await _store.RemoveSigningKeyAsync(first); + + Assert.Equal(LiteWebhookSigningKeys.RemoveNotSavedText, error); + Assert.Contains("held back", error, StringComparison.Ordinal); + var state = await _store.SigningStateAsync(first); + Assert.True(state!.Value.State.HasKey); + Assert.True(state.Value.Unreadable); + Assert.False((await _store.SigningStateAsync(second))!.Value.State.HasKey); + } + + // ---- round 2: the route dialog's facts come from the row ------------------------------------ + + [Fact] + public async Task TheStoresSigningRead_ReportsUnreadable_ForARouteWithTheFlagAndNoKeyInCredentialManager() + { + using var endpoint = new RawEndpoint(); + var id = await SaveRouteAsync(_store, endpoint.Url); + Assert.NotNull((await _store.GenerateSigningKeyAsync(id)).Key); + Assert.False((await _store.SigningStateAsync(id))!.Value.Unreadable); + Assert.True(_secrets.Delete(LiteWebhookSigningKeys.RouteName(id, false))); + + var read = await _store.SigningStateAsync(id); + + Assert.True(read!.Value.State.HasKey); + Assert.True(read.Value.Unreadable); + + /* The route as loaded has no key and its generic channel off: that is what the dialog could not tell from the URL case. */ + var loaded = Assert.Single(await _store.ListAsync(Ct)); + Assert.True(loaded.IsOffFor(NotificationRouter.GenericChannel)); + Assert.False(loaded.GenericSigning!.HasKey); + } + + [Fact] + public async Task TheStoresSigningRead_IsNotUnreadable_ForARouteWhoseGenericChannelIsOffBecauseItsUrlWillNotOpen() + { + using var endpoint = new RawEndpoint(); + var id = await SaveRouteAsync(_store, endpoint.Url); + _secrets.Lose(id, NotificationRouter.GenericChannel); + Assert.True(Assert.Single(await _store.ListAsync(Ct)).IsOffFor(NotificationRouter.GenericChannel)); + + var read = await _store.SigningStateAsync(id); + + Assert.False(read!.Value.State.HasKey); + Assert.False(read.Value.Unreadable); + Assert.Null(await _store.SigningStateAsync(id + 1000)); + } + + [Fact] + public async Task TheStoresSigningRead_ReportsTheRotationEndTheRowSaved() + { + using var endpoint = new RawEndpoint(); + var id = await SaveRouteAsync(_store, endpoint.Url); + Assert.NotNull((await _store.GenerateSigningKeyAsync(id)).Key); + Assert.Null((await _store.SigningStateAsync(id))!.Value.State.PreviousUntilUtc); + + /* The old key will not open, so the rotation saves no overlap: the dialog must not guess one. */ + Assert.True(_secrets.Delete(LiteWebhookSigningKeys.RouteName(id, false))); + Assert.NotNull((await _store.GenerateSigningKeyAsync(id)).Key); + Assert.Null((await _store.SigningStateAsync(id))!.Value.State.PreviousUntilUtc); + + Assert.NotNull((await _store.GenerateSigningKeyAsync(id)).Key); + var until = (await _store.SigningStateAsync(id))!.Value.State.PreviousUntilUtc; + Assert.NotNull(until); + Assert.True(until > DateTime.UtcNow.AddDays(6)); + } + + internal sealed record Captured(string Head, byte[] Body) + { + public string? Header(string name) => Head.Split("\r\n") + .Select(l => l.Split(':', 2)) + .Where(p => p.Length == 2 && string.Equals(p[0].Trim(), name, StringComparison.OrdinalIgnoreCase)) + .Select(p => p[1].Trim()) + .FirstOrDefault(); + } + + /// A loopback endpoint that answers every POST 200 and keeps each request's head and exact body bytes. + private sealed class RawEndpoint : IDisposable + { + private readonly TcpListener _listener = new(IPAddress.Loopback, 0); + private readonly CancellationTokenSource _stop = new(); + private readonly ConcurrentQueue _requests = new(); + + public RawEndpoint() + { + _listener.Start(); + Url = $"http://127.0.0.1:{((IPEndPoint)_listener.LocalEndpoint).Port}/hook"; + _ = Task.Run(AcceptLoopAsync); + } + + public string Url { get; } + + public IReadOnlyList Requests => _requests.ToArray(); + + private async Task AcceptLoopAsync() + { + while (!_stop.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(_stop.Token); + } + catch (Exception) + { + return; + } + + using (client) + { + var stream = client.GetStream(); + var buffer = new MemoryStream(); + var chunk = new byte[8192]; + var headEnd = -1; + var length = 0; + while (true) + { + var read = await stream.ReadAsync(chunk, _stop.Token); + if (read == 0) + { + break; + } + + buffer.Write(chunk, 0, read); + var all = buffer.ToArray(); + if (headEnd < 0) + { + headEnd = IndexOf(all, "\r\n\r\n"u8.ToArray()); + if (headEnd >= 0) + { + var head = Encoding.ASCII.GetString(all, 0, headEnd); + var line = head.Split("\r\n").FirstOrDefault(l => l.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase)); + length = line is null ? 0 : int.Parse(line[15..].Trim(), CultureInfo.InvariantCulture); + } + } + + if (headEnd >= 0 && all.Length >= headEnd + 4 + length) + { + _requests.Enqueue(new Captured(Encoding.ASCII.GetString(all, 0, headEnd), all.AsSpan(headEnd + 4, length).ToArray())); + await stream.WriteAsync("HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"u8.ToArray(), _stop.Token); + break; + } + } + } + } + } + + private static int IndexOf(byte[] haystack, byte[] needle) + { + for (var i = 0; i <= haystack.Length - needle.Length; i++) + { + if (haystack.AsSpan(i, needle.Length).SequenceEqual(needle)) + { + return i; + } + } + + return -1; + } + + public void Dispose() + { + _stop.Cancel(); + _listener.Stop(); + _stop.Dispose(); + } + } +} diff --git a/Lite.Tests/NotificationRouteStoreTests.cs b/Lite.Tests/NotificationRouteStoreTests.cs index cbb7b8acf4..1d000f14fc 100644 --- a/Lite.Tests/NotificationRouteStoreTests.cs +++ b/Lite.Tests/NotificationRouteStoreTests.cs @@ -192,7 +192,7 @@ public async Task NoUrlOrKey_IsEverWrittenToDuckDb_InAnyColumn() while (await reader.ReadAsync()) columns.Add(reader.GetString(0)); } - Assert.Equal(19, columns.Count); /* the 14 route columns plus the five agent settings (#5820), which hold numbers only */ + Assert.Equal(21, columns.Count); /* the 14 route columns plus the five agent settings (#5820) and the signing key flag and rotation end (#6012), which hold numbers and a time only */ var needles = new[] { "r3-secret", "R3PAGERDUTY", "hooks.slack", "https://", "hook/abc123" }; foreach (var column in columns) { diff --git a/Lite/App.xaml.cs b/Lite/App.xaml.cs index 83d36c3a89..8bc61c15ba 100644 --- a/Lite/App.xaml.cs +++ b/Lite/App.xaml.cs @@ -379,6 +379,11 @@ background slices. Default ON. Turn it off when a heavy catch-up is costing the public static string GenericWebhookBodyTemplate { get; set; } = ""; public static string GenericWebhookProxyAddress { get; set; } = ""; + /* #6012: the settings row's generic signing key. Only these two facts are plain prefs (settings.json): whether a key is saved + and when the previous key stops signing. The keys themselves are in Credential Manager (LiteWebhookSigningKeys). */ + public static bool GenericSigningSet { get; set; } = false; + public static DateTime? GenericSigningUntilUtc { get; set; } + /* PagerDuty webhook settings — Events API v2. The routing key is a bearer secret like the Teams/Slack URLs, so it lives in Credential Manager, never in settings.json; only the enable flag and the EU-region toggle are plain prefs. */ @@ -1520,6 +1525,13 @@ an out-of-range timeout here has always been ignored in favour of the current va if (read.TryGetProperty("generic_webhook_enabled", out v)) GenericWebhookEnabled = v.Bool(GenericWebhookEnabled); if (read.TryGetProperty("generic_proxy_address", out v)) GenericWebhookProxyAddress = v.Text(GenericWebhookProxyAddress); if (read.TryGetProperty("generic_body_template", out v)) GenericWebhookBodyTemplate = v.Text(GenericWebhookBodyTemplate); + if (read.TryGetProperty("generic_signing_set", out v)) GenericSigningSet = v.Bool(GenericSigningSet); + if (read.TryGetProperty("generic_signing_until", out v) + && DateTime.TryParse(v.TextOrNull(), System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.AssumeUniversal | System.Globalization.DateTimeStyles.AdjustToUniversal, out var signingUntil)) + { + GenericSigningUntilUtc = signingUntil; + } /* PagerDuty webhook settings. The routing key is a secret and loads from Credential Manager below; only the enable flag and EU-region toggle are plain prefs. */ @@ -1666,17 +1678,25 @@ internal static bool WriteSettingsDocument(JsonNode root, string what) /// , which is what keeps an unparseable file from being replaced /// unrecorded. /// - public static void WriteSetting(string what, Action mutate) + public static void WriteSetting(string what, Action mutate) => _ = TryWriteSetting(what, mutate); + + /// + /// that says whether settings.json was written (#6012): true only when the file was written, false + /// when the document could not be read for write, threw, or the write failed. The error is logged + /// the same way. A caller that must not show something the file does not hold (a new signing key) uses this one. + /// + public static bool TryWriteSetting(string what, Action mutate) { try { JsonNode root = SettingsRootForWrite(); mutate(root); - WriteSettingsDocument(root, what); + return WriteSettingsDocument(root, what); } catch (Exception ex) { AppLogger.Error("Settings", $"Failed to save {what}: {ex.Message}"); + return false; } } diff --git a/Lite/Database/DuckDbInitializer.cs b/Lite/Database/DuckDbInitializer.cs index b3b541900e..8d2c8c74bc 100644 --- a/Lite/Database/DuckDbInitializer.cs +++ b/Lite/Database/DuckDbInitializer.cs @@ -423,7 +423,7 @@ public void Dispose() /// /// Current schema version. Increment this when schema changes require table rebuilds. /// - internal const int CurrentSchemaVersion = 74; + internal const int CurrentSchemaVersion = 75; private readonly string _archivePath; @@ -3061,6 +3061,13 @@ table statements that follow retry each CREATE on the next start. */ _logger?.LogInformation("Running migration to v74: adding investigation_reports.family_of and queries_run"); await AddMissingColumnsAsync(connection, AddedColumnsForVersion(74)); } + + if (fromVersion < 75) + { + /* v75 (#6012): investigation_requests.follow_up_of and follow_up_at, Darling's V193 follow-up columns (#6008), and the route signing-key flag (the shared added-columns list). */ + _logger?.LogInformation("Running migration to v75: adding investigation_requests.follow_up_of and follow_up_at, notification_routes.generic_signing_set and generic_signing_until"); + await AddMissingColumnsAsync(connection, AddedColumnsForVersion(75)); + } } /* #5990: set when a migration step failed in a way that must be retried at the next start; the stamp then stops at this @@ -3124,6 +3131,13 @@ internal static readonly (int Version, string Table, string Column, string Type) NULL on a report stored before v74. family_of carries no foreign key here: the tool's existence check is the guard. */ (74, "investigation_reports", "family_of", "UUID"), (74, "investigation_reports", "queries_run", "VARCHAR"), + /* #6012: the request this follow-up was opened for, and the time a reported request asked to be followed up at (Darling's V193 + follow_up_of and follow_up_at, #6008). NULL on a request stored before v75. follow_up_of carries no foreign key: the purge clears it. */ + (75, "investigation_requests", "follow_up_of", "UUID"), + (75, "investigation_requests", "follow_up_at", "TIMESTAMP"), + /* #6012: a route's generic signing key flag and the end of its rotation overlap (non-secret; the keys are in Credential Manager). NULL reads as no key. */ + (75, "notification_routes", "generic_signing_set", "BOOLEAN"), + (75, "notification_routes", "generic_signing_until", "TIMESTAMP"), }; internal static IEnumerable<(int Version, string Table, string Column, string Type)> AddedColumnsForVersion(int version) => diff --git a/Lite/Database/DuckDbInvestigationStore.cs b/Lite/Database/DuckDbInvestigationStore.cs index 3d41887a9f..fc72209883 100644 --- a/Lite/Database/DuckDbInvestigationStore.cs +++ b/Lite/Database/DuckDbInvestigationStore.cs @@ -128,10 +128,57 @@ FROM investigation_requests WHERE request_id = $1"; internal const string SubmitLockSql = @" -SELECT state, claim_id, server_id, first_metric, first_dedup_key, story_path_hash +SELECT state, claim_id, server_id, first_metric, first_dedup_key, story_path_hash, route_id FROM investigation_requests WHERE request_id = $1"; + /* #6012 (Darling's PendingFollowUpsSql, #6008): the pending follow-ups of a route: reported requests that set follow_up_at and + whose follow-up has not opened. Lite has no practice requests, so there is no practice_of filter. $1 route. */ + internal const string PendingFollowUpsSql = @" +SELECT count(*) +FROM investigation_requests AS r +WHERE r.route_id = $1 + AND r.state = 'reported' + AND r.follow_up_at IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM investigation_requests AS f WHERE f.follow_up_of = r.request_id)"; + + /* Written only when the report set a time. $1 request, $2 time. */ + internal const string SetFollowUpSql = @" +UPDATE investigation_requests SET follow_up_at = $2 WHERE request_id = $1"; + + /* The reported requests whose follow-up is due and not yet opened, oldest first (Darling's DueFollowUpsSql without the deploy + condition: Lite takes the time only). $1 now. */ + internal const string DueFollowUpsSql = @" +SELECT r.request_id +FROM investigation_requests AS r +WHERE r.state = 'reported' + AND r.follow_up_at IS NOT NULL + AND r.follow_up_at <= $1 + AND NOT EXISTS (SELECT 1 FROM investigation_requests AS f WHERE f.follow_up_of = r.request_id) +ORDER BY r.follow_up_at, r.request_id +LIMIT 50"; + + /* The follow-up request copies its source's route, server, family, metric, story path, alert times, life and level, and names the + source in follow_up_of (Darling's OpenFollowUpSql). $1 new id, $2 source, $3 now. */ + internal const string OpenFollowUpSql = @" +INSERT INTO investigation_requests + (request_id, route_id, server_id, server_name, family, first_metric, story_path_hash, first_alert_at, last_alert_at, + window_until, opened_at, expires_at, lease_minutes, autonomy_level, alert_count, max_severity, follow_up_of) +SELECT $1, r.route_id, r.server_id, r.server_name, r.family, r.first_metric, r.story_path_hash, r.first_alert_at, r.last_alert_at, + r.window_until, $3, $3 + (r.expires_at - r.opened_at), r.lease_minutes, r.autonomy_level, r.alert_count, r.max_severity, r.request_id +FROM investigation_requests AS r +WHERE r.request_id = $2 +RETURNING route_id, server_id"; + + /* The source's alert rows, without the dedup key, so the brief's alerts section reads what the source's read. $1 new, $2 source. */ + internal const string CopyFollowUpAlertsSql = @" +INSERT INTO investigation_request_alerts + (request_alert_id, request_id, alert_time, server_id, metric_name, dedup_key, current_value, threshold_value, muted, detail_text, context_json) +SELECT (SELECT coalesce(max(request_alert_id), 0) FROM investigation_request_alerts) + row_number() OVER (ORDER BY request_alert_id), + $1, alert_time, server_id, metric_name, NULL, current_value, threshold_value, muted, detail_text, context_json +FROM investigation_request_alerts +WHERE request_id = $2"; + internal const string ReportInsertSql = @" INSERT INTO investigation_reports (report_id, request_id, report_format, agent_id, claim_id, submitted_at, verdict, root_cause, root_cause_found, summary, @@ -341,6 +388,12 @@ UPDATE investigation_requests "UPDATE investigation_reports SET family_of = NULL WHERE family_of IN (" + RetentionDoomedRequests + ")" + " AND request_id NOT IN (" + RetentionDoomedRequests + ")"; + /* #6012: Darling's follow_up_of is a foreign key ON DELETE SET NULL; a purge clears it the way it clears family_of, so a served + follow_up_of always names a request. A follow-up that is itself doomed goes with the delete. */ + internal const string RetentionFollowUpOfSql = + "UPDATE investigation_requests SET follow_up_of = NULL WHERE follow_up_of IN (" + RetentionDoomedRequests + ")" + + " AND request_id NOT IN (" + RetentionDoomedRequests + ")"; + internal const string RetentionRequestsSql = "DELETE FROM investigation_requests WHERE state NOT IN ('open', 'claimed') AND opened_at < $1"; @@ -593,7 +646,7 @@ public async Task SubmitAsync(InvestigationReportSub string? state = null; Guid? currentClaim = null; - int serverId = 0; + int serverId = 0, routeId = 0; string? metric = null, dedupKey = null, story = null; using (var command = Command(connection, SubmitLockSql, report.RequestId)) using (var reader = await command.ExecuteReaderAsync(cancellationToken)) @@ -606,6 +659,7 @@ public async Task SubmitAsync(InvestigationReportSub metric = reader.GetString(3); dedupKey = reader.IsDBNull(4) ? null : reader.GetString(4); story = reader.IsDBNull(5) ? null : reader.GetString(5); + routeId = reader.GetInt32(6); } } @@ -644,6 +698,23 @@ await JournalAsync(connection, now, state is null ? null : report.RequestId, nul return outcome; } + /* #6012 (Darling's #6008): a follow-up time over the route's pending cap refuses the whole report, before anything is written. + The write lock held since the top makes concurrent submits count in turn. */ + if (report.FollowUpAtUtc is not null) + { + long pending; + using (var count = Command(connection, PendingFollowUpsSql, routeId)) + { + pending = Convert.ToInt64(await count.ExecuteScalarAsync(cancellationToken), CultureInfo.InvariantCulture); + } + + if (pending >= InvestigationReportFields.MaxPendingFollowUpsPerRoute) + { + transaction.Commit(); + return InvestigationSubmitOutcome.FollowUpLimit; + } + } + await ExecAsync(connection, ReportInsertSql, cancellationToken, report.RequestId, report.ReportFormat, report.AgentId, report.ClaimId, now, report.Verdict, report.RootCause, report.RootCauseFound, report.Summary, report.EvidenceJson, serverId, @@ -654,6 +725,11 @@ await ExecAsync(connection, ReportInsertSql, cancellationToken, } await ExecAsync(connection, ReportedSql, cancellationToken, report.RequestId, now); + if (report.FollowUpAtUtc is { } followUpAt) + { + await ExecAsync(connection, SetFollowUpSql, cancellationToken, report.RequestId, followUpAt); + } + await JournalAsync(connection, now, report.RequestId, null, serverId, "reported", actor, report.Verdict, null, null, cancellationToken); /* #5914, Darling's rule in the shared contract: each link a link_recorded row, once per request and URL. Lite's submit @@ -913,7 +989,52 @@ public async Task SweepAsync(CancellationToken cancell using var resolvedTransaction = connection.BeginTransaction(); var resolved = await SweepResolvedAsync(connection, now, cancellationToken); resolvedTransaction.Commit(); - return new InvestigationSweepResult(lapsed, expired) { Resolved = resolved }; + + /* #6012 (Darling's #6008): its own transaction too, so a follow-up that cannot open holds back neither the expiry nor the + resolved journal. */ + using var followUpTransaction = connection.BeginTransaction(); + var followUps = await SweepFollowUpsAsync(connection, now, cancellationToken); + followUpTransaction.Commit(); + return new InvestigationSweepResult(lapsed, expired) { Resolved = resolved, FollowUps = followUps }; + } + + /// + /// The follow-up sweep (#6012, Darling's #6008): for each reported request whose follow_up_at has come and that has no + /// follow-up yet, opens a new request that names it in follow_up_of and journals one follow_up event (trigger + /// time) for the new request, once per earlier request. The new request's brief reads the earlier report as it reads any + /// earlier report of the same server and family. Lite has no event delivery, so an agent finds the follow-up in the request list. + /// Returns the follow-ups opened. + /// + private async Task SweepFollowUpsAsync(DuckDBConnection connection, DateTime now, CancellationToken cancellationToken) + { + var due = new List(); + using (var candidates = Command(connection, DueFollowUpsSql, now)) + using (var reader = await candidates.ExecuteReaderAsync(cancellationToken)) + { + while (await reader.ReadAsync(cancellationToken)) + { + due.Add(reader.GetGuid(0)); + } + } + + foreach (var earlier in due) + { + var followUpId = Guid.NewGuid(); + int routeId, serverId; + using (var insert = Command(connection, OpenFollowUpSql, followUpId, earlier, now)) + using (var returned = await insert.ExecuteReaderAsync(cancellationToken)) + { + await returned.ReadAsync(cancellationToken); + routeId = returned.GetInt32(0); + serverId = returned.GetInt32(1); + } + + await ExecAsync(connection, CopyFollowUpAlertsSql, cancellationToken, followUpId, earlier); + await JournalAsync(connection, now, followUpId, routeId, serverId, InvestigationEvents.FollowUp, "service", + null, "{\"follow_up_of\":\"" + earlier.ToString("D") + "\",\"trigger\":\"time\"}", InvestigationEvents.FollowUpOnceKey(earlier), cancellationToken); + } + + return due.Count; } /* #6010, D7 (Darling's #6003): a live request is resolved when EVERY metric linked to it has a resolution row after that @@ -1010,6 +1131,7 @@ public async Task PurgeAsync(CancellationToken cancellationToken = default) await ExecAsync(connection, RetentionAlertRowsSql, cancellationToken, cutoff); await ExecAsync(connection, RetentionEvidenceSql, cancellationToken, cutoff); await ExecAsync(connection, RetentionFamilyOfSql, cancellationToken, cutoff); + await ExecAsync(connection, RetentionFollowUpOfSql, cancellationToken, cutoff); await ExecAsync(connection, RetentionReportsSql, cancellationToken, cutoff); var requests = await ExecAsync(connection, RetentionRequestsSql, cancellationToken, cutoff); await ExecAsync(connection, RetentionJournalSql, cancellationToken, cutoff); diff --git a/Lite/Database/Schema.cs b/Lite/Database/Schema.cs index df8ce4497c..b9b2f11feb 100644 --- a/Lite/Database/Schema.cs +++ b/Lite/Database/Schema.cs @@ -397,6 +397,8 @@ CREATE TABLE IF NOT EXISTS notification_routes ( agent_max_per_day INTEGER, agent_lease_minutes INTEGER, agent_expiry_minutes INTEGER, + generic_signing_set BOOLEAN, + generic_signing_until TIMESTAMP, CHECK (scope_server_id IS NULL OR scope_tag_id IS NULL), CHECK (metric_match <> '*' OR scope_server_id IS NOT NULL OR scope_tag_id IS NOT NULL) )"; @@ -455,6 +457,8 @@ CREATE TABLE IF NOT EXISTS investigation_requests ( closed_reason VARCHAR, evidence_state VARCHAR, max_severity VARCHAR, + follow_up_of UUID, + follow_up_at TIMESTAMP, CHECK (family IN ('blocking', 'query_performance', 'resources', 'availability', 'jobs', 'self_monitor', 'other')), CHECK (state IN ('open', 'claimed', 'reported', 'closed', 'expired')) )"; diff --git a/Lite/Mcp/McpInvestigationTools.cs b/Lite/Mcp/McpInvestigationTools.cs index 578300297c..0c58be3dcb 100644 --- a/Lite/Mcp/McpInvestigationTools.cs +++ b/Lite/Mcp/McpInvestigationTools.cs @@ -210,7 +210,11 @@ await store.RecordRefusalAsync(requestId, InvestigationRules.InvalidAgentId, "cl "Lite stores the link and never calls Jira or git. " + "family_of (optional) is the request_id of the first report in this family; it must name a request (else invalid). " + "queries_run (optional) is a JSON array of the statements you ran, " + InvestigationReportFields.QueriesRunRule + " (over a cap: refused, never cut); " + - "it is masked as statement text.")] + "it is masked as statement text. " + + "follow_up_at (optional) asks Lite to open a new request for this case later: " + InvestigationReportFields.FollowUpRule + ". " + + "When the time comes the new request names this one (follow_up_of), its brief carries this report, and one follow_up event is " + + "journaled; Lite sends no event, so find it in list_investigation_requests. Lite takes a time only, no deploy condition. " + + "A time in the past, more than 30 days ahead, or a route already holding 20 pending follow-ups is refused (invalid), and nothing is stored.")] public static Task SubmitInvestigationReport( IInvestigationRequestStore store, [Description("The request you claimed.")] string request_id, @@ -225,9 +229,10 @@ public static Task SubmitInvestigationReport( [Description("Optional JSON array of at most 10 {kind, url, key}; https only. See the guide.")] string? links = null, [Description("Optional. The request_id of the first report in this family; it must exist.")] string? family_of = null, [Description("Optional JSON array of statements you ran, " + InvestigationReportFields.QueriesRunRule + ".")] string? queries_run = null, + [Description("Optional. When to open a follow-up request: " + InvestigationReportFields.FollowUpRule + ".")] string? follow_up_at = null, CancellationToken cancellationToken = default) => SubmitAsync(store, request_id, claim_id, report_format, verdict, root_cause, evidence, summary, - proposed_note, action_proposals, cancellationToken, links, family_of, queries_run); + proposed_note, action_proposals, cancellationToken, links, family_of, queries_run, follow_up_at); /// The reported answer: links_recorded only when the report carried a link (Darling's answer, #5914). private static string ReportedAnswer(Guid requestId, string? proposedNote, int linksRecorded) @@ -247,7 +252,7 @@ private static string ReportedAnswer(Guid requestId, string? proposedNote, int l internal static async Task SubmitAsync(IInvestigationRequestStore store, string request_id, string claim_id, int report_format, string verdict, string root_cause, string evidence, string summary, string? proposed_note, string? action_proposals, CancellationToken cancellationToken, string? links = null, string? family_of = null, - string? queries_run = null) + string? queries_run = null, string? follow_up_at = null) { var problems = InvestigationToolCore.ValidateReport(request_id, claim_id, verdict, root_cause, evidence, summary, proposed_note, action_proposals, out var requestId, out var claimId, out var evidenceJson, requireBasis: true); @@ -265,6 +270,9 @@ internal static async Task SubmitAsync(IInvestigationRequestStore store, if (InvestigationReportFields.Validate(queries_run, out var queriesRunJson) is { } queriesProblem) problems.Add("queries_run: " + queriesProblem); + /* #6012 (Darling's #6008): follow_up_at after queries_run, for the same reason. The route's pending cap is the store's. */ + if (InvestigationReportFields.ValidateFollowUp(follow_up_at, DateTime.UtcNow, out var followUpAtUtc) is { } followUpProblem) + problems.Add("follow_up_at: " + followUpProblem); try { var agent = requestId != Guid.Empty && claimId != Guid.Empty @@ -315,7 +323,7 @@ internal static async Task SubmitAsync(IInvestigationRequestStore store, requestId, claimId, agent, (short)report_format, verdict, root_cause.Trim(), RootCauseFound: verdict != "needs_person", summary.Trim(), evidenceJson!, string.IsNullOrWhiteSpace(proposed_note) ? null : proposed_note.Trim(), reportLinks.Count > 0 ? reportLinks : null, - familyOf, queriesRunJson), cancellationToken); + familyOf, queriesRunJson, followUpAtUtc), cancellationToken); return outcome switch { @@ -326,6 +334,9 @@ internal static async Task SubmitAsync(IInvestigationRequestStore store, InvestigationSubmitOutcome.NotOpen => InvestigationToolCore.Status("not_open", "The request is closed."), InvestigationSubmitOutcome.NotFound => InvestigationToolCore.Status("not_found", $"No investigation request has id {requestId}."), InvestigationSubmitOutcome.AgentIdInvalid => InvestigationToolCore.Status("agent_id_invalid", "The claim's agent id is not valid. " + InvestigationRules.AgentIdRule), + InvestigationSubmitOutcome.FollowUpLimit => McpHelpers.Status("refused", + $"The report was not stored: this request's route already holds {InvestigationReportFields.MaxPendingFollowUpsPerRoute} pending follow-ups. Submit without follow_up_at, or wait for one to open.", + new { parameter = "follow_up_at", cap = InvestigationReportFields.MaxPendingFollowUpsPerRoute }), _ => InvestigationToolCore.Status("journal_capped", $"The request already holds {InvestigationRules.MaxAgentJournalRows} agent journal rows; the report was refused."), }; diff --git a/Lite/Services/AppAlertSettings.cs b/Lite/Services/AppAlertSettings.cs index 6378914751..f488c36fc8 100644 --- a/Lite/Services/AppAlertSettings.cs +++ b/Lite/Services/AppAlertSettings.cs @@ -6,6 +6,7 @@ * Licensed under the MIT License. See LICENSE file in the project root for full license information. */ +using System; using PerformanceMonitor.Notifications; namespace PerformanceMonitorLite.Services; @@ -38,10 +39,46 @@ public sealed class AppAlertSettings : IAlertSettings public string SlackWebhookUrl => App.SlackWebhookUrl; public string SlackProxyAddress => App.SlackProxyAddress; - public bool GenericWebhookEnabled => App.GenericWebhookEnabled; - public string GenericWebhookUrl => App.GenericWebhookUrl; + /* #6012 (Darling's #5998): a saved signing key that will not open turns the settings row's generic URL off, so a destination + that was set up signed is never sent unsigned. */ + public bool GenericWebhookEnabled => App.GenericWebhookEnabled && !GenericSigningKeyUnreadable; + public string GenericWebhookUrl => GenericSigningKeyUnreadable ? "" : App.GenericWebhookUrl; public string GenericWebhookHeadersJson => App.GenericWebhookHeadersJson; public string GenericWebhookBodyTemplate => App.GenericWebhookBodyTemplate; + + private readonly RouteSecretAccess? _signingSecrets; + + /// The app's settings; is where the signing keys are read (null: Windows Credential + /// Manager). A test passes an in-memory pair. + public AppAlertSettings(RouteSecretAccess? signingSecrets = null) + { + _signingSecrets = signingSecrets; + } + + private RouteSecretAccess Secrets => _signingSecrets ?? RouteSecretAccess.CredentialManager(); + + private WebhookSigningKeys OpenSigning(out bool unreadable) => + LiteWebhookSigningKeys.Open( + Secrets, LiteWebhookSigningKeys.SettingsName(false), LiteWebhookSigningKeys.SettingsName(true), + new LiteSigningState(App.GenericSigningSet, App.GenericSigningUntilUtc), DateTime.UtcNow, out unreadable); + + /// #6012: the signing keys of the settings row's generic URL, opened. None when no key is saved. + public WebhookSigningKeys GenericWebhookSigning => OpenSigning(out _); + + /// #6012: a key is saved and will not open. + public bool GenericSigningKeyUnreadable + { + get + { + if (!App.GenericSigningSet) + { + return false; + } + + OpenSigning(out var unreadable); + return unreadable; + } + } public string GenericWebhookProxyAddress => App.GenericWebhookProxyAddress; /* PagerDuty: the enable flag AND a non-empty routing key, matching the sibling channels' shape diff --git a/Lite/Services/DuckDbNotificationRouteStore.cs b/Lite/Services/DuckDbNotificationRouteStore.cs index 4f9160b6dd..da1dcd7622 100644 --- a/Lite/Services/DuckDbNotificationRouteStore.cs +++ b/Lite/Services/DuckDbNotificationRouteStore.cs @@ -83,7 +83,8 @@ public sealed class DuckDbNotificationRouteStore private const string RowColumns = "route_id, metric_match, scope_server_id, scope_tag_id, teams_set, slack_set, generic_set, pagerduty_set, " + "smtp_recipients, enabled, teams_secret_gen, slack_secret_gen, generic_secret_gen, pagerduty_secret_gen, " - + "agent_level, agent_max_per_hour, agent_max_per_day, agent_lease_minutes, agent_expiry_minutes"; + + "agent_level, agent_max_per_hour, agent_max_per_day, agent_lease_minutes, agent_expiry_minutes, " + + "generic_signing_set, generic_signing_until"; private readonly DuckDbInitializer _dbInitializer; private readonly Func> _knownServerIds; @@ -467,6 +468,13 @@ public async Task DeleteAsync(int routeId) } DeleteKeysLeniently(routeId, keys.Select(kv => (kv.Value, kv.Key)), "route"); + + /* #6012: the route's signing keys go with it; a key that will not go is read by no route. */ + DeleteKeysLeniently(routeId, new[] + { + (NotificationRouter.GenericChannel, LiteWebhookSigningKeys.RouteName(routeId, false)), + (NotificationRouter.GenericChannel, LiteWebhookSigningKeys.RouteName(routeId, true)), + }, "signing"); } NotificationRouteCache.Invalidate(); @@ -477,7 +485,8 @@ public async Task DeleteAsync(int routeId) private sealed record Row( int RouteId, string MetricMatch, int? ScopeServerId, int? ScopeTagId, bool[] Set, string SmtpRecipients, bool Enabled, string?[] Gens, - int? AgentLevel, int AgentMaxPerHour, int AgentMaxPerDay, int AgentLeaseMinutes, int AgentExpiryMinutes); + int? AgentLevel, int AgentMaxPerHour, int AgentMaxPerDay, int AgentLeaseMinutes, int AgentExpiryMinutes, + bool SigningSet, DateTime? SigningUntilUtc); private static Row ReadRowFrom(System.Data.Common.DbDataReader reader) => new( @@ -497,7 +506,10 @@ private static Row ReadRowFrom(System.Data.Common.DbDataReader reader) => reader.IsDBNull(15) ? NotificationRoute.DefaultAgentMaxPerHour : reader.GetInt32(15), reader.IsDBNull(16) ? NotificationRoute.DefaultAgentMaxPerDay : reader.GetInt32(16), reader.IsDBNull(17) ? NotificationRoute.DefaultAgentLeaseMinutes : reader.GetInt32(17), - reader.IsDBNull(18) ? NotificationRoute.DefaultAgentExpiryMinutes : reader.GetInt32(18)); + reader.IsDBNull(18) ? NotificationRoute.DefaultAgentExpiryMinutes : reader.GetInt32(18), + /* #6012: the signing key's flag and its rotation end (UTC); NULL is no key. */ + !reader.IsDBNull(19) && reader.GetBoolean(19), + reader.IsDBNull(20) ? null : DateTime.SpecifyKind(reader.GetDateTime(20), DateTimeKind.Utc)); private static async Task> ReadAllRowsAsync(DuckDBConnection connection, CancellationToken cancellationToken = default) { @@ -573,6 +585,29 @@ string Read(int index) var generic = Read(2); var pagerDuty = Read(3); + /* #6012 (Darling's #5998): a route's own generic URL is signed with the route's own keys. A saved key that will not open + turns the generic channel off for this route, so its alerts are neither sent unsigned nor sent to the app-wide URL. */ + var signing = WebhookSigningKeys.None; + if (row.SigningSet && !string.IsNullOrEmpty(generic)) + { + signing = LiteWebhookSigningKeys.Open( + _secrets, LiteWebhookSigningKeys.RouteName(row.RouteId, false), LiteWebhookSigningKeys.RouteName(row.RouteId, true), + new LiteSigningState(true, row.SigningUntilUtc), DateTime.UtcNow, out var unreadable); + if (unreadable) + { + generic = ""; + if (!off.Contains(NotificationRouter.GenericChannel)) + { + off.Add(NotificationRouter.GenericChannel); + } + + if (_warnedUnreadable.TryAdd($"{row.RouteId}_signing", 0)) + { + AppLogger.Warn("NotificationRoutes", $"Notification route {row.RouteId} has a saved generic webhook signing key that could not be read from Credential Manager, so its generic channel is off until the key is made again."); + } + } + } + return new NotificationRoute( row.RouteId, row.MetricMatch, teams, slack, generic, pagerDuty, row.SmtpRecipients, row.Enabled) { @@ -583,10 +618,132 @@ string Read(int index) AgentMaxPerHour = row.AgentMaxPerHour, AgentMaxPerDay = row.AgentMaxPerDay, AgentLeaseMinutes = row.AgentLeaseMinutes, - AgentExpiryMinutes = row.AgentExpiryMinutes + AgentExpiryMinutes = row.AgentExpiryMinutes, + GenericSigning = signing }; } + /// + /// Makes a signing key for a route's own generic URL, or rotates it (#6012, Darling's #5998 D2 and D3): the current key + /// becomes the previous one for . Returns the new key, to be shown once, or a + /// sentence saying why not (no such route, no saved generic URL, Credential Manager refused). The key goes only to Credential + /// Manager; the row keeps the flag and the rotation's end. A URL change keeps the key: Credential Manager does not bind a key + /// to the URL text as Darling's seal does. + /// + public async Task<(string? Key, string? Error)> GenerateSigningKeyAsync(int routeId) + { + string? key; + string? error; + using (var writeLock = _dbInitializer.AcquireWriteLock()) + { + PreservedTableRestore.ThrowIfRestorePending(_dbInitializer.ArchivePath, TableName); + using var connection = _dbInitializer.CreateConnection(); + await connection.OpenAsync(); + var current = await ReadRowAsync(connection, routeId); + if (current is null) + { + return (null, "This route no longer exists."); + } + + if (!current.Set[Array.IndexOf(s_channels, NotificationRouter.GenericChannel)]) + { + return (null, "Save a generic webhook URL on this route first. A key signs the route's own generic URL."); + } + + /* The new key is returned only after Credential Manager and this row both took it; a flag write that fails puts + Credential Manager back and returns an error, never a key (#6012, round 2). */ + var result = LiteWebhookSigningKeys.Generate( + _secrets, LiteWebhookSigningKeys.RouteName(routeId, false), LiteWebhookSigningKeys.RouteName(routeId, true), + new LiteSigningState(current.SigningSet, current.SigningUntilUtc), DateTime.UtcNow, + next => TryWriteSigningFlag(connection, routeId, next)); + key = result.NewKey; + error = result.Error; + } + + NotificationRouteCache.Invalidate(); + return (key, error); + } + + /// + /// What the route's row says about its key and whether that key opens (#6012, round 2), for the route dialog: the saved flag, + /// the rotation's end, and when the flag is set and the current key will not open. + /// Independent of the destination URL: a generic channel that is off because its URL will not open, with no flag, is not an + /// unreadable key. Null when no such route exists. + /// + public async Task SigningStateAsync(int routeId) + { + using var readLock = _dbInitializer.AcquireReadLock(); + using var connection = _dbInitializer.CreateConnection(); + await connection.OpenAsync(); + var row = await ReadRowAsync(connection, routeId); + if (row is null) + { + return null; + } + + var state = new LiteSigningState(row.SigningSet, row.SigningUntilUtc); + var unreadable = false; + if (state.HasKey) + { + _ = LiteWebhookSigningKeys.Open( + _secrets, LiteWebhookSigningKeys.RouteName(routeId, false), LiteWebhookSigningKeys.RouteName(routeId, true), + state, DateTime.UtcNow, out unreadable); + } + + return new LiteRouteSigning(state, unreadable); + } + + /// Removes a route's signing keys; its generic URL is sent unsigned from then on. Null when done, else why not. + public async Task RemoveSigningKeyAsync(int routeId) + { + string? error; + using (var writeLock = _dbInitializer.AcquireWriteLock()) + { + PreservedTableRestore.ThrowIfRestorePending(_dbInitializer.ArchivePath, TableName); + using var connection = _dbInitializer.CreateConnection(); + await connection.OpenAsync(); + var current = await ReadRowAsync(connection, routeId); + if (current is null) + { + return "This route no longer exists."; + } + + var state = LiteWebhookSigningKeys.Remove( + _secrets, LiteWebhookSigningKeys.RouteName(routeId, false), LiteWebhookSigningKeys.RouteName(routeId, true), + new LiteSigningState(current.SigningSet, current.SigningUntilUtc), out error); + if (error is null && TryWriteSigningFlag(connection, routeId, state) is not null) + { + /* The keys are gone and the row still says one is saved: the route reads as unreadable and its generic channel is + held back (fail closed) until Remove is tried again. */ + error = LiteWebhookSigningKeys.RemoveNotSavedText; + } + } + + NotificationRouteCache.Invalidate(); + return error; + } + + /// Writes the flag and the rotation's end to the route's row. Null when it worked, else a short reason (no key, no + /// value, no exception text beyond its type's message about the statement). + private static string? TryWriteSigningFlag(DuckDBConnection connection, int routeId, LiteSigningState state) + { + try + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $"UPDATE {TableName} SET generic_signing_set = $2, generic_signing_until = $3 WHERE route_id = $1"; + cmd.Parameters.Add(new DuckDBParameter { Value = routeId }); + cmd.Parameters.Add(new DuckDBParameter { Value = state.HasKey }); + cmd.Parameters.Add(new DuckDBParameter { Value = (object?)state.PreviousUntilUtc ?? DBNull.Value }); + cmd.ExecuteNonQuery(); + return null; + } + catch (Exception ex) + { + AppLogger.Error("NotificationRoutes", $"Notification route {routeId}: the signing flag could not be saved ({ex.GetType().Name})."); + return "the route could not be updated (" + ex.GetType().Name + ")"; + } + } + private static NotificationRoute Validate(NotificationRoute route) { ArgumentNullException.ThrowIfNull(route); diff --git a/Lite/Services/LiteWebhookSigningKeys.cs b/Lite/Services/LiteWebhookSigningKeys.cs new file mode 100644 index 0000000000..d8f65a5c86 --- /dev/null +++ b/Lite/Services/LiteWebhookSigningKeys.cs @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor Lite. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Globalization; +using PerformanceMonitor.Notifications; + +namespace PerformanceMonitorLite.Services; + +/// +/// What Lite keeps about one generic webhook destination's signing key besides the key itself (#6012): whether a key is +/// saved and when the previous key stops signing. Both are plain facts: the settings row keeps them in settings.json, a +/// route in its table row. The keys are in Windows Credential Manager. +/// +public readonly record struct LiteSigningState(bool HasKey, DateTime? PreviousUntilUtc) +{ + /// No key. + public static readonly LiteSigningState None = new(false, null); +} + +/// What the route row says about a key and whether it opens: for the route dialog, which cannot learn it from the loaded +/// route (a key that will not open turns the generic channel off and loads the route with no key). +public readonly record struct LiteRouteSigning(LiteSigningState State, bool Unreadable); + +/// What a Generate or Rotate gave: the new key (shown once, never stored in a place that shows it back), the state to save, +/// or the sentence for why nothing changed. The error never contains a key. +public sealed record LiteSigningKeyResult(string? NewKey, LiteSigningState State, string? Error) +{ + /// Names no key. + public override string ToString() => "LiteSigningKeyResult { Generated = " + (NewKey is not null) + ", Error = " + (Error ?? "none") + " }"; +} + +/// +/// The signing keys of Lite's generic webhooks (#6012, Darling's #5998 ported). Lite is the sender, so it holds the keys: +/// the settings row's generic URL has one key and each route with its own generic URL has one (decision D2). A key lives in +/// Credential Manager (through , as Lite's bearer token headers and route destinations do), +/// the current key and, for the 7 days of after a rotation, the previous one +/// (D3). A saved flag with a key that will not open is reported as unreadable, and the caller turns the destination off, so a +/// signed destination is never sent unsigned. Nothing here logs, returns in an error, or prints a key. +/// +public static class LiteWebhookSigningKeys +{ + /// What the key block says next to a new key's box (Darling's words). + public const string ShownOnceText = "Copy this key now and give it to the receiver. It will not be shown again."; + + /// What a destination without a key says. + public const string NoKeyText = "No key. Posts to this URL are not signed."; + + /// What a route's key block says before the route is saved. + public const string SaveRouteFirstText = "Save the route with a generic webhook URL first, then open it again to make a key."; + + /// What a test or a send says when a saved key will not open. + public const string UnreadableText = "Not sent: a generic webhook signing key could not be opened, and a signed destination is never sent unsigned."; + + /// What Remove says when the keys are gone but the flag could not be saved (#6012): the flag still says a key is saved, + /// so the destination reads as unreadable and is held back, never sent unsigned. + public const string RemoveNotSavedText = "The keys were removed, but the settings could not be saved, so settings still say a key is saved. Posts to this URL are held back until you try Remove again."; + + /// The Credential Manager key of the settings row's generic signing key (current or previous). + public static string SettingsName(bool previous) => previous ? "GenericWebhookSigningKeyPrevious" : "GenericWebhookSigningKey"; + + /// Every route signing key starts with this. It is not NotificationRoute_, so the destination sweep never reads it. + public const string RouteKeyPrefix = "NotificationRouteSigning_"; + + /// The Credential Manager key of a route's generic signing key (current or previous). + public static string RouteName(int routeId, bool previous) => + string.Create(CultureInfo.InvariantCulture, $"{RouteKeyPrefix}{routeId}_{(previous ? "Previous" : "Current")}"); + + /// The line the key block shows: never the key, only that one is saved and when a rotation ends. + public static string StatusText(LiteSigningState state, DateTime nowUtc, bool unreadable = false) + { + if (!state.HasKey) + { + return NoKeyText; + } + + if (unreadable) + { + return "A key is saved but could not be opened, so posts to this URL are held back. Rotate it to replace it, or remove it."; + } + + return state.PreviousUntilUtc is { } until && until > nowUtc + ? "Saved. The previous key is also sent until " + until.ToUniversalTime().ToString("yyyy-MM-dd HH:mm 'UTC'", CultureInfo.InvariantCulture) + "." + : "Saved. Rotate to replace it, or remove it."; + } + + /// The button the key block offers: Generate when there is no key, Rotate when there is one. + public static string ActionText(LiteSigningState state) => state.HasKey ? "Rotate key" : "Generate key"; + + /// + /// Makes a new key and saves it as the current one; a current key already saved becomes the previous one for + /// . The previous key is written first and the new one read back, so a refused + /// write leaves the saved state as it was. The caller saves the returned . + /// + public static LiteSigningKeyResult Generate(RouteSecretAccess secrets, string currentName, string previousName, LiteSigningState state, DateTime nowUtc) + { + ArgumentNullException.ThrowIfNull(secrets); + var newKey = WebhookSigning.NewKey(); + DateTime? until = null; + try + { + if (state.HasKey) + { + var old = secrets.Read(currentName) ?? ""; + if (!string.IsNullOrWhiteSpace(old)) + { + if (secrets.Write(previousName, old) is { } previousFailure) + { + return new LiteSigningKeyResult(null, state, "The previous key could not be kept in Windows Credential Manager because " + previousFailure + ", so the key was not changed."); + } + + until = nowUtc + WebhookSigning.RotationOverlap; + } + } + + if (secrets.Write(currentName, newKey) is { } failure) + { + return new LiteSigningKeyResult(null, state, "The key could not be saved to Windows Credential Manager because " + failure + ", so the key was not changed."); + } + + if (!string.Equals(secrets.Read(currentName), newKey, StringComparison.Ordinal)) + { + return new LiteSigningKeyResult(null, state, "Windows Credential Manager did not keep the key, so the key was not changed."); + } + } + catch (Exception ex) + { + return new LiteSigningKeyResult(null, state, "The key could not be saved to Windows Credential Manager (" + ex.GetType().Name + "), so the key was not changed."); + } + + return new LiteSigningKeyResult(newKey, new LiteSigningState(true, until), null); + } + + /// + /// plus the save of the flag's home + /// (#6012, round 2): a new key is returned only after Credential Manager and both took it. + /// writes the new state (settings.json for the settings row, the route's row for a route) and + /// returns null when it worked, else why not. When it fails or throws, both Credential Manager names are put back as they + /// were before the call, no key is returned, and the error says so; if the put-back fails too, the error says that. + /// The error never contains a key. + /// + public static LiteSigningKeyResult Generate( + RouteSecretAccess secrets, string currentName, string previousName, LiteSigningState state, DateTime nowUtc, + Func saveState) + { + ArgumentNullException.ThrowIfNull(secrets); + ArgumentNullException.ThrowIfNull(saveState); + string oldCurrent; + string oldPrevious; + try + { + oldCurrent = secrets.Read(currentName) ?? ""; + oldPrevious = secrets.Read(previousName) ?? ""; + } + catch (Exception ex) + { + return new LiteSigningKeyResult(null, state, "The saved key could not be read from Windows Credential Manager (" + ex.GetType().Name + "), so the key was not changed."); + } + + var made = Generate(secrets, currentName, previousName, state, nowUtc); + if (made.Error is not null || made.NewKey is null) + { + /* Generate may have written the previous key before it stopped; put both back as they were. */ + if (made.Error is not null) + { + _ = Restore(secrets, currentName, previousName, oldCurrent, oldPrevious); + } + + return made; + } + + string? saveFailure; + try + { + saveFailure = saveState(made.State); + } + catch (Exception ex) + { + saveFailure = "it failed (" + ex.GetType().Name + ")"; + } + + if (saveFailure is null) + { + return made; + } + + var restoreFailure = Restore(secrets, currentName, previousName, oldCurrent, oldPrevious); + var message = "The new key was not kept: the signing setting could not be saved because " + saveFailure + "."; + message += restoreFailure is null + ? " Windows Credential Manager was put back as it was, so the key was not changed." + : " Windows Credential Manager could not be put back (" + restoreFailure + "), so it may hold a key that was never shown. Rotate the key again or remove it."; + return new LiteSigningKeyResult(null, state, message); + } + + /// Puts the two Credential Manager names back to what they held. A name that held nothing is deleted. Null when both + /// are as they were, else a short reason. Never contains a key. + private static string? Restore(RouteSecretAccess secrets, string currentName, string previousName, string oldCurrent, string oldPrevious) + { + try + { + var ok = RestoreOne(secrets, currentName, oldCurrent); + ok &= RestoreOne(secrets, previousName, oldPrevious); + return ok ? null : "Windows Credential Manager refused the change"; + } + catch (Exception ex) + { + return ex.GetType().Name; + } + } + + private static bool RestoreOne(RouteSecretAccess secrets, string name, string old) + { + if (string.Equals(secrets.Read(name) ?? "", old, StringComparison.Ordinal)) + { + return true; + } + + return old.Length == 0 ? secrets.Delete(name) : secrets.Write(name, old) is null; + } + + /// Removes both keys. Returns the new state (none), or the same state and an error when a key would not go. + public static LiteSigningState Remove(RouteSecretAccess secrets, string currentName, string previousName, LiteSigningState state, out string? error) + { + ArgumentNullException.ThrowIfNull(secrets); + error = null; + try + { + if (!secrets.Delete(currentName) || !secrets.Delete(previousName)) + { + error = "Windows Credential Manager would not remove the key, so it is still saved."; + return state; + } + } + catch (Exception ex) + { + error = "Windows Credential Manager would not remove the key (" + ex.GetType().Name + "), so it is still saved."; + return state; + } + + return LiteSigningState.None; + } + + /// + /// Opens the keys of a destination. No saved flag is , unsigned. A saved flag whose + /// current key will not open sets and gives none: the caller turns the destination off. The + /// previous key is read only while its window is open, and one that will not open is left out (the current key still signs). + /// + public static WebhookSigningKeys Open(RouteSecretAccess secrets, string currentName, string previousName, LiteSigningState state, DateTime nowUtc, out bool unreadable) + { + unreadable = false; + if (!state.HasKey) + { + return WebhookSigningKeys.None; + } + + string key; + try + { + key = secrets.Read(currentName) ?? ""; + } + catch (Exception) + { + key = ""; + } + + if (!WebhookSigning.TryDecodeKey(key, out _)) + { + unreadable = true; + return WebhookSigningKeys.None; + } + + var previous = ""; + if (state.PreviousUntilUtc is { } until && until > nowUtc) + { + try + { + previous = secrets.Read(previousName) ?? ""; + } + catch (Exception) + { + previous = ""; + } + + if (!WebhookSigning.TryDecodeKey(previous, out _)) + { + previous = ""; + } + } + + return new WebhookSigningKeys(key.Trim(), previous.Trim(), previous.Length == 0 ? null : state.PreviousUntilUtc); + } +} diff --git a/Lite/Windows/NotificationRouteEditDialog.xaml b/Lite/Windows/NotificationRouteEditDialog.xaml index 49241f79e9..e94acf1aac 100644 --- a/Lite/Windows/NotificationRouteEditDialog.xaml +++ b/Lite/Windows/NotificationRouteEditDialog.xaml @@ -66,6 +66,7 @@ + + + + + + + +