diff --git a/Darling/Darling.Tests/AlertHistoryRangePageTests.cs b/Darling/Darling.Tests/AlertHistoryRangePageTests.cs index b056b45b4..2417c02e8 100644 --- a/Darling/Darling.Tests/AlertHistoryRangePageTests.cs +++ b/Darling/Darling.Tests/AlertHistoryRangePageTests.cs @@ -193,6 +193,27 @@ public void TheInvestigationsPanel_ReadsWhenOpened_ShowsTheNewestVerdict_AndTheJ Assert.Equal(0, r.GetProperty("writes").GetArrayLength()); } + /// #5834: the Investigations panel says when the list, the reports read or a request's journal held more + /// than the page it returned, and says nothing when none did. + [Fact] + public void TheInvestigationsPanel_SaysWhenAReadWasCut_AndNothingWhenNoneWas() + { + var r = Run("investigationsTruncated"); + const string ListCut = "Showing the newest 2 requests; older ones are not listed."; + const string ReportsCut = "held more requests than one reports read returns"; + const string JournalCut = "Showing the newest 2 journal rows; older ones are not listed."; + var clear = r.GetProperty("clear"); + var cut = r.GetProperty("cut"); + foreach (var text in new[] { ListCut, ReportsCut }) + { + Assert.DoesNotContain(text, clear.GetProperty("panel").GetString()!, StringComparison.Ordinal); + Assert.Contains(text, cut.GetProperty("panel").GetString()!, StringComparison.Ordinal); + } + + Assert.DoesNotContain(JournalCut, clear.GetProperty("journal").GetString()!, StringComparison.Ordinal); + Assert.Contains(JournalCut, cut.GetProperty("journal").GetString()!, StringComparison.Ordinal); + } + [Fact] public void ATruncatedReplySaysMoreAlertsExistAndAClearOneDoesNot() { diff --git a/Darling/Darling.Tests/DarlingMcpInvestigationToolsTests.cs b/Darling/Darling.Tests/DarlingMcpInvestigationToolsTests.cs index 9d7c41d52..81695427d 100644 --- a/Darling/Darling.Tests/DarlingMcpInvestigationToolsTests.cs +++ b/Darling/Darling.Tests/DarlingMcpInvestigationToolsTests.cs @@ -276,6 +276,22 @@ [new InvestigationReportRow(request, 1, "agent-a", at, "real", Canaries("root_ca brief); } + /* #5834: the answer names its page count requests_returned and carries the read's two truncation flags. */ + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + public void ReportsAnswer_SaysWhenTheWindowOrTheJournalHeldMore(bool requestsTruncated, bool journalTruncated) + { + var read = StoredRead() with { RequestsTruncated = requestsTruncated, JournalTruncated = journalTruncated }; + var root = JsonNode.Parse(DarlingMcpInvestigationTools.ReportsAnswer(read, 24, McpDataDetail.Full))!; + + Assert.Equal(read.Requests.Count, (int)root["requests_returned"]!); + Assert.Equal(requestsTruncated, (bool)root["truncated"]!); + Assert.Equal(journalTruncated, (bool)root["journal_truncated"]!); + Assert.Null(root["request_count"]); + } + [Fact] public void ReportsAnswer_MaskLiterals_ReplacesEveryFieldsLiteral_KeepsTheStructuredFields() { diff --git a/Darling/Darling.Tests/InvestigationBriefComposerTests.cs b/Darling/Darling.Tests/InvestigationBriefComposerTests.cs index adfadd7a8..d9fdabf37 100644 --- a/Darling/Darling.Tests/InvestigationBriefComposerTests.cs +++ b/Darling/Darling.Tests/InvestigationBriefComposerTests.cs @@ -248,6 +248,29 @@ public void Budget_CutsEachSectionFromItsTail_AndNamesTheCut() Assert.True((int)brief["truncated"]!["chars"]! > 0); } + /* #5834: a field whose cut index lands between the halves of an emoji's surrogate pair. The raw UTF-16 cut kept the + lone high surrogate, which serializes as U+FFFD; the text-element cut drops the emoji whole, and an emoji that ends + at the cut is kept whole. truncated.chars counts every unit not kept. */ + [Theory] + [InlineData(1, false)] // the emoji straddles the cut index: dropped whole + [InlineData(2, true)] // the emoji ends exactly at the cut index: kept whole + public void FieldCut_NeverSplitsASurrogatePair(int emojiStartBeforeCut, bool emojiKept) + { + const string Emoji = "\U0001F600"; + var keepAt = InvestigationBriefComposer.FieldCharCap - InvestigationBriefComposer.CutMarker.Length; + var head = new string('a', keepAt - emojiStartBeforeCut); + var text = head + Emoji + new string('z', 500); + var json = InvestigationBriefComposer.Compose( + Sources(notes: new[] { new InvestigationBriefNote(LoginCanary, "dba-lead", Now.AddDays(30), text) }), McpDataDetail.Full, Now); + + var brief = JsonNode.Parse(json)!.AsObject(); + var cut = (string)brief["notes"]![0]!["text"]!; + Assert.DoesNotContain('�', cut); + Assert.Equal((emojiKept ? head + Emoji : head) + InvestigationBriefComposer.CutMarker, cut); + Assert.Equal(1, (int)brief["truncated"]!["fields_cut"]!); + Assert.Equal(text.Length - (cut.Length - InvestigationBriefComposer.CutMarker.Length), (int)brief["truncated"]!["chars"]!); + } + /* ---- masking under all three levels ---- */ [Fact] diff --git a/Darling/Darling.Tests/InvestigationToolsLiveTests.cs b/Darling/Darling.Tests/InvestigationToolsLiveTests.cs index 5e97eea5b..aadc3cd7d 100644 --- a/Darling/Darling.Tests/InvestigationToolsLiveTests.cs +++ b/Darling/Darling.Tests/InvestigationToolsLiveTests.cs @@ -218,6 +218,58 @@ await RunAsync(async (store, dataSource, connection, ct) => }); } + /* #5834: both reads are pages, and each says when the window held more than the page (the #3699 shape). At the bound: + 101 requests answer 100 returned and truncated; 100 answer 100 and not truncated, on the reports read and the list + alike; and 201 journal rows on the page's requests answer 200 and journal_truncated, 200 answer 200 and not. */ + [Fact] + public async Task ReportsAndList_SayWhenTheWindowHeldMoreThanThePage() + { + await RunAsync(async (store, dataSource, connection, ct) => + { + await ExecAsync(connection, @" +INSERT INTO config.investigation_requests + (request_id, route_id, server_id, server_name, family, first_metric, first_alert_at, last_alert_at, window_until, + opened_at, expires_at) +SELECT gen_random_uuid(), 1, 7, 'srv-7', 'blocking', 'Blocking', now() - make_interval(secs => i), now() - make_interval(secs => i), + now(), now() - make_interval(secs => i), now() + interval '4 hours' +FROM generate_series(1, " + (DarlingMcpInvestigationTools.MaxLimit + 1) + ") AS i", ct); + var newest = await ScalarAsync(connection, + "SELECT request_id FROM config.investigation_requests ORDER BY opened_at DESC LIMIT 1", ct); + + async Task<(JsonElement Reports, JsonElement List)> ReadAsync() => ( + Root(await DarlingMcpInvestigationTools.ReportsAsync(dataSource, McpDataDetail.Full, null, null, 24, ct)), + Root(await DarlingMcpInvestigationTools.ListInvestigationRequests(dataSource, limit: DarlingMcpInvestigationTools.MaxLimit, cancellationToken: ct))); + + async Task JournalRowsAsync(int count) => await ExecAsync(connection, + "INSERT INTO config.investigation_journal (event_time, request_id, server_id, event, actor) " + + $"SELECT now() - make_interval(secs => i), '{newest}', 7, 'opened', 'service' FROM generate_series(1, {count}) AS i", ct); + + var (reports, list) = await ReadAsync(); + Assert.Equal(DarlingMcpInvestigationTools.MaxLimit, reports.GetProperty("requests_returned").GetInt32()); + Assert.Equal(DarlingMcpInvestigationTools.MaxLimit, reports.GetProperty("requests").GetArrayLength()); + Assert.True(reports.GetProperty("truncated").GetBoolean()); + Assert.Equal(DarlingMcpInvestigationTools.MaxLimit, list.GetProperty("requests_returned").GetInt32()); + Assert.Equal(DarlingMcpInvestigationTools.MaxLimit, list.GetProperty("requests").GetArrayLength()); + Assert.True(list.GetProperty("truncated").GetBoolean()); + + await ExecAsync(connection, + "DELETE FROM config.investigation_requests WHERE request_id = (SELECT request_id FROM config.investigation_requests ORDER BY opened_at LIMIT 1)", ct); + await JournalRowsAsync(DarlingMcpInvestigationTools.JournalLimit); + (reports, list) = await ReadAsync(); + Assert.Equal(DarlingMcpInvestigationTools.MaxLimit, reports.GetProperty("requests_returned").GetInt32()); + Assert.False(reports.GetProperty("truncated").GetBoolean()); + Assert.Equal(DarlingMcpInvestigationTools.JournalLimit, reports.GetProperty("journal").GetArrayLength()); + Assert.False(reports.GetProperty("journal_truncated").GetBoolean()); + Assert.Equal(DarlingMcpInvestigationTools.MaxLimit, list.GetProperty("requests_returned").GetInt32()); + Assert.False(list.GetProperty("truncated").GetBoolean()); + + await JournalRowsAsync(1); + (reports, _) = await ReadAsync(); + Assert.Equal(DarlingMcpInvestigationTools.JournalLimit, reports.GetProperty("journal").GetArrayLength()); + Assert.True(reports.GetProperty("journal_truncated").GetBoolean()); + }); + } + private static async Task ExecAsync(NpgsqlConnection connection, string sql, CancellationToken ct) { await using var command = new NpgsqlCommand(sql, connection); diff --git a/Darling/Darling.Tests/alert-history-range-harness.mjs b/Darling/Darling.Tests/alert-history-range-harness.mjs index c19a53915..2393abca1 100644 --- a/Darling/Darling.Tests/alert-history-range-harness.mjs +++ b/Darling/Darling.Tests/alert-history-range-harness.mjs @@ -69,11 +69,11 @@ let investigationsReply = { status: "ok", requests: [ { request_id: REQ, opened_at: "2026-10-14T12:00:00.0000000", server_name: "srv-a", family: "blocking", state: "reported", alert_count: 3 }, { request_id: "22222222-2222-2222-2222-222222222222", opened_at: "2026-10-14T11:00:00.0000000", server_name: "srv-b", family: "resources", state: "open", alert_count: 1 }, ] }; -const reportsReply = { status: "ok", reports: [ +let reportsReply = { status: "ok", reports: [ { request_id: REQ, submitted_at: "2026-10-14T12:05:00", verdict: "false_positive", summary: "older report" }, { request_id: REQ, submitted_at: "2026-10-14T12:10:00", verdict: "real", summary: "a long blocking chain from the nightly ETL" }, ] }; -const journalReply = { status: "ok", journal: [ +let journalReply = { status: "ok", journal: [ { request_id: REQ, event_time: "2026-10-14T12:00:00", event: "opened", actor: "service" }, { request_id: REQ, event_time: "2026-10-14T12:10:00", event: "reported", actor: "agent:agent-a" }, ] }; @@ -243,6 +243,23 @@ try { panel.open = false; urls.length = 0; await renderAlerts(main); out.closedPollReads = invReads().length; out.writes = urls.filter((u) => !u.startsWith("/api/read") && !u.startsWith("/api/session") && !u.includes("/api/catalog")); }, + /* #5834: each read is a page; the panel says when the list, the reports read or a journal held more. */ + async investigationsTruncated() { + alertsReply = { alerts: [row(1)], truncated: false }; + const openPanel = async (truncated) => { + investigationsReply = { ...investigationsReply, truncated }; + reportsReply = { ...reportsReply, truncated }; + journalReply = { ...journalReply, journal_truncated: truncated }; + const main = newMain(); await renderAlerts(main); + const panel = main.all("details").find((d) => d.textContent.startsWith("Investigations")); + panel.open = true; panel.fire("toggle"); await settle(); + const journal = panel.all("table")[0].children[1].children[0].all("details")[0]; + journal.open = true; journal.fire("toggle"); await settle(); + return { panel: panel.textContent, journal: journal.textContent }; + }; + out.clear = await openPanel(false); + out.cut = await openPanel(true); + }, async mute() { alertsReply = { alerts: [row(1)], truncated: false }; const main = newMain(); await renderAlerts(main); diff --git a/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationBriefComposer.cs b/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationBriefComposer.cs index df6ca832f..85810f3b1 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationBriefComposer.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationBriefComposer.cs @@ -410,14 +410,19 @@ private sealed class CutCounter { public int Fields { get; private set; } + /// The characters the cuts omitted, in the unit the caps count: UTF-16 units of the value. public int Chars { get; private set; } + /* #5834: the cut lands on a text-element boundary (the #3625 rule), never between the halves of a surrogate pair: + a lone high surrogate serializes as U+FFFD, so the agent would read a replacement glyph where an emoji was. An + element that straddles the cut is dropped whole, and the omission counts every unit not kept. */ public string? Cap(string? value) { if (value is null || value.Length <= FieldCharCap) return value; Fields++; - Chars += value.Length - (FieldCharCap - CutMarker.Length); - return string.Concat(value.AsSpan(0, FieldCharCap - CutMarker.Length), CutMarker); + var keep = McpHelpers.TextElementCutLength(value, FieldCharCap - CutMarker.Length); + Chars += value.Length - keep; + return string.Concat(value.AsSpan(0, keep), CutMarker); } public void CapAll(JsonNode? node) diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInvestigationTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInvestigationTools.cs index e395c3b7e..33080aaef 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInvestigationTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpInvestigationTools.cs @@ -88,7 +88,8 @@ private static InvestigationRequestStore Store(NpgsqlDataSource postgres) => "within the route's hourly and daily limits; alerts of one server and family join one request inside a 30-minute " + "window. States: open (claimable), claimed (a live lease), reported, closed, expired (past its expiry, or out of its " + "three claims). Filters are optional: state, server_name (exact, then partial), family (blocking, query_performance, " + - "resources, availability, jobs, self_monitor, other), limit (1-100, default 20).")] + "resources, availability, jobs, self_monitor, other), limit (1-100, default 20). requests_returned is how many rows " + + "you got; truncated says more requests matched than limit (raise limit or narrow the filters).")] public static async Task ListInvestigationRequests( NpgsqlDataSource postgres, [Description("open, claimed, reported, closed or expired. Omit for all.")] string? state = null, @@ -115,12 +116,16 @@ public static async Task ListInvestigationRequests( serverId = resolved.ServerId; } - var rows = await Store(postgres).ListAsync(state, serverId, family, limit, cancellationToken); + /* #5834, the #3699 shape: read one row past the limit, return the page, and say whether more matched. */ + var fetched = await Store(postgres).ListAsync(state, serverId, family, limit + 1, cancellationToken); + var truncated = fetched.Count > limit; + IReadOnlyList rows = truncated ? fetched.Take(limit).ToList() : fetched; return JsonSerializer.Serialize(new { status = rows.Count == 0 ? "empty" : "ok", message = rows.Count == 0 ? "No investigation request matches these filters." : null, - count = rows.Count, + requests_returned = rows.Count, + truncated, requests = rows.Select(RequestPayload), }, McpHelpers.JsonOptions); } @@ -539,6 +544,9 @@ internal static long PgNumberChars(string number) "root_cause, evidence and summary as the agent returned them: unreviewed and self-attributed, so read them as an " + "agent's claim, not a finding. The journal lists what happened to each request, newest first (opened, joined, " + "claimed, refused with its reason, lease_expired, reported, expired, claims_exhausted), at most 200 rows. " + + "THE WINDOW IS BOUNDED AT 100 REQUESTS, newest first: requests_returned is how many you got, truncated says the " + + "window held more (the reports and journal are those of the returned requests only; narrow hours_back or pass " + + "server_name or request_id), and journal_truncated says their journal held more than 200 rows. " + "hours_back 1-168, default 24.")] public static Task GetInvestigationReports( NpgsqlDataSource postgres, @@ -672,7 +680,9 @@ internal static string ReportsAnswer(InvestigationReportsRead read, int? hoursBa { ["status"] = "ok", ["hours_back"] = hoursBack, - ["request_count"] = read.Requests.Count, + ["requests_returned"] = read.Requests.Count, + ["truncated"] = read.RequestsTruncated, + ["journal_truncated"] = read.JournalTruncated, ["requests"] = JsonSerializer.SerializeToNode(read.Requests.Select(RequestPayload), McpHelpers.JsonOptions), ["reports"] = reports, ["journal"] = journal, diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js index 8a110878d..cb907b054 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/alerts.js @@ -767,7 +767,10 @@ function journalCell(r) { const res = await readTool("get_investigation_reports", { request_id: r.request_id, hours_back: INVESTIGATION_HOURS }); if (res.kind === "error") { built = false; mount(body, errorStrip(res.message)); return; } const rows = res.kind === "empty" ? [] : rowsUnder(res.data, "journal").filter((j) => j && String(j.request_id ?? r.request_id) === String(r.request_id)); - mount(body, rows.length ? rows.map((j) => el("div", { text: journalLine(j) })) : [emptyStrip("No journal rows for this request.")]); + const lines = rows.length ? rows.map((j) => el("div", { text: journalLine(j) })) : [emptyStrip("No journal rows for this request.")]; + /* #5834: the journal is a page of the newest rows; say so when the request held more. */ + if (res.data && res.data.journal_truncated) lines.unshift(noticeStrip(`Showing the newest ${rows.length} journal rows; older ones are not listed.`)); + mount(body, lines); }); return node; } @@ -799,6 +802,11 @@ async function refreshInvestigations(state) { if (!rows.length) { mount(state.box, emptyStrip("No investigation requests.")); return; } const parts = [VIZ.table({ requests: rows }, { rowsKey: "requests", columns: INVESTIGATION_COLUMNS })]; if (reports.kind === "error") parts.unshift(noticeStrip("The reports could not be read, so no verdicts are shown: " + reports.message)); + /* #5834: both reads are pages of the newest requests; say when either one held more than it returned. */ + if (list.data && list.data.truncated) parts.unshift(noticeStrip(`Showing the newest ${rows.length} requests; older ones are not listed.`)); + if (reports.kind !== "error" && reports.data && reports.data.truncated) { + parts.unshift(noticeStrip(`The last ${INVESTIGATION_HOURS} hours held more requests than one reports read returns, so older requests may show no verdict.`)); + } mount(state.box, parts); } diff --git a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs index c56d7f6b5..9cd01da3c 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/InvestigationRequestStore.cs @@ -147,12 +147,17 @@ public sealed record InvestigationJournalRow( string? Reason, string? DetailJson); -/// What read; only for one request. +/// What read; only for one request. +/// says the window held more requests than the page (the reports and the journal are +/// those of the page's requests only), and that those requests held more journal rows +/// than the journal page (#5834). public sealed record InvestigationReportsRead( IReadOnlyList Requests, IReadOnlyList Reports, IReadOnlyList Journal, - string? Brief); + string? Brief, + bool RequestsTruncated = false, + bool JournalTruncated = false); /// One request as a list reads it, with the state computed from the lease and the expiry, not the stored word. public sealed record InvestigationRequestRow( @@ -826,7 +831,9 @@ await JournalAsync(connection, transaction, now, serverId is null ? null : reque /// /// Reads requests with their reports and journal (#5632 Phase A, get_investigation_reports): one request by id /// (then its stored brief too), or the requests opened since , optionally on one server. The - /// state is computed as computes it. Writes nothing. + /// state is computed as computes it. Writes nothing. Each page is read one row past its limit + /// and trimmed, so the read says when the window held more requests, or those requests more journal rows, than it + /// returns (#5834, the #3699 shape); the reports and the journal are read for the returned requests only. /// public async Task ReadReportsAsync(Guid? requestId, int? serverId, DateTime sinceUtc, int limit, int journalLimit, CancellationToken cancellationToken = default) @@ -838,7 +845,7 @@ public async Task ReadReportsAsync(Guid? requestId, in string? brief = null; await using (var command = Command(connection, null, ReportsRequestsSql, P(Now()), new NpgsqlParameter { TypedValue = requestId, NpgsqlDbType = NpgsqlDbType.Uuid }, - new NpgsqlParameter { TypedValue = serverId, NpgsqlDbType = NpgsqlDbType.Integer }, P(sinceUtc), P(limit))) + new NpgsqlParameter { TypedValue = serverId, NpgsqlDbType = NpgsqlDbType.Integer }, P(sinceUtc), P(limit + 1))) await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) @@ -858,12 +865,18 @@ public async Task ReadReportsAsync(Guid? requestId, in } } + var requestsTruncated = requests.Count > limit; + if (requestsTruncated) + { + requests.RemoveRange(limit, requests.Count - limit); + } + var ids = requests.Select(r => r.RequestId).ToArray(); var reports = new List(); var journal = new List(); if (ids.Length == 0) { - return new InvestigationReportsRead(requests, reports, journal, brief); + return new InvestigationReportsRead(requests, reports, journal, brief, requestsTruncated); } await using (var command = Command(connection, null, ReportsSql, P(ids))) @@ -877,7 +890,7 @@ public async Task ReadReportsAsync(Guid? requestId, in } } - await using (var command = Command(connection, null, ReportsJournalSql, P(ids), P(journalLimit))) + await using (var command = Command(connection, null, ReportsJournalSql, P(ids), P(journalLimit + 1))) await using (var reader = await command.ExecuteReaderAsync(cancellationToken)) { while (await reader.ReadAsync(cancellationToken)) @@ -889,7 +902,13 @@ public async Task ReadReportsAsync(Guid? requestId, in } } - return new InvestigationReportsRead(requests, reports, journal, brief); + var journalTruncated = journal.Count > journalLimit; + if (journalTruncated) + { + journal.RemoveRange(journalLimit, journal.Count - journalLimit); + } + + return new InvestigationReportsRead(requests, reports, journal, brief, requestsTruncated, journalTruncated); } ///