Skip to content

Job History: GROUP BY join replaces the per-job window function, tab off the 30s timer (#4229) #4256

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erikdarlingdata merged 3 commits into from
Sep 25, 2026
Merged

Job History: GROUP BY join replaces the per-job window function, tab off the 30s timer (#4229) #4256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cbbc75c
Job History: replace the per-job window function with a GROUP BY join…
erikdarlingdata Sep 25, 2026
668da86
Merge remote-tracking branch 'origin/dev' into fix/4229-job-history
erikdarlingdata Sep 25, 2026
753adee
Lite parity: Job History group-by/join now has a DuckDB regression te…
erikdarlingdata Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions Darling/Darling.Tests/EventWindowedReadsAreBoundedLivePostgresTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,13 @@ await AssertChunksAsync(connection,
await AssertChunksAsync(connection,
OldJobHistorySql, new NpgsqlParameter[] { P(marks.WindowStart), P(2000) },
ViewerDataService.BuildJobHistorySql(scopedToServer: false), new NpgsqlParameter[] { P(marks.WindowStart), P(floor), P(2000) },
ct, "BuildJobHistorySql(false)", maxChunks: 3);
/* minReduction: 2, not the default 3 — #4229's own GROUP BY fix split the single scan
OldJobHistorySql has into two (job_stats + base), so "new" now scans job_history TWICE,
each independently floor-bounded. The per-scan bound still holds (each touches at most the
window's own chunk plus the floor's one extra day), but the STATEMENT-level chunk count
roughly doubles against a genuinely single-scan oracle, unlike BlockingTrendSql/
BlockingDurationStatsSql above, whose OLD oracle is itself already two-scan shaped. */
ct, "BuildJobHistorySql(false)", maxChunks: 3, minReduction: 2);

bodySucceeded = true;
}
Expand Down Expand Up @@ -439,16 +445,16 @@ private static async Task AssertEqualAsync(
/// </summary>
private static async Task AssertChunksAsync(
NpgsqlConnection connection, string oldSql, NpgsqlParameter[] oldArgs, string newSql, NpgsqlParameter[] newArgs,
CancellationToken ct, string label, int maxChunks)
CancellationToken ct, string label, int maxChunks, int minReduction = 3)
{
_ = maxChunks;
var oldPlan = await ExplainAsync(connection, "EXPLAIN (COSTS OFF) " + oldSql, oldArgs, ct);
var newPlan = await ExplainAsync(connection, "EXPLAIN (COSTS OFF) " + newSql, newArgs, ct);
var oldChunks = PlanChunkScans.Count(oldPlan);
var newChunks = PlanChunkScans.Count(newPlan);

Assert.True(oldChunks - newChunks >= 3,
$"{label}: expected the floor to exclude at least 3 of the 4 seeded old-day chunks (old={oldChunks}, new={newChunks}):\nOLD:\n{oldPlan}\nNEW:\n{newPlan}");
Assert.True(oldChunks - newChunks >= minReduction,
$"{label}: expected the floor to exclude at least {minReduction} of the 4 seeded old-day chunks (old={oldChunks}, new={newChunks}):\nOLD:\n{oldPlan}\nNEW:\n{newPlan}");
}

/* ─────────────────────────── seeding ─────────────────────────── */
Expand Down
8 changes: 6 additions & 2 deletions Darling/Darling.Tests/EventWindowedReadsCarryTheFloorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,12 @@ public sealed class EventWindowedReadsCarryTheFloorTests
{ nameof(DarlingDataReader) + "." + nameof(DarlingDataReader.BlockingDurationStatsSql), DarlingDataReader.BlockingDurationStatsSql, 2 },
{ nameof(ViewerDataService) + "." + nameof(ViewerDataService.BlockingPairRowsSql), ViewerDataService.BlockingPairRowsSql, 1 },
{ nameof(ViewerDataService) + "." + nameof(ViewerDataService.MemoryPressureEventsSql), ViewerDataService.MemoryPressureEventsSql, 1 },
{ nameof(ViewerDataService) + ".BuildJobHistorySql(false)", ViewerDataService.BuildJobHistorySql(scopedToServer: false), 1 },
{ nameof(ViewerDataService) + ".BuildJobHistorySql(true)", ViewerDataService.BuildJobHistorySql(scopedToServer: true), 1 },
/* 2, not 1: #4229's own fix split the single job_history scan into two — job_stats (the per-job
GROUP BY) and base (the newest-2000 selection) — so this is the "two-CTE reads have TWO scans"
case the class summary calls out, and a floor on only one would leave the other opening every
chunk again. */
{ nameof(ViewerDataService) + ".BuildJobHistorySql(false)", ViewerDataService.BuildJobHistorySql(scopedToServer: false), 2 },
{ nameof(ViewerDataService) + ".BuildJobHistorySql(true)", ViewerDataService.BuildJobHistorySql(scopedToServer: true), 2 },
};

[Theory]
Expand Down
Loading
Loading