diff --git a/Darling/Darling.Tests/ScratchPostgres.cs b/Darling/Darling.Tests/ScratchPostgres.cs index 251e109a6a..f56e7332fa 100644 --- a/Darling/Darling.Tests/ScratchPostgres.cs +++ b/Darling/Darling.Tests/ScratchPostgres.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Npgsql; @@ -169,14 +169,16 @@ internal static async Task DropRememberedAsync(params string[] databaseNames) /// PostgreSQL postmaster log "terminated by exception 0xC0000005" for a TimescaleDB worker, twice, both times /// with a scratch-database FORCE drop in flight (PR #5480); no worker is running in the dropped database /// once this returns, so the drop has nothing to kill. - /// Best-effort and bounded: it never throws, never fails a test, and never skips the drop. A database + /// Best-effort and bounded: it never throws, never fails a test, and never skips the drop. A caller's + /// ends it early too, without a message (the exit drain passes its deadline). A database /// without the extension, or one already gone, returns at once. It does NOT call /// _timescaledb_functions.stop_background_workers(), which SIGTERMs running jobs, the very kill this /// avoids. /// - internal static async Task QuiesceTimescaleJobsAsync(string adminConnectionString, string databaseName) + internal static async Task QuiesceTimescaleJobsAsync(string adminConnectionString, string databaseName, CancellationToken cancellationToken = default) { - using var cap = new CancellationTokenSource(QuiesceCap); + using var cap = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cap.CancelAfter(QuiesceCap); try { var scratchBuilder = new NpgsqlConnectionStringBuilder(adminConnectionString) @@ -221,6 +223,10 @@ starting up and not yet show in pg_stat_activity on the first look. */ { /* The database is already gone: nothing to quiesce, and the drop below is IF EXISTS. */ } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + /* The caller's own deadline (the exit drain's), not the cap: the caller decides whether the drop still runs. */ + } catch (Exception ex) { /* The cap, a missing database or a refused connect all land here. The drop that follows is the @@ -293,8 +299,11 @@ private static void DropEverythingRemembered() /// One database the exit drain was asked to drop: its name, the connection string that drops it, and the test that created it. internal readonly record struct RememberedDatabase(string Name, string AdminConnectionString, string Creator); - /// What an exit drain did: connections it tried to open, databases it dropped, drops that failed, and databases it left alone. - internal readonly record struct ExitDrainOutcome(int ConnectAttempts, int Dropped, int Failed, int Skipped); + /// + /// What an exit drain did: connections it tried to open, databases it dropped, drops that failed, databases it left + /// alone because their cluster did not answer, and databases it had not finished when its deadline passed. + /// + internal readonly record struct ExitDrainOutcome(int ConnectAttempts, int Dropped, int Failed, int Skipped, int Unfinished); /// /// How long the exit drain waits to connect to a cluster, in seconds. The default is 15, and the drain ran once per @@ -311,19 +320,6 @@ private static void DropEverythingRemembered() /// internal const int ExitDrainCommandTimeoutSeconds = 30; - /// - /// How long the exit drain keeps starting new drops, in seconds. The test runner (xunit.v3 under Microsoft.Testing.Platform) - /// gives the process 10 seconds after the run returns to exit, then prints "Foreground threads were left running, - /// forcing process exit" and exits with code 1, whatever the tests did. This drain runs at that exit, so it is the - /// one thing that can hold the process. Each drop here also unschedules the database's TimescaleDB jobs and waits - /// for their workers (#5480), about 0.4 seconds a database on a CI runner, so 29 databases that outlived their tests - /// took 11 seconds and failed the nightly with every test green. The drain stops starting drops after this long and - /// names the rest; a database left this way is removed by the next run's start-of-run sweep, or goes with the - /// throwaway cluster. Tests that mint a database must still drop it themselves (see - /// ScratchDatabaseDisposalCensusTests); this budget only keeps a leak from failing a run. - /// - internal const int ExitDrainBudgetSeconds = 6; - /// The admin connection string with the exit drain's own connect and command timeouts, and no pooling: one connection per drop. internal static string ExitDrainConnectionString(string adminConnectionString) => new NpgsqlConnectionStringBuilder(adminConnectionString) @@ -333,71 +329,200 @@ internal static string ExitDrainConnectionString(string adminConnectionString) = Pooling = false, }.ConnectionString; + /// + /// How long the whole exit drain may take, in seconds, in-flight drops included. The drain is a ProcessExit handler, + /// and xUnit v3 under Microsoft.Testing.Platform gives that stage 11 seconds after the run returns (one second, then + /// its shutdownForegroundThreadWaitSeconds default of 10) before it prints "[FATAL ERROR] Foreground threads + /// were left running, forcing process exit" and exits with code 1. No thread was left running: the 2026-10-08 + /// nightly passed every test and still failed because this drain, then dropping 39 leaked databases one at a time + /// with a TimescaleDB quiesce before each (#5480), was still at work when that window closed. The drain returns when + /// this passes even with a drop in flight (one DROP DATABASE waited 26 seconds for its checkpoint on a local rig), and + /// names every database it did not finish; the next run's start-of-run sweep removes those, or they go with the + /// throwaway cluster. Tests that mint a database must still drop it themselves (see + /// ScratchDatabaseDisposalCensusTests); this budget only keeps a leak from failing a run. + /// + internal const int ExitDrainBudgetSeconds = 6; + + /// How many databases the exit drain quiesces and drops at once. Each holds at most two connections at a time. + internal const int ExitDrainParallelism = 16; + /// /// The exit drain's body, taking its list as a parameter so a test can run it without a process exit and without /// touching the databases that other tests, running in parallel, still hold. /// /// - /// Connects with the short connect wait above, and stops trying a cluster once a connect to it fails without the - /// server answering (a stopped cluster, a refused or timed-out connection), so N databases on a dead cluster cost - /// one short wait, not N. A server that answers with an error (a bad password, a missing database) is not a dead - /// cluster, and a failed drop on a live cluster never stops the others; each skipped database is still named. It - /// stops starting drops once (default seconds) has - /// passed, counting the rest as skipped. + /// Clusters drain side by side, and within a live cluster up to databases + /// drop at once, all under one (default seconds). + /// When it passes, the drain returns and names every + /// database it did not finish. A drop already sent is not cancelled (an interrupted DROP DATABASE can leave an + /// invalid database behind), but no new connect, quiesce or drop starts. + /// The first database on each cluster goes alone and probes it. A connect that fails without the server + /// answering (a stopped cluster, a refused or timed-out connection) marks the cluster dead, so N databases on it + /// cost one short wait, not N, and the rest are skipped by name. A server that answers with an error (a bad + /// password, a missing database) is not a dead cluster, and a failed drop on a live cluster never stops the + /// others. /// internal static ExitDrainOutcome DropRememberedAtExit(IEnumerable remembered, TimeSpan? budget = null) { - var unreachable = new HashSet(StringComparer.Ordinal); - var limit = budget ?? TimeSpan.FromSeconds(ExitDrainBudgetSeconds); - var clock = Stopwatch.StartNew(); - int attempts = 0, dropped = 0, failed = 0, skipped = 0; - foreach (var db in remembered) + var deadline = budget ?? TimeSpan.FromSeconds(ExitDrainBudgetSeconds); + var databases = remembered.ToList(); + var drain = new ExitDrain(databases.Count); + using var stop = new CancellationTokenSource(deadline); + using var gate = new SemaphoreSlim(ExitDrainParallelism); + + var clusters = databases + .Select((db, index) => (Db: db, Index: index)) + .GroupBy(entry => ClusterKeyOrOwn(entry.Db), StringComparer.Ordinal) + .Select(cluster => Task.Run(() => DrainClusterAsync(cluster.ToList(), drain, gate, stop.Token))) + .ToArray(); + + try { - try + Task.WaitAll(clusters, deadline); + } + catch (AggregateException) + { + /* Every per-database step catches its own failure; nothing here should throw, and nothing may stop the exit. */ + } + + /* One snapshot: a drop sent before the deadline can still finish after this line, and the counts and the names + printed below must agree. */ + var states = drain.Snapshot(); + for (var i = 0; i < databases.Count; i++) + { + if (states[i] == ExitDrain.Unsettled) { - if (clock.Elapsed >= limit) - { - skipped++; - Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped at exit: the exit drain used its {limit.TotalSeconds:0.#} seconds."); - continue; - } + var db = databases[i]; + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped within the exit drain's {deadline.TotalSeconds:0.#}-second limit; the next run's start-of-run sweep removes it."); + } + } - var cluster = ClusterKey(db.AdminConnectionString); - if (unreachable.Contains(cluster)) - { - skipped++; - Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped at exit: its cluster did not answer an earlier connect."); - continue; - } + return new ExitDrainOutcome( + Volatile.Read(ref drain.ConnectAttempts), + states.Count(state => state == ExitDrain.Dropped), + states.Count(state => state == ExitDrain.Failed), + states.Count(state => state == ExitDrain.Skipped), + states.Count(state => state == ExitDrain.Unsettled)); + } - attempts++; - using var admin = new NpgsqlConnection(ExitDrainConnectionString(db.AdminConnectionString)); - try - { - admin.Open(); - } - catch (Exception ex) when (ex is not PostgresException) - { - unreachable.Add(cluster); - throw; - } + /// The database's cluster by host and port, or a key of its own when its connection string does not parse, so it fails alone. + private static string ClusterKeyOrOwn(RememberedDatabase db) + { + try + { + return ClusterKey(db.AdminConnectionString); + } + catch (Exception) + { + return "unparsed:" + db.Name; + } + } - /* Process exit has no async context; the helper is bounded (10 s) and never throws. */ - QuiesceTimescaleJobsAsync(ExitDrainConnectionString(db.AdminConnectionString), db.Name).GetAwaiter().GetResult(); - using var drop = new NpgsqlCommand($"DROP DATABASE IF EXISTS \"{db.Name}\" WITH (FORCE)", admin); - drop.ExecuteNonQuery(); - Remembered.TryRemove(db.Name, out _); - dropped++; - Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}); dropped at process exit."); + private static async Task DrainClusterAsync(List<(RememberedDatabase Db, int Index)> cluster, ExitDrain drain, SemaphoreSlim gate, CancellationToken stop) + { + if (await DropOneAtExitAsync(cluster[0].Db, cluster[0].Index, drain, gate, stop) == ExitDrop.ClusterDead) + { + foreach (var (db, index) in cluster.Skip(1)) + { + drain.Settle(index, ExitDrain.Skipped); + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped at exit: its cluster did not answer an earlier connect."); } - catch (Exception ex) + + return; + } + + await Task.WhenAll(cluster.Skip(1).Select(entry => DropOneAtExitAsync(entry.Db, entry.Index, drain, gate, stop))); + } + + private static async Task DropOneAtExitAsync(RememberedDatabase db, int index, ExitDrain drain, SemaphoreSlim gate, CancellationToken stop) + { + try + { + await gate.WaitAsync(stop); + } + catch (OperationCanceledException) + { + return ExitDrop.OutOfTime; + } + + try + { + Interlocked.Increment(ref drain.ConnectAttempts); + var connectionString = ExitDrainConnectionString(db.AdminConnectionString); + await using var admin = new NpgsqlConnection(connectionString); + try { - failed++; + await admin.OpenAsync(stop); + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + return ExitDrop.OutOfTime; + } + catch (Exception ex) when (ex is not PostgresException) + { + drain.Settle(index, ExitDrain.Failed); Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and could not be dropped at exit: {ex.Message}"); + return ExitDrop.ClusterDead; + } + + /* Bounded by its own cap and by the drain's deadline, and it never throws. */ + await QuiesceTimescaleJobsAsync(connectionString, db.Name, stop); + if (stop.IsCancellationRequested) + { + return ExitDrop.OutOfTime; } + + await using var drop = new NpgsqlCommand($"DROP DATABASE IF EXISTS \"{db.Name}\" WITH (FORCE)", admin); + await drop.ExecuteNonQueryAsync(CancellationToken.None); + Remembered.TryRemove(db.Name, out _); + drain.Settle(index, ExitDrain.Dropped); + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}); dropped at process exit."); + return ExitDrop.Done; + } + catch (Exception ex) + { + drain.Settle(index, ExitDrain.Failed); + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and could not be dropped at exit: {ex.Message}"); + return ExitDrop.Done; } + finally + { + gate.Release(); + } + } + + /// How one database's exit drop ended, as far as the rest of its cluster is concerned. + private enum ExitDrop + { + Done, + ClusterDead, + OutOfTime, + } + + /// Each database's outcome in the drain, set once, and the connect count, safe to update from every drop at once. + private sealed class ExitDrain(int count) + { + public const int Unsettled = 0; + public const int Dropped = 1; + public const int Failed = 2; + public const int Skipped = 3; + + private readonly int[] _states = new int[count]; + + public int ConnectAttempts; + + public void Settle(int index, int state) => Interlocked.CompareExchange(ref _states[index], state, Unsettled); - return new ExitDrainOutcome(attempts, dropped, failed, skipped); + public int[] Snapshot() + { + var copy = new int[_states.Length]; + for (var i = 0; i < copy.Length; i++) + { + copy[i] = Volatile.Read(ref _states[i]); + } + + return copy; + } } /// The cluster a connection string names, by host and port: what the start-of-run sweep and the exit drain each treat as one cluster. diff --git a/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs b/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs index ed30c0e642..e397f30cd2 100644 --- a/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs +++ b/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs @@ -48,6 +48,83 @@ private static string StoppedClusterConnectionString(int port) => private static ScratchPostgres.RememberedDatabase Remembered(string connectionString, string creator) => new($"darling_scratch_20010101000000_{Guid.NewGuid():N}"[..43], connectionString, creator); + /// + /// A cluster that hangs: a listener that never accepts, so the OS completes each TCP handshake and the startup + /// message then waits for an answer that never comes, until the drain's connect timeout. Stop it when done. + /// + private static (TcpListener Listener, string ConnectionString) HangingCluster() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return (listener, StoppedClusterConnectionString(((IPEndPoint)listener.LocalEndpoint).Port)); + } + + /// + /// The 2026-10-08 nightly passed all 28,763 tests and still failed with "[FATAL ERROR] Foreground threads were + /// left running": this drain, a ProcessExit handler, was still dropping 39 leaked databases one at a time when + /// xUnit's 11-second end-of-run window closed. Four clusters that each hang for the full connect timeout cost + /// four timeouts in a row on the old drain (about 12 seconds, past that window) and one when they drain side by side. + /// + [Fact] + public void TheDrain_OfSeveralHangingClusters_WaitsOutTheirConnectsSideBySide() + { + var clusters = new[] { HangingCluster(), HangingCluster(), HangingCluster(), HangingCluster() }; + try + { + var databases = Array.ConvertAll(clusters, cluster => Remembered(cluster.ConnectionString, "a hanging one")); + + var clock = Stopwatch.StartNew(); + var outcome = ScratchPostgres.DropRememberedAtExit(databases); + clock.Stop(); + + Assert.True( + clock.Elapsed < TimeSpan.FromSeconds(ScratchPostgres.ExitDrainConnectTimeoutSeconds * 2), + $"the drain of 4 hanging clusters took {clock.Elapsed.TotalSeconds:0.0} seconds; one at a time costs 4 connect timeouts of {ScratchPostgres.ExitDrainConnectTimeoutSeconds} seconds"); + Assert.Equal(4, outcome.ConnectAttempts); + Assert.Equal(4, outcome.Failed); + Assert.Equal(0, outcome.Dropped); + } + finally + { + foreach (var cluster in clusters) + { + cluster.Listener.Stop(); + } + } + } + + /// + /// Whatever the clusters do, the drain returns by its deadline and names each database it did not finish, so a + /// leak, a slow drop or a hung cluster can never again run the exit past xUnit's window and fail a green run. + /// + [Fact] + public void TheDrain_ReturnsAtItsDeadline_AndCountsWhatItDidNotFinish() + { + var (listener, hanging) = HangingCluster(); + try + { + var deadline = TimeSpan.FromSeconds(1); + var clock = Stopwatch.StartNew(); + var outcome = ScratchPostgres.DropRememberedAtExit( + new[] { Remembered(hanging, "hung one"), Remembered(hanging, "hung two"), Remembered(hanging, "hung three") }, + deadline); + clock.Stop(); + + /* Under the 3-second connect timeout, so it was the deadline, not the timeout, that ended the drain. */ + Assert.True( + clock.Elapsed < TimeSpan.FromSeconds(ScratchPostgres.ExitDrainConnectTimeoutSeconds) - TimeSpan.FromMilliseconds(500), + $"the drain took {clock.Elapsed.TotalSeconds:0.0} seconds against a {deadline.TotalSeconds:0}-second deadline"); + Assert.Equal(3, outcome.Unfinished); + Assert.Equal(0, outcome.Dropped); + Assert.Equal(0, outcome.Failed); + Assert.Equal(0, outcome.Skipped); + } + finally + { + listener.Stop(); + } + } + [Fact] public void TheDrain_OfThreeDatabases_OnAStoppedCluster_TriesTheClusterOnce_AndEndsQuickly() { @@ -77,7 +154,7 @@ public void TheDrain_OutOfBudget_StartsNoDrop_AndNamesEveryDatabaseItLeaves() /* The test runner gives the process 10 seconds to exit once the run returns, then forces exit code 1. The nightly of 2026-10-08 lost its publish to that with 28,763 tests green. A drain whose budget is spent starts no connect at all: nothing is attempted, nothing is dropped, and every database is counted as - left, so the exit ends on time. */ + unfinished, so the exit ends on time. */ var stopped = StoppedClusterConnectionString(ClosedPort()); var databases = new[] { Remembered(stopped, "budget one"), Remembered(stopped, "budget two"), Remembered(stopped, "budget three") }; @@ -86,15 +163,16 @@ public void TheDrain_OutOfBudget_StartsNoDrop_AndNamesEveryDatabaseItLeaves() Assert.Equal(0, outcome.ConnectAttempts); Assert.Equal(0, outcome.Dropped); Assert.Equal(0, outcome.Failed); - Assert.Equal(3, outcome.Skipped); + Assert.Equal(0, outcome.Skipped); + Assert.Equal(3, outcome.Unfinished); } [Fact] public void TheDrainBudget_StaysInsideTheRunnersExitWait_WithRoomForTheProcessToLeave() { /* The runner waits 10 seconds (shutdownForegroundThreadWaitSeconds) and the wait starts when the run returns. - The budget is how long the drain keeps starting drops; one drop already in flight can still run past it, so - the budget must leave real headroom and not equal the wait. */ + The budget bounds the whole drain, a drop in flight included, and still leaves headroom for the process to + leave. */ Assert.InRange(ScratchPostgres.ExitDrainBudgetSeconds, 1, 7); } @@ -163,12 +241,16 @@ public async Task TheDrain_OnALiveCluster_DropsWhatItWasGiven_EvenWhenAStoppedCl await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); var stopped = StoppedClusterConnectionString(ClosedPort()); - var outcome = ScratchPostgres.DropRememberedAtExit(new[] - { - Remembered(stopped, "dead one"), - new ScratchPostgres.RememberedDatabase(scratch.DatabaseName, baseConnectionString!, "the live test"), - Remembered(stopped, "dead two"), - }); + /* A deadline as long as one drop's command timeout: this pins the dead-and-live mix, not the deadline, and a drop + mid-suite waits for a checkpoint that a busy cluster can take seconds to finish. */ + var outcome = ScratchPostgres.DropRememberedAtExit( + new[] + { + Remembered(stopped, "dead one"), + new ScratchPostgres.RememberedDatabase(scratch.DatabaseName, baseConnectionString!, "the live test"), + Remembered(stopped, "dead two"), + }, + TimeSpan.FromSeconds(ScratchPostgres.ExitDrainConnectTimeoutSeconds + ScratchPostgres.ExitDrainCommandTimeoutSeconds) + ScratchPostgres.QuiesceCap); /* One attempt on the stopped cluster, one on the live one; the second database on the stopped cluster is left. */ Assert.Equal(2, outcome.ConnectAttempts); @@ -183,4 +265,64 @@ public async Task TheDrain_OnALiveCluster_DropsWhatItWasGiven_EvenWhenAStoppedCl exists.Parameters.AddWithValue(scratch.DatabaseName); Assert.Equal(0L, Convert.ToInt64(await exists.ExecuteScalarAsync(ct))); } + + /// + /// A run that leaks many databases on a live cluster: the drain returns by its deadline and accounts for every one, + /// dropped or not finished, with none failed. The 2026-10-08 nightly leaked 39, and dropping them one at a time + /// outran xUnit's end-of-run window. + /// How many it drops in time is not pinned. Each DROP DATABASE waits for an immediate checkpoint, and on a + /// cluster busy with other tests' writes one checkpoint took 30 seconds on a local rig (2026-10-08), so a drop can + /// legitimately miss the deadline mid-suite; at the real process exit the cluster is idle. + /// + [Fact] + public async Task TheDrain_OfManyLeakedDatabases_OnALiveCluster_ReturnsByItsDeadline_AndAccountsForEveryOne() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the live exit-drain test."); + + var ct = TestContext.Current.CancellationToken; + const int Leaked = 6; + var scratches = new ScratchPostgres[Leaked]; + try + { + for (var i = 0; i < Leaked; i++) + { + scratches[i] = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + } + + var clock = Stopwatch.StartNew(); + var outcome = ScratchPostgres.DropRememberedAtExit( + Array.ConvertAll(scratches, scratch => new ScratchPostgres.RememberedDatabase(scratch.DatabaseName, baseConnectionString!, "the leak test"))); + clock.Stop(); + + Assert.True( + clock.Elapsed < TimeSpan.FromSeconds(ScratchPostgres.ExitDrainBudgetSeconds) + TimeSpan.FromSeconds(1), + $"the drain of {Leaked} databases took {clock.Elapsed.TotalSeconds:0.0} seconds against its {TimeSpan.FromSeconds(ScratchPostgres.ExitDrainBudgetSeconds).TotalSeconds:0}-second deadline"); + Assert.Equal(Leaked, outcome.Dropped + outcome.Unfinished); + Assert.Equal(0, outcome.Failed); + Assert.Equal(0, outcome.Skipped); + Assert.True(outcome.ConnectAttempts >= 1, "the drain never tried the live cluster"); + + /* A database the drain dropped is gone from the cluster and from the queue. A drop it sent before the + deadline may finish after it returned, so the queue can hold fewer than the outcome's unfinished count. */ + var dropped = Array.FindAll(scratches, scratch => !ScratchPostgres.IsRemembered(scratch.DatabaseName)); + Assert.True(dropped.Length >= outcome.Dropped, $"the drain reported {outcome.Dropped} dropped, and only {dropped.Length} left the queue"); + await using var probe = new NpgsqlConnection(baseConnectionString); + await probe.OpenAsync(ct); + await using var exists = new NpgsqlCommand("SELECT count(*) FROM pg_database WHERE datname = ANY($1)", probe); + exists.Parameters.AddWithValue(Array.ConvertAll(dropped, scratch => scratch.DatabaseName)); + Assert.Equal(0L, Convert.ToInt64(await exists.ExecuteScalarAsync(ct))); + } + finally + { + /* A drop the drain already made makes this a no-op; one it missed (a failing run) is dropped here, not at exit. */ + foreach (var scratch in scratches) + { + if (scratch is not null) + { + await scratch.DisposeAsync(); + } + } + } + } }