From c29d8b88d2a348dd04eef2b1cc78a3157e9ab827 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:55:57 -0400 Subject: [PATCH] Darling.Tests: the exit drain returns at its budget even with a drop in flight The drain stopped starting drops after 6 seconds, but a drop already started could still run its quiesce (up to 10 s) and its DROP DATABASE (up to 30 s), and one DROP DATABASE waited 26 seconds for its immediate checkpoint on a local rig, so the exit could still pass xUnit's 11-second window. The drain now runs clusters side by side and up to 16 drops at once under one deadline, returns when it passes whatever is in flight, cancels connects and quiesces at the deadline (never a sent drop), and counts what it did not finish as Unfinished. The one-connect probe of a dead cluster and the quiesce before each drop are kept. Tests: four hanging clusters drain in 3.0 s (12.0 s on the old drain, past the window); a 1-second budget returns in 1.04 s with all three unfinished; a live leak of six databases returns by the budget and accounts for each. Refs #5480 --- Darling/Darling.Tests/ScratchPostgres.cs | 255 +++++++++++++----- .../ScratchPostgresExitDrainTests.cs | 162 ++++++++++- 2 files changed, 342 insertions(+), 75 deletions(-) diff --git a/Darling/Darling.Tests/ScratchPostgres.cs b/Darling/Darling.Tests/ScratchPostgres.cs index 251e109a6a..f56e7332fa 100644 --- a/Darling/Darling.Tests/ScratchPostgres.cs +++ b/Darling/Darling.Tests/ScratchPostgres.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Npgsql; @@ -169,14 +169,16 @@ internal static async Task DropRememberedAsync(params string[] databaseNames) /// PostgreSQL postmaster log "terminated by exception 0xC0000005" for a TimescaleDB worker, twice, both times /// with a scratch-database FORCE drop in flight (PR #5480); no worker is running in the dropped database /// once this returns, so the drop has nothing to kill. - /// Best-effort and bounded: it never throws, never fails a test, and never skips the drop. A database + /// Best-effort and bounded: it never throws, never fails a test, and never skips the drop. A caller's + /// ends it early too, without a message (the exit drain passes its deadline). A database /// without the extension, or one already gone, returns at once. It does NOT call /// _timescaledb_functions.stop_background_workers(), which SIGTERMs running jobs, the very kill this /// avoids. /// - internal static async Task QuiesceTimescaleJobsAsync(string adminConnectionString, string databaseName) + internal static async Task QuiesceTimescaleJobsAsync(string adminConnectionString, string databaseName, CancellationToken cancellationToken = default) { - using var cap = new CancellationTokenSource(QuiesceCap); + using var cap = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cap.CancelAfter(QuiesceCap); try { var scratchBuilder = new NpgsqlConnectionStringBuilder(adminConnectionString) @@ -221,6 +223,10 @@ starting up and not yet show in pg_stat_activity on the first look. */ { /* The database is already gone: nothing to quiesce, and the drop below is IF EXISTS. */ } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + /* The caller's own deadline (the exit drain's), not the cap: the caller decides whether the drop still runs. */ + } catch (Exception ex) { /* The cap, a missing database or a refused connect all land here. The drop that follows is the @@ -293,8 +299,11 @@ private static void DropEverythingRemembered() /// One database the exit drain was asked to drop: its name, the connection string that drops it, and the test that created it. internal readonly record struct RememberedDatabase(string Name, string AdminConnectionString, string Creator); - /// What an exit drain did: connections it tried to open, databases it dropped, drops that failed, and databases it left alone. - internal readonly record struct ExitDrainOutcome(int ConnectAttempts, int Dropped, int Failed, int Skipped); + /// + /// What an exit drain did: connections it tried to open, databases it dropped, drops that failed, databases it left + /// alone because their cluster did not answer, and databases it had not finished when its deadline passed. + /// + internal readonly record struct ExitDrainOutcome(int ConnectAttempts, int Dropped, int Failed, int Skipped, int Unfinished); /// /// How long the exit drain waits to connect to a cluster, in seconds. The default is 15, and the drain ran once per @@ -311,19 +320,6 @@ private static void DropEverythingRemembered() /// internal const int ExitDrainCommandTimeoutSeconds = 30; - /// - /// How long the exit drain keeps starting new drops, in seconds. The test runner (xunit.v3 under Microsoft.Testing.Platform) - /// gives the process 10 seconds after the run returns to exit, then prints "Foreground threads were left running, - /// forcing process exit" and exits with code 1, whatever the tests did. This drain runs at that exit, so it is the - /// one thing that can hold the process. Each drop here also unschedules the database's TimescaleDB jobs and waits - /// for their workers (#5480), about 0.4 seconds a database on a CI runner, so 29 databases that outlived their tests - /// took 11 seconds and failed the nightly with every test green. The drain stops starting drops after this long and - /// names the rest; a database left this way is removed by the next run's start-of-run sweep, or goes with the - /// throwaway cluster. Tests that mint a database must still drop it themselves (see - /// ScratchDatabaseDisposalCensusTests); this budget only keeps a leak from failing a run. - /// - internal const int ExitDrainBudgetSeconds = 6; - /// The admin connection string with the exit drain's own connect and command timeouts, and no pooling: one connection per drop. internal static string ExitDrainConnectionString(string adminConnectionString) => new NpgsqlConnectionStringBuilder(adminConnectionString) @@ -333,71 +329,200 @@ internal static string ExitDrainConnectionString(string adminConnectionString) = Pooling = false, }.ConnectionString; + /// + /// How long the whole exit drain may take, in seconds, in-flight drops included. The drain is a ProcessExit handler, + /// and xUnit v3 under Microsoft.Testing.Platform gives that stage 11 seconds after the run returns (one second, then + /// its shutdownForegroundThreadWaitSeconds default of 10) before it prints "[FATAL ERROR] Foreground threads + /// were left running, forcing process exit" and exits with code 1. No thread was left running: the 2026-10-08 + /// nightly passed every test and still failed because this drain, then dropping 39 leaked databases one at a time + /// with a TimescaleDB quiesce before each (#5480), was still at work when that window closed. The drain returns when + /// this passes even with a drop in flight (one DROP DATABASE waited 26 seconds for its checkpoint on a local rig), and + /// names every database it did not finish; the next run's start-of-run sweep removes those, or they go with the + /// throwaway cluster. Tests that mint a database must still drop it themselves (see + /// ScratchDatabaseDisposalCensusTests); this budget only keeps a leak from failing a run. + /// + internal const int ExitDrainBudgetSeconds = 6; + + /// How many databases the exit drain quiesces and drops at once. Each holds at most two connections at a time. + internal const int ExitDrainParallelism = 16; + /// /// The exit drain's body, taking its list as a parameter so a test can run it without a process exit and without /// touching the databases that other tests, running in parallel, still hold. /// /// - /// Connects with the short connect wait above, and stops trying a cluster once a connect to it fails without the - /// server answering (a stopped cluster, a refused or timed-out connection), so N databases on a dead cluster cost - /// one short wait, not N. A server that answers with an error (a bad password, a missing database) is not a dead - /// cluster, and a failed drop on a live cluster never stops the others; each skipped database is still named. It - /// stops starting drops once (default seconds) has - /// passed, counting the rest as skipped. + /// Clusters drain side by side, and within a live cluster up to databases + /// drop at once, all under one (default seconds). + /// When it passes, the drain returns and names every + /// database it did not finish. A drop already sent is not cancelled (an interrupted DROP DATABASE can leave an + /// invalid database behind), but no new connect, quiesce or drop starts. + /// The first database on each cluster goes alone and probes it. A connect that fails without the server + /// answering (a stopped cluster, a refused or timed-out connection) marks the cluster dead, so N databases on it + /// cost one short wait, not N, and the rest are skipped by name. A server that answers with an error (a bad + /// password, a missing database) is not a dead cluster, and a failed drop on a live cluster never stops the + /// others. /// internal static ExitDrainOutcome DropRememberedAtExit(IEnumerable remembered, TimeSpan? budget = null) { - var unreachable = new HashSet(StringComparer.Ordinal); - var limit = budget ?? TimeSpan.FromSeconds(ExitDrainBudgetSeconds); - var clock = Stopwatch.StartNew(); - int attempts = 0, dropped = 0, failed = 0, skipped = 0; - foreach (var db in remembered) + var deadline = budget ?? TimeSpan.FromSeconds(ExitDrainBudgetSeconds); + var databases = remembered.ToList(); + var drain = new ExitDrain(databases.Count); + using var stop = new CancellationTokenSource(deadline); + using var gate = new SemaphoreSlim(ExitDrainParallelism); + + var clusters = databases + .Select((db, index) => (Db: db, Index: index)) + .GroupBy(entry => ClusterKeyOrOwn(entry.Db), StringComparer.Ordinal) + .Select(cluster => Task.Run(() => DrainClusterAsync(cluster.ToList(), drain, gate, stop.Token))) + .ToArray(); + + try { - try + Task.WaitAll(clusters, deadline); + } + catch (AggregateException) + { + /* Every per-database step catches its own failure; nothing here should throw, and nothing may stop the exit. */ + } + + /* One snapshot: a drop sent before the deadline can still finish after this line, and the counts and the names + printed below must agree. */ + var states = drain.Snapshot(); + for (var i = 0; i < databases.Count; i++) + { + if (states[i] == ExitDrain.Unsettled) { - if (clock.Elapsed >= limit) - { - skipped++; - Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped at exit: the exit drain used its {limit.TotalSeconds:0.#} seconds."); - continue; - } + var db = databases[i]; + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped within the exit drain's {deadline.TotalSeconds:0.#}-second limit; the next run's start-of-run sweep removes it."); + } + } - var cluster = ClusterKey(db.AdminConnectionString); - if (unreachable.Contains(cluster)) - { - skipped++; - Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped at exit: its cluster did not answer an earlier connect."); - continue; - } + return new ExitDrainOutcome( + Volatile.Read(ref drain.ConnectAttempts), + states.Count(state => state == ExitDrain.Dropped), + states.Count(state => state == ExitDrain.Failed), + states.Count(state => state == ExitDrain.Skipped), + states.Count(state => state == ExitDrain.Unsettled)); + } - attempts++; - using var admin = new NpgsqlConnection(ExitDrainConnectionString(db.AdminConnectionString)); - try - { - admin.Open(); - } - catch (Exception ex) when (ex is not PostgresException) - { - unreachable.Add(cluster); - throw; - } + /// The database's cluster by host and port, or a key of its own when its connection string does not parse, so it fails alone. + private static string ClusterKeyOrOwn(RememberedDatabase db) + { + try + { + return ClusterKey(db.AdminConnectionString); + } + catch (Exception) + { + return "unparsed:" + db.Name; + } + } - /* Process exit has no async context; the helper is bounded (10 s) and never throws. */ - QuiesceTimescaleJobsAsync(ExitDrainConnectionString(db.AdminConnectionString), db.Name).GetAwaiter().GetResult(); - using var drop = new NpgsqlCommand($"DROP DATABASE IF EXISTS \"{db.Name}\" WITH (FORCE)", admin); - drop.ExecuteNonQuery(); - Remembered.TryRemove(db.Name, out _); - dropped++; - Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}); dropped at process exit."); + private static async Task DrainClusterAsync(List<(RememberedDatabase Db, int Index)> cluster, ExitDrain drain, SemaphoreSlim gate, CancellationToken stop) + { + if (await DropOneAtExitAsync(cluster[0].Db, cluster[0].Index, drain, gate, stop) == ExitDrop.ClusterDead) + { + foreach (var (db, index) in cluster.Skip(1)) + { + drain.Settle(index, ExitDrain.Skipped); + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and was not dropped at exit: its cluster did not answer an earlier connect."); } - catch (Exception ex) + + return; + } + + await Task.WhenAll(cluster.Skip(1).Select(entry => DropOneAtExitAsync(entry.Db, entry.Index, drain, gate, stop))); + } + + private static async Task DropOneAtExitAsync(RememberedDatabase db, int index, ExitDrain drain, SemaphoreSlim gate, CancellationToken stop) + { + try + { + await gate.WaitAsync(stop); + } + catch (OperationCanceledException) + { + return ExitDrop.OutOfTime; + } + + try + { + Interlocked.Increment(ref drain.ConnectAttempts); + var connectionString = ExitDrainConnectionString(db.AdminConnectionString); + await using var admin = new NpgsqlConnection(connectionString); + try { - failed++; + await admin.OpenAsync(stop); + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + return ExitDrop.OutOfTime; + } + catch (Exception ex) when (ex is not PostgresException) + { + drain.Settle(index, ExitDrain.Failed); Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and could not be dropped at exit: {ex.Message}"); + return ExitDrop.ClusterDead; + } + + /* Bounded by its own cap and by the drain's deadline, and it never throws. */ + await QuiesceTimescaleJobsAsync(connectionString, db.Name, stop); + if (stop.IsCancellationRequested) + { + return ExitDrop.OutOfTime; } + + await using var drop = new NpgsqlCommand($"DROP DATABASE IF EXISTS \"{db.Name}\" WITH (FORCE)", admin); + await drop.ExecuteNonQueryAsync(CancellationToken.None); + Remembered.TryRemove(db.Name, out _); + drain.Settle(index, ExitDrain.Dropped); + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}); dropped at process exit."); + return ExitDrop.Done; + } + catch (Exception ex) + { + drain.Settle(index, ExitDrain.Failed); + Console.Error.WriteLine($"Scratch database {db.Name} outlived its test ({db.Creator}) and could not be dropped at exit: {ex.Message}"); + return ExitDrop.Done; } + finally + { + gate.Release(); + } + } + + /// How one database's exit drop ended, as far as the rest of its cluster is concerned. + private enum ExitDrop + { + Done, + ClusterDead, + OutOfTime, + } + + /// Each database's outcome in the drain, set once, and the connect count, safe to update from every drop at once. + private sealed class ExitDrain(int count) + { + public const int Unsettled = 0; + public const int Dropped = 1; + public const int Failed = 2; + public const int Skipped = 3; + + private readonly int[] _states = new int[count]; + + public int ConnectAttempts; + + public void Settle(int index, int state) => Interlocked.CompareExchange(ref _states[index], state, Unsettled); - return new ExitDrainOutcome(attempts, dropped, failed, skipped); + public int[] Snapshot() + { + var copy = new int[_states.Length]; + for (var i = 0; i < copy.Length; i++) + { + copy[i] = Volatile.Read(ref _states[i]); + } + + return copy; + } } /// The cluster a connection string names, by host and port: what the start-of-run sweep and the exit drain each treat as one cluster. diff --git a/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs b/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs index ed30c0e642..e397f30cd2 100644 --- a/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs +++ b/Darling/Darling.Tests/ScratchPostgresExitDrainTests.cs @@ -48,6 +48,83 @@ private static string StoppedClusterConnectionString(int port) => private static ScratchPostgres.RememberedDatabase Remembered(string connectionString, string creator) => new($"darling_scratch_20010101000000_{Guid.NewGuid():N}"[..43], connectionString, creator); + /// + /// A cluster that hangs: a listener that never accepts, so the OS completes each TCP handshake and the startup + /// message then waits for an answer that never comes, until the drain's connect timeout. Stop it when done. + /// + private static (TcpListener Listener, string ConnectionString) HangingCluster() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return (listener, StoppedClusterConnectionString(((IPEndPoint)listener.LocalEndpoint).Port)); + } + + /// + /// The 2026-10-08 nightly passed all 28,763 tests and still failed with "[FATAL ERROR] Foreground threads were + /// left running": this drain, a ProcessExit handler, was still dropping 39 leaked databases one at a time when + /// xUnit's 11-second end-of-run window closed. Four clusters that each hang for the full connect timeout cost + /// four timeouts in a row on the old drain (about 12 seconds, past that window) and one when they drain side by side. + /// + [Fact] + public void TheDrain_OfSeveralHangingClusters_WaitsOutTheirConnectsSideBySide() + { + var clusters = new[] { HangingCluster(), HangingCluster(), HangingCluster(), HangingCluster() }; + try + { + var databases = Array.ConvertAll(clusters, cluster => Remembered(cluster.ConnectionString, "a hanging one")); + + var clock = Stopwatch.StartNew(); + var outcome = ScratchPostgres.DropRememberedAtExit(databases); + clock.Stop(); + + Assert.True( + clock.Elapsed < TimeSpan.FromSeconds(ScratchPostgres.ExitDrainConnectTimeoutSeconds * 2), + $"the drain of 4 hanging clusters took {clock.Elapsed.TotalSeconds:0.0} seconds; one at a time costs 4 connect timeouts of {ScratchPostgres.ExitDrainConnectTimeoutSeconds} seconds"); + Assert.Equal(4, outcome.ConnectAttempts); + Assert.Equal(4, outcome.Failed); + Assert.Equal(0, outcome.Dropped); + } + finally + { + foreach (var cluster in clusters) + { + cluster.Listener.Stop(); + } + } + } + + /// + /// Whatever the clusters do, the drain returns by its deadline and names each database it did not finish, so a + /// leak, a slow drop or a hung cluster can never again run the exit past xUnit's window and fail a green run. + /// + [Fact] + public void TheDrain_ReturnsAtItsDeadline_AndCountsWhatItDidNotFinish() + { + var (listener, hanging) = HangingCluster(); + try + { + var deadline = TimeSpan.FromSeconds(1); + var clock = Stopwatch.StartNew(); + var outcome = ScratchPostgres.DropRememberedAtExit( + new[] { Remembered(hanging, "hung one"), Remembered(hanging, "hung two"), Remembered(hanging, "hung three") }, + deadline); + clock.Stop(); + + /* Under the 3-second connect timeout, so it was the deadline, not the timeout, that ended the drain. */ + Assert.True( + clock.Elapsed < TimeSpan.FromSeconds(ScratchPostgres.ExitDrainConnectTimeoutSeconds) - TimeSpan.FromMilliseconds(500), + $"the drain took {clock.Elapsed.TotalSeconds:0.0} seconds against a {deadline.TotalSeconds:0}-second deadline"); + Assert.Equal(3, outcome.Unfinished); + Assert.Equal(0, outcome.Dropped); + Assert.Equal(0, outcome.Failed); + Assert.Equal(0, outcome.Skipped); + } + finally + { + listener.Stop(); + } + } + [Fact] public void TheDrain_OfThreeDatabases_OnAStoppedCluster_TriesTheClusterOnce_AndEndsQuickly() { @@ -77,7 +154,7 @@ public void TheDrain_OutOfBudget_StartsNoDrop_AndNamesEveryDatabaseItLeaves() /* The test runner gives the process 10 seconds to exit once the run returns, then forces exit code 1. The nightly of 2026-10-08 lost its publish to that with 28,763 tests green. A drain whose budget is spent starts no connect at all: nothing is attempted, nothing is dropped, and every database is counted as - left, so the exit ends on time. */ + unfinished, so the exit ends on time. */ var stopped = StoppedClusterConnectionString(ClosedPort()); var databases = new[] { Remembered(stopped, "budget one"), Remembered(stopped, "budget two"), Remembered(stopped, "budget three") }; @@ -86,15 +163,16 @@ public void TheDrain_OutOfBudget_StartsNoDrop_AndNamesEveryDatabaseItLeaves() Assert.Equal(0, outcome.ConnectAttempts); Assert.Equal(0, outcome.Dropped); Assert.Equal(0, outcome.Failed); - Assert.Equal(3, outcome.Skipped); + Assert.Equal(0, outcome.Skipped); + Assert.Equal(3, outcome.Unfinished); } [Fact] public void TheDrainBudget_StaysInsideTheRunnersExitWait_WithRoomForTheProcessToLeave() { /* The runner waits 10 seconds (shutdownForegroundThreadWaitSeconds) and the wait starts when the run returns. - The budget is how long the drain keeps starting drops; one drop already in flight can still run past it, so - the budget must leave real headroom and not equal the wait. */ + The budget bounds the whole drain, a drop in flight included, and still leaves headroom for the process to + leave. */ Assert.InRange(ScratchPostgres.ExitDrainBudgetSeconds, 1, 7); } @@ -163,12 +241,16 @@ public async Task TheDrain_OnALiveCluster_DropsWhatItWasGiven_EvenWhenAStoppedCl await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); var stopped = StoppedClusterConnectionString(ClosedPort()); - var outcome = ScratchPostgres.DropRememberedAtExit(new[] - { - Remembered(stopped, "dead one"), - new ScratchPostgres.RememberedDatabase(scratch.DatabaseName, baseConnectionString!, "the live test"), - Remembered(stopped, "dead two"), - }); + /* A deadline as long as one drop's command timeout: this pins the dead-and-live mix, not the deadline, and a drop + mid-suite waits for a checkpoint that a busy cluster can take seconds to finish. */ + var outcome = ScratchPostgres.DropRememberedAtExit( + new[] + { + Remembered(stopped, "dead one"), + new ScratchPostgres.RememberedDatabase(scratch.DatabaseName, baseConnectionString!, "the live test"), + Remembered(stopped, "dead two"), + }, + TimeSpan.FromSeconds(ScratchPostgres.ExitDrainConnectTimeoutSeconds + ScratchPostgres.ExitDrainCommandTimeoutSeconds) + ScratchPostgres.QuiesceCap); /* One attempt on the stopped cluster, one on the live one; the second database on the stopped cluster is left. */ Assert.Equal(2, outcome.ConnectAttempts); @@ -183,4 +265,64 @@ public async Task TheDrain_OnALiveCluster_DropsWhatItWasGiven_EvenWhenAStoppedCl exists.Parameters.AddWithValue(scratch.DatabaseName); Assert.Equal(0L, Convert.ToInt64(await exists.ExecuteScalarAsync(ct))); } + + /// + /// A run that leaks many databases on a live cluster: the drain returns by its deadline and accounts for every one, + /// dropped or not finished, with none failed. The 2026-10-08 nightly leaked 39, and dropping them one at a time + /// outran xUnit's end-of-run window. + /// How many it drops in time is not pinned. Each DROP DATABASE waits for an immediate checkpoint, and on a + /// cluster busy with other tests' writes one checkpoint took 30 seconds on a local rig (2026-10-08), so a drop can + /// legitimately miss the deadline mid-suite; at the real process exit the cluster is idle. + /// + [Fact] + public async Task TheDrain_OfManyLeakedDatabases_OnALiveCluster_ReturnsByItsDeadline_AndAccountsForEveryOne() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the live exit-drain test."); + + var ct = TestContext.Current.CancellationToken; + const int Leaked = 6; + var scratches = new ScratchPostgres[Leaked]; + try + { + for (var i = 0; i < Leaked; i++) + { + scratches[i] = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + } + + var clock = Stopwatch.StartNew(); + var outcome = ScratchPostgres.DropRememberedAtExit( + Array.ConvertAll(scratches, scratch => new ScratchPostgres.RememberedDatabase(scratch.DatabaseName, baseConnectionString!, "the leak test"))); + clock.Stop(); + + Assert.True( + clock.Elapsed < TimeSpan.FromSeconds(ScratchPostgres.ExitDrainBudgetSeconds) + TimeSpan.FromSeconds(1), + $"the drain of {Leaked} databases took {clock.Elapsed.TotalSeconds:0.0} seconds against its {TimeSpan.FromSeconds(ScratchPostgres.ExitDrainBudgetSeconds).TotalSeconds:0}-second deadline"); + Assert.Equal(Leaked, outcome.Dropped + outcome.Unfinished); + Assert.Equal(0, outcome.Failed); + Assert.Equal(0, outcome.Skipped); + Assert.True(outcome.ConnectAttempts >= 1, "the drain never tried the live cluster"); + + /* A database the drain dropped is gone from the cluster and from the queue. A drop it sent before the + deadline may finish after it returned, so the queue can hold fewer than the outcome's unfinished count. */ + var dropped = Array.FindAll(scratches, scratch => !ScratchPostgres.IsRemembered(scratch.DatabaseName)); + Assert.True(dropped.Length >= outcome.Dropped, $"the drain reported {outcome.Dropped} dropped, and only {dropped.Length} left the queue"); + await using var probe = new NpgsqlConnection(baseConnectionString); + await probe.OpenAsync(ct); + await using var exists = new NpgsqlCommand("SELECT count(*) FROM pg_database WHERE datname = ANY($1)", probe); + exists.Parameters.AddWithValue(Array.ConvertAll(dropped, scratch => scratch.DatabaseName)); + Assert.Equal(0L, Convert.ToInt64(await exists.ExecuteScalarAsync(ct))); + } + finally + { + /* A drop the drain already made makes this a no-op; one it missed (a failing run) is dropped here, not at exit. */ + foreach (var scratch in scratches) + { + if (scratch is not null) + { + await scratch.DisposeAsync(); + } + } + } + } }