Fix kernel console-capture ExecutionContext leak (MeshHubDisposalLeakTest flake)

src/MeshWeaver.Kernel.Hub/CapturingTextWriter.cs

@@ -20,19 +20,41 @@ namespace MeshWeaver.Kernel.Hub;
 /// capture's stdout target, <c>Console.Error.WriteLine</c> on its stderr target —
 /// so a script's error prints reach the ActivityLog (as error-level messages)
 /// instead of vanishing into the host process's stderr.</para>
+///
+/// <para>🚨 The AsyncLocal holds a mutable <see cref="CaptureCell"/>, NEVER the
+/// writer itself. Anything that snapshots the ExecutionContext while a capture is
+/// active — a lazily-created long-lived <c>TimerQueueTimer</c>, a pending Rx delay,
+/// a pooled work item — freezes the AsyncLocal map into that snapshot, and the
+/// scope's Restorer can never reach it. When the map held the
+/// <c>LoggerTextWriter</c> directly, one such timer pinned the writer →
+/// <c>ActivityLogLogger</c> → the DISPOSED kernel activity hub → the whole mesh for
+/// the timer's lifetime (the <c>MeshHubDisposalLeakTest</c> CI retention path, run
+/// 30068597014). With the cell indirection, scope disposal nulls
+/// <see cref="CaptureCell.Target"/> and SEVERS the graph even inside frozen
+/// snapshots — a stray context keeps only the empty cell.</para>
 /// </summary>
-internal sealed class CapturingTextWriter(AsyncLocal<TextWriter?> channel, TextWriter fallback) : TextWriter
+internal sealed class CapturingTextWriter(AsyncLocal<CapturingTextWriter.CaptureCell?> channel, TextWriter fallback) : TextWriter
 {
+    /// <summary>
+    /// Indirection between the AsyncLocal map and the capture target. Captured
+    /// ExecutionContexts reference this cell; disposing the capture scope nulls
+    /// <see cref="Target"/>, releasing the writer graph from every snapshot.
+    /// </summary>
+    internal sealed class CaptureCell
+    {
+        public volatile TextWriter? Target;
+    }
+
     // AsyncLocal flow-state (NOT a cache/collection): scopes each capture to its own
     // logical execution context. One channel per standard stream.
-    private static readonly AsyncLocal<TextWriter?> CurrentOutTarget = new();
-    private static readonly AsyncLocal<TextWriter?> CurrentErrorTarget = new();
+    private static readonly AsyncLocal<CaptureCell?> CurrentOutTarget = new();
+    private static readonly AsyncLocal<CaptureCell?> CurrentErrorTarget = new();
     private static int installedOut;
     private static int installedError;
 
     public override Encoding Encoding => fallback.Encoding;
 
-    private TextWriter Active => channel.Value ?? fallback;
+    private TextWriter Active => channel.Value?.Target ?? fallback;
 
     public override void Write(char value) => Active.Write(value);
     public override void Write(string? value) => Active.Write(value);
@@ -44,7 +66,9 @@ internal sealed class CapturingTextWriter(AsyncLocal<TextWriter?> channel, TextW
     /// <summary>
     /// Begin capturing <c>Console.Out</c> writes to <paramref name="outTarget"/> and
     /// <c>Console.Error</c> writes to <paramref name="errorTarget"/> on the current
-    /// async-flow. Dispose the returned scope to restore the previous targets.
+    /// async-flow. Dispose the returned scope to restore the previous targets AND
+    /// sever the targets from any ExecutionContext snapshotted during the scope
+    /// (see the class remarks — this is what keeps disposed meshes collectible).
     /// Idempotently installs the global console hooks on first use.
     /// </summary>
     public static IDisposable Capture(TextWriter outTarget, TextWriter errorTarget)
@@ -56,10 +80,18 @@ public static IDisposable Capture(TextWriter outTarget, TextWriter errorTarget)
 
         var prevOut = CurrentOutTarget.Value;
         var prevError = CurrentErrorTarget.Value;
-        CurrentOutTarget.Value = outTarget;
-        CurrentErrorTarget.Value = errorTarget;
+        var outCell = new CaptureCell { Target = outTarget };
+        var errorCell = new CaptureCell { Target = errorTarget };
+        CurrentOutTarget.Value = outCell;
+        CurrentErrorTarget.Value = errorCell;
         return new Restorer(() =>
         {
+            // Sever FIRST: contexts already captured by timers / pool items during
+            // the scope hold these cells — nulling Target releases the writer graph
+            // inside those frozen snapshots (their late writes fall back to the real
+            // console instead of a disposed pipe). Then restore the current flow.
+            outCell.Target = null;
+            errorCell.Target = null;
             CurrentOutTarget.Value = prevOut;
             CurrentErrorTarget.Value = prevError;
         });

test/MeshWeaver.Hosting.Monolith.Test/ConsoleCaptureExecutionContextLeakTest.cs

@@ -0,0 +1,75 @@
+using System;
+using System.IO;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using MeshWeaver.Kernel.Hub;
+using Xunit;
+
+namespace MeshWeaver.Hosting.Monolith.Test;
+
+/// <summary>
+/// Deterministic repro for the <see cref="MeshHubDisposalLeakTest"/> CI flake
+/// (run 30068597014, shard 1): the ClrMD retention path was
+/// <c>TimerQueueTimer → ExecutionContext → AsyncLocalValueMap →
+/// MeshWeaver.Kernel.Hub.LoggerTextWriter → ActivityLogLogger → MessageHub
+/// (kernelExec/…/_Activity/…, RunLevel=6)</c> — a DISPOSED kernel activity hub,
+/// and through it the whole mesh, pinned forever.
+///
+/// <para>Mechanism: <see cref="CapturingTextWriter.Capture"/> stores the script's
+/// stdout/stderr pipe in an <see cref="AsyncLocal{T}"/>. Any long-lived timer (or
+/// pooled work item) whose creation falls INSIDE the capture window snapshots the
+/// ExecutionContext — including that AsyncLocal map. Restoring the AsyncLocal on
+/// scope disposal only affects the current flow; it can never reach the frozen
+/// snapshots, so the writer → activity-hub → mesh graph stays rooted for the
+/// timer's lifetime. Intermittent on CI because it needs an infrastructure
+/// timer's first creation to land inside a script run.</para>
+///
+/// <para>The fix stores the target behind a mutable indirection cell; disposing
+/// the capture scope nulls the cell's target, severing the graph even inside
+/// already-captured ExecutionContexts. This test creates a timer inside a capture
+/// scope, keeps the timer alive, and asserts the capture target is collectible
+/// after the scope is disposed.</para>
+/// </summary>
+public class ConsoleCaptureExecutionContextLeakTest
+{
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static (WeakReference Target, Timer Timer) CreateTimerInsideCaptureScope()
+    {
+        var target = new StringWriter(); // stands in for LoggerTextWriter → ActivityLogLogger → hub
+        Timer timer;
+        using (CapturingTextWriter.Capture(target, target))
+        {
+            // A long-lived timer created inside the capture scope snapshots the
+            // ExecutionContext — exactly what a lazily-initialized infrastructure
+            // timer does when its first creation falls inside a kernel script run.
+            timer = new Timer(static _ => { }, null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan);
+        }
+        return (new WeakReference(target), timer);
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void ForceCollect(WeakReference weak)
+    {
+        for (var i = 0; i < 12 && weak.IsAlive; i++)
+        {
+            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true);
+            GC.WaitForPendingFinalizers();
+            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true);
+        }
+    }
+
+    [Fact]
+    public void CaptureScopeDisposal_SeversTargetFromCapturedExecutionContexts()
+    {
+        var (weak, timer) = CreateTimerInsideCaptureScope();
+        using (timer) // the timer stays alive across the collect — the CI TimerQueue pin
+        {
+            ForceCollect(weak);
+            weak.IsAlive.Should().BeFalse(
+                "disposing the console-capture scope must sever the capture target from " +
+                "ExecutionContexts snapshotted by timers created inside the scope — otherwise " +
+                "the LoggerTextWriter → ActivityLogLogger → kernel activity hub → mesh graph " +
+                "stays pinned for the timer's lifetime (MeshHubDisposalLeakTest CI retention path)");
+        }
+    }
+}