Fix the exit=139 teardown SIGSEGV for good: pin the DAC — ClrMD's unload left a dangling pthread-key destructor

.github/workflows/alc-unload-probe.yml

@@ -80,15 +80,9 @@ jobs:
       # identical binary that crashes in CI.
       - name: Build
         run: dotnet build --no-restore -c Release -p:CIRun=true -warnaserror
-      # The mesh-local #r feed dynamic-compilation tests resolve at runtime (version-less
-      # `#r "nuget:MeshWeaver.X"`) — packed exactly as dotnet-test.yml / flake-repro.yml do.
-      - name: Pack mesh-local #r packages
-        run: |
-          set -euo pipefail
-          dotnet pack src/MeshWeaver.BusinessRules/MeshWeaver.BusinessRules.csproj \
-            -c Release --no-build --no-restore -o dist/packages --nologo
-          dotnet pack src/MeshWeaver.BusinessRules.Generator/MeshWeaver.BusinessRules.Generator.csproj \
-            -c Release --no-build --no-restore -o dist/packages --nologo
+      # No mesh-local `#r` pack step: BusinessRules left the platform (f7a8c086c — ships
+      # as a plugin now), same removal dotnet-test.yml already made. Version-less
+      # `#r "nuget:MeshWeaver.BusinessRules*"` directives are filtered out at runtime.
       - name: Loop the native host under the probe until it crashes
         run: |
           set -uo pipefail

test/MeshWeaver.Hosting.Monolith.Test/ClrMdDacPin.cs

@@ -0,0 +1,78 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+namespace MeshWeaver.Hosting.Monolith.Test;
+
+/// <summary>
+/// 🚨 Pins the CLR Data Access Component (DAC, <c>libmscordaccore.so</c> /
+/// <c>mscordaccore.dll</c>) for the lifetime of the process — the root-cause fix for the
+/// endemic <c>MeshWeaver.Hosting.Monolith.Test exit=139</c> teardown SIGSEGV that kept
+/// turning main red.
+///
+/// <para><b>The defect (proven from the corruption-time core dump of CI run 29471199052,
+/// attempt 2, and reproduced identically by the ALC-unload probe run 29482327524):</b>
+/// ClrMD's <c>DataTarget.CreateSnapshotAndAttach</c> dlopens the runtime's DAC in-process.
+/// The DAC carries its own copy of the CoreCLR PAL, whose initialization calls
+/// <c>pthread_key_create(&amp;key, InternalEndCurrentThreadWrapper)</c> — a PROCESS-GLOBAL
+/// registration whose destructor pointer targets the DAC's own code
+/// (<c>pal/src/thread/thread.cpp</c>). Every thread that executes DAC code (the ClrMD heap
+/// walk) gets a <c>CorUnix::CPalThread*</c> stored in that key. When the
+/// <c>DataTarget</c> is disposed, ClrMD dlcloses the DAC and glibc UNMAPS it — but the
+/// pthread key is never deleted, and the per-thread values survive on every thread that
+/// touched the DAC. Minutes later, when any such thread exits (ordinary ThreadPool
+/// retirement — under CI GC pressure, or a dedicated compile thread ending), glibc's
+/// <c>__nptl_deallocate_tsd</c> invokes the registered destructor: an instruction fetch
+/// into unmapped (or, worse, reused) pages → native SIGSEGV, <c>exit=139</c>, in whatever
+/// UNRELATED test happens to be running. The dump showed TWO dangling keys — one per
+/// ClrMD-using test class (<c>KernelScriptMemoryLeakTest</c>,
+/// <c>MeshHubDisposalLeakTest</c>) — each from a separate DAC load/unload cycle, with the
+/// faulting pointer resolving to DAC offset <c>0x2201b0</c> =
+/// <c>InternalEndCurrentThreadWrapper</c> and the TLS payload's vptr to
+/// <c>vtable for CorUnix::CPalThread</c>.</para>
+///
+/// <para><b>The invariant this enforces:</b> a shared library that registers
+/// process-global callbacks (pthread TLS destructors) must live as long as the process —
+/// the same reason <c>libcoreclr.so</c> itself is never unloaded. Taking one extra
+/// <see cref="NativeLibrary.Load(string)"/> reference before ClrMD ever loads the DAC
+/// gives the module <c>RTLD_NODELETE</c>-equivalent lifetime: ClrMD's own dlclose then
+/// merely drops the refcount to ours and the mapping — and with it every registered
+/// destructor and vtable — stays valid for any thread that exits later. Subsequent
+/// ClrMD sessions re-dlopen the SAME mapping (glibc refcounts by path), so no new keys
+/// dangle either.</para>
+///
+/// <para>NoStaticState note: the pin is a single immutable native HANDLE whose lifetime
+/// is deliberately the process (that is its entire purpose) — not a collection, not a
+/// cache, and there is nothing to clear between tests: the pthread-key registration it
+/// protects is itself process-global.</para>
+/// </summary>
+internal static class ClrMdDacPin
+{
+    private static IntPtr pinnedHandle;
+
+    /// <summary>
+    /// Idempotently takes the process-lifetime reference on the runtime's DAC. Call
+    /// BEFORE the first <c>DataTarget.CreateSnapshotAndAttach</c> so ClrMD's
+    /// load/unload cycle can never drop the module's refcount to zero.
+    /// </summary>
+    public static void EnsurePinned()
+    {
+        if (Volatile.Read(ref pinnedHandle) != IntPtr.Zero)
+            return;
+
+        var dacFileName =
+            OperatingSystem.IsWindows() ? "mscordaccore.dll" :
+            OperatingSystem.IsMacOS() ? "libmscordaccore.dylib" :
+            "libmscordaccore.so";
+        var dacPath = Path.Combine(RuntimeEnvironment.GetRuntimeDirectory(), dacFileName);
+        if (!File.Exists(dacPath))
+            return; // self-contained/trimmed host without a DAC — nothing ClrMD could load either
+
+        var handle = NativeLibrary.Load(dacPath);
+        // First pin wins; a concurrent second load just returns the same module with an
+        // extra refcount — release ours, the winner's reference is the process pin.
+        if (Interlocked.CompareExchange(ref pinnedHandle, handle, IntPtr.Zero) != IntPtr.Zero)
+            NativeLibrary.Free(handle);
+    }
+}

test/MeshWeaver.Hosting.Monolith.Test/ClrMdDacUnloadCrashTest.cs

@@ -0,0 +1,85 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using Microsoft.Diagnostics.Runtime;
+using Xunit;
+
+namespace MeshWeaver.Hosting.Monolith.Test;
+
+/// <summary>
+/// 🚨 Deterministic repro + pin for the endemic <c>Hosting.Monolith.Test exit=139</c>
+/// teardown SIGSEGV (main flaking red since 2026-07-15; survived #467, #483 and #489).
+///
+/// <para><b>Root cause (named from the corruption-time core of CI run 29471199052 and
+/// probe run 29482327524 — identical fault signature in both):</b> ClrMD's
+/// <c>DataTarget.CreateSnapshotAndAttach</c> dlopens the DAC (<c>libmscordaccore.so</c>)
+/// in-process; the DAC's embedded PAL registers a process-global pthread TLS destructor
+/// (<c>pthread_key_create(&amp;key, InternalEndCurrentThreadWrapper)</c>) pointing into
+/// the DAC's own code, and stores a <c>CorUnix::CPalThread*</c> in that key on every
+/// thread that runs DAC code. Disposing the <c>DataTarget</c> dlcloses and UNMAPS the
+/// DAC without deleting the key. Any poisoned thread that exits later (ThreadPool
+/// retirement under GC pressure — which the ALC-unload GC probe's forced collections
+/// amplify, hence the false "ALC unload" lead) has glibc's
+/// <c>__nptl_deallocate_tsd</c> call the dangling destructor: instruction fetch into
+/// unmapped/reused pages → native SIGSEGV in whatever unrelated test is running.
+/// Fault context proof: <c>RIP == CR2</c> at DAC offset <c>0x2201b0</c>
+/// (= <c>InternalEndCurrentThreadWrapper</c>), page-fault error 0x15 (user-mode
+/// instruction fetch, protection violation), TLS payload vptr at DAC offset
+/// <c>0x243a78</c> (= <c>vtable for CorUnix::CPalThread</c>), and one dangling key per
+/// ClrMD-using test class in <c>__pthread_keys</c>.</para>
+///
+/// <para><b>This test IS the crash, made deterministic:</b> it runs one ClrMD snapshot
+/// session on a dedicated thread and then EXITS that thread — the exact
+/// destructor-at-thread-exit path that randomly killed CI hosts. Without
+/// <see cref="ClrMdDacPin"/> the process dies with SIGSEGV right at
+/// <c>Join()</c>; with the pin the DAC stays mapped, the destructor executes real code,
+/// and the mapping assertion documents the invariant readably.</para>
+/// </summary>
+public class ClrMdDacUnloadCrashTest
+{
+    [Fact(Timeout = 120_000)]
+    public void ThreadThatRanAClrMdSnapshot_MustExitSafely_AfterDataTargetDisposal()
+    {
+        Assert.SkipUnless(OperatingSystem.IsLinux(),
+            "ClrMD snapshot attach (createdump) + glibc __nptl_deallocate_tsd are the Linux CI " +
+            "crash path; /proc/self/maps is the Linux view of the mapping invariant.");
+
+        // THE FIX under test: take the process-lifetime reference BEFORE ClrMD's first
+        // DAC load, so its Dispose-time dlclose can never unmap the module.
+        ClrMdDacPin.EnsurePinned();
+
+        Exception? failure = null;
+        var thread = new Thread(() =>
+        {
+            try
+            {
+                // Loads the DAC in-process; the heap read below executes DAC code ON THIS
+                // THREAD, which plants a CPalThread* in the DAC-registered pthread TLS key.
+                using var dataTarget = DataTarget.CreateSnapshotAndAttach(Environment.ProcessId);
+                using var runtime = dataTarget.ClrVersions[0].CreateRuntime();
+                if (runtime.Heap.Segments.Count() == 0)
+                    throw new InvalidOperationException("snapshot heap walk saw no segments — the DAC was never exercised");
+            }
+            catch (Exception ex)
+            {
+                failure = ex;
+            }
+            // Disposals above dropped ClrMD's own DAC reference. This thread now EXITS with
+            // a live value in the DAC's pthread key → glibc __nptl_deallocate_tsd invokes
+            // the registered destructor. Unpinned, the DAC is unmapped by now and the call
+            // is an instruction fetch into freed pages → native SIGSEGV (exit=139) — the
+            // CI crash, deterministically. Pinned, the destructor is real code and frees
+            // the CPalThread as designed.
+        });
+        thread.Start();
+        thread.Join();
+
+        Assert.Null(failure);
+
+        // The invariant, in readable form: the DAC must still be mapped after ClrMD
+        // released its handle — its pthread-key registration is process-global and
+        // irrevocable, so the module's lifetime must be the process's.
+        Assert.Contains("libmscordaccore", File.ReadAllText("/proc/self/maps"));
+    }
+}

test/MeshWeaver.Hosting.Monolith.Test/KernelScriptMemoryLeakTest.cs

@@ -144,6 +144,11 @@ private static (bool Pinned, string Report) AnalyzeScriptRoots()
         var pinned = false;
         try
         {
+            // 🚨 Pin the DAC for process lifetime BEFORE ClrMD loads it: DataTarget.Dispose
+            // otherwise dlcloses libmscordaccore.so while its PAL's process-global pthread-key
+            // destructor still points into it → any later thread exit SIGSEGVs the host
+            // (the endemic exit=139). See ClrMdDacPin / ClrMdDacUnloadCrashTest.
+            ClrMdDacPin.EnsurePinned();
             using var dt = DataTarget.CreateSnapshotAndAttach(Environment.ProcessId);
             if (dt.ClrVersions.Length == 0) return (false, "[clrmd] no CLR runtime found in snapshot");
             using var runtime = dt.ClrVersions[0].CreateRuntime();

test/MeshWeaver.Hosting.Monolith.Test/MeshHubDisposalLeakTest.cs

@@ -130,6 +130,11 @@ private static (bool StaticRooted, string Report) AnalyzeMeshHubRoots()
         var staticRooted = false;
         try
         {
+            // 🚨 Pin the DAC for process lifetime BEFORE ClrMD loads it: DataTarget.Dispose
+            // otherwise dlcloses libmscordaccore.so while its PAL's process-global pthread-key
+            // destructor still points into it → any later thread exit SIGSEGVs the host
+            // (the endemic exit=139). See ClrMdDacPin / ClrMdDacUnloadCrashTest.
+            ClrMdDacPin.EnsurePinned();
             using var dt = DataTarget.CreateSnapshotAndAttach(Environment.ProcessId);
             if (dt.ClrVersions.Length == 0) return (false, "[clrmd] no CLR runtime found in snapshot");
             using var runtime = dt.ClrVersions[0].CreateRuntime();