From 3cfdc16364d02d2bdd52c36ab73821e8d287bab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= Date: Tue, 28 Jul 2026 05:23:51 +0100 Subject: [PATCH] =?UTF-8?q?Pre-warm=20one=20type=20at=20a=20time=20?= =?UTF-8?q?=E2=80=94=204-at-once=20is=20the=20storm=20that=20keeps=20it=20?= =?UTF-8?q?disabled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The boot warm-up already exists and already solves the problem it was built for: front-load the cold per-type compile so the first visitor does not wait for it. It ships DISABLED, and its own summary says why — enumerating every NodeType and activating them together is "a boot-time subscribe/compile storm that starves the hubs", the 2026-07-22 "I have been told I am dead" restart loop. That storm is a concurrency choice, not an inherent property. DefaultMaxConcurrency was 4. I reproduced exactly that shape by accident on 2026-07-28 04:05 — four compiles fired at memex in quick succession while gathering diagnostics — and within minutes SIX plugin roots (Claims, Edu, Underwriting, Chess, Publish, Training) fell to the "did not settle" overlay and needed a scale-to-zero to recover. Concurrency buys nothing here: the compiles already serialize on the Compile IoPool, so 4 adds only simultaneous cold ACTIVATIONS — and activation is the expensive part. Issue #686 measured it: the same NodeType shape, same image, compiles in 109ms with 0ms discovery on a fresh mesh versus 45.20s of discovery on memex. The difference is cold cross-partition hubs, which is precisely what warming exists to pay off, and precisely what doing four at once turns into an outage. So: default concurrency 1, plus a 2s stagger between types so back-to-back cold activations do not read as a burst. The sweep has no deadline and lazy compilation still covers correctness, so a trickle costs nothing. This is what makes the warm-up safe to actually turn on (PreWarm:DynamicTypes) — which is the real fix for a stall that only ever bites the first request after a deploy. Pinned by DefaultConcurrency_IsOne_AndTypesArePaced, because the damage only appears at production scale where no test runs. 2/2 DynamicTypePreWarmerTest pass. Co-Authored-By: Claude Opus 5 (1M context) --- .../DynamicTypePreWarmer.cs | 37 +++++++++++++++++-- .../DynamicTypePreWarmerTest.cs | 27 ++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/MeshWeaver.Hosting/DynamicTypePreWarmer.cs b/src/MeshWeaver.Hosting/DynamicTypePreWarmer.cs index d31f863ff4..152adb37ae 100644 --- a/src/MeshWeaver.Hosting/DynamicTypePreWarmer.cs +++ b/src/MeshWeaver.Hosting/DynamicTypePreWarmer.cs @@ -55,9 +55,32 @@ public record PreWarmOutcome(string TypePath, PreWarmStatus Status, string? Deta /// public static class DynamicTypePreWarmer { - /// How many dynamic NodeType hubs to activate concurrently. The compiles - /// themselves serialize on the Compile IoPool; this just caps in-flight activations. - public const int DefaultMaxConcurrency = 4; + /// + /// How many dynamic NodeType hubs to activate concurrently. ONE. + /// + /// 🚨 This was 4, and 4 is measurably harmful on a real mesh. On 2026-07-28 04:05 four + /// compiles were triggered on memex in quick succession — nothing to do with this warmer, but + /// the identical load shape — and within minutes SIX plugin roots (Claims, Edu, Underwriting, + /// Chess, Publish, Training) fell to the "did not settle" overlay and needed a scale-to-zero to + /// recover. That is the same storm this class's own summary warns about as the reason it ships + /// disabled, and the same one behind the 2026-07-22 "I have been told I am dead" restart + /// loop. + /// + /// The compiles already serialize on the Compile IoPool, so concurrency here buys nothing + /// except simultaneous cold ACTIVATIONS — which is exactly the expensive part: source discovery + /// on a large mesh is dominated by cold cross-partition hub activation (109ms and 0ms discovery + /// for the same NodeType shape on a fresh mesh, versus 45.20s on memex — issue #686). Warming + /// one at a time still removes the first-visitor stall; doing four at once trades a slow page + /// for an outage. + /// + public const int DefaultMaxConcurrency = 1; + + /// + /// Pause between types, so a long warm sweep stays a background trickle rather than a queue of + /// back-to-back cold activations. Cheap insurance: the sweep is a latency optimisation with no + /// deadline, so there is no reason for it to ever look like load. + /// + public static readonly TimeSpan BetweenTypes = TimeSpan.FromSeconds(2); /// Per-type warm budget — generous, because a cold Roslyn compile queued behind /// others on the Compile IoPool can legitimately take a while. On elapse we log @@ -120,7 +143,13 @@ public static IObservable WarmDynamicTypes( return typePaths.Length == 0 ? Observable.Empty() : typePaths - .Select(p => WarmOne(workspace, accessService, p, budget, logger)) + // Space the types out as well as capping concurrency. With + // maxConcurrency=1 the Merge already serializes them, but back-to-back + // cold activations still read as a burst to the hubs being woken; the + // delay makes the sweep a trickle. It costs nothing — the warm-up has + // no deadline and Part 2 compiles lazily regardless. + .Select((p, i) => WarmOne(workspace, accessService, p, budget, logger) + .DelaySubscription(TimeSpan.FromTicks(BetweenTypes.Ticks * i))) .Merge(Math.Max(1, maxConcurrency)); }) .Catch(ex => diff --git a/test/MeshWeaver.Hosting.Monolith.Test/DynamicTypePreWarmerTest.cs b/test/MeshWeaver.Hosting.Monolith.Test/DynamicTypePreWarmerTest.cs index 47b8474fcf..db8a656475 100644 --- a/test/MeshWeaver.Hosting.Monolith.Test/DynamicTypePreWarmerTest.cs +++ b/test/MeshWeaver.Hosting.Monolith.Test/DynamicTypePreWarmerTest.cs @@ -81,4 +81,31 @@ await NodeFactory.CreateNode(new MeshNode("BrokenType", Partition) broken.Status.Should().Be(PreWarmStatus.CompileError, "a non-compiling type surfaces a bounded CompileError — never a hang, never blocking the good type"); } + + /// + /// 🚨 THE STORM GUARD. The warm-up must default to ONE activation at a time. + /// + /// It shipped defaulting to 4, and 4 is measurably harmful: on 2026-07-28 04:05 four + /// compiles fired at memex in quick succession — the identical load shape — and within minutes + /// SIX plugin roots fell to the "did not settle" overlay and needed a scale-to-zero. That is the + /// storm this class's own summary cites as the reason it ships disabled, and the same one behind + /// the 2026-07-22 "told I am dead" restart loop. + /// + /// Concurrency buys nothing here — the compiles already serialize on the Compile IoPool — + /// so the only thing it adds is simultaneous cold ACTIVATIONS, which is the expensive part + /// (issue #686: the same NodeType shape is 0ms discovery on a fresh mesh and 45.20s on memex). + /// Pinned as a value because the damage only shows up at production scale, where no test runs. + /// + [Fact] + public void DefaultConcurrency_IsOne_AndTypesArePaced() + { + DynamicTypePreWarmer.DefaultMaxConcurrency.Should().Be(1, + "warming several cold NodeType hubs at once is what knocks roots offline; the sweep has " + + "no deadline, so it must trickle"); + + DynamicTypePreWarmer.BetweenTypes.Should().BeGreaterThan(TimeSpan.Zero, + "back-to-back cold activations still read as a burst even at concurrency 1"); + DynamicTypePreWarmer.BetweenTypes.Should().BeLessThan(TimeSpan.FromSeconds(30), + "…but the sweep must still finish in a sensible time on a mesh with many types"); + } }