From dd2f99aca40edf5bb9d5be5802edb29e42936407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Sat, 10 Oct 2026 07:41:19 +0200 Subject: [PATCH] fix(create): a stalled validators stage names the validator it waits on (#6391) The create chain runs every INodeValidator in sequence, and its stalled verdict said only "stage 'validators'". On memex it fired ten times in three minutes on a freshly booted pod for System-written creates, where every core validator answers synchronously, so the reading could not say which validator was silent, or whether the chain stalled before any was asked. The stage now reads 'validators: resolving' while the scoped validators are constructed, then 'validators: (i/n)' from the moment the chain subscribes to each, on the verdict and the fate trail. Refs #6391 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Data/Architecture/WriteVerdictTotality.md | 16 +++++++++++- .../MeshExtensions.cs | 25 ++++++++++++++++--- ...ifecycleRequestsAlwaysReachAVerdictTest.cs | 3 +++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/MeshWeaver.Documentation/Data/Architecture/WriteVerdictTotality.md b/src/MeshWeaver.Documentation/Data/Architecture/WriteVerdictTotality.md index caf460996f..71cf209bed 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/WriteVerdictTotality.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/WriteVerdictTotality.md @@ -709,7 +709,7 @@ RequestTimeout"). | Handler | Bound | What the stalled verdict says | |---|---|---| -| Create | `Timeout` from handler entry, over the chain up to the written row | `Unavailable`, naming the stage: `authorship-source`, `existence-read`, `partition-bootstrap`, `write-guards`, `validators`, `nodetype-resolution`, `write` | +| Create | `Timeout` from handler entry, over the chain up to the written row | `Unavailable`, naming the stage: `authorship-source`, `existence-read`, `partition-bootstrap`, `write-guards`, `validators: (i/n)` (or `validators: resolving`), `nodetype-resolution`, `write` | | Create, after the write | the same deadline, as a VERDICT only — a post-creation handler still running is neither cancelled nor rolled back, because it may yet land its own writes | `Unavailable`: "written, a post-creation step had not finished, the outcome is unknown" — a later completion or compensation is not answered twice | | Copy | `Timeout` from handler entry | `Unknown` with the `Copy reached no verdict:` prefix, naming the stage and, for the create leg, the TARGET paths still outstanding | | Move | ONE move-wide deadline, `Timeout` from entry; its delete pre-flight's ladder moved one rung inside it (`NestedTimeout` stage, then leg, then absence probe) so the pre-flight can still name the silent descendant first | `Unavailable` naming the stage (`delete-preflight`, `copy`) when nothing at the source was touched; `Unknown` with "the copy had already landed" when it stalls in `delete-source`; a copy that ANSWERED a failure carries the copy's own reason and transcript, never re-derived from the wording | @@ -717,6 +717,20 @@ RequestTimeout"). Each stage is also stamped on the request's fate trail (`CREATE_STAGE …`, `COPY_STAGE …`), so a trail that ends early names the leg that was running instead of ending at `CREATE_CHAIN_SUBSCRIBED`. +**The validators stage names the validator it is waiting on** (MeshWeaver#6391). The create runs its +`INodeValidator`s one after another (`Concat`), so `stalled at stage 'validators'` named every +registered validator at once. On memex.systemorph.com it fired ten +times in three minutes on one freshly booted pod, for System-written creates +(`Admin/_Notification/…`, `Admin/PlatformVersion/_Activity/startup-…`, `Doc/_Activity/import-…`), +and every core validator answers SYNCHRONOUSLY for a System write — so the reading could not even say +whether a validator was silent or the chain stalled before any was asked. The stage now reads +`validators: resolving` while the scoped validators are constructed (DI), then +`validators: (i/n)` from the moment the chain subscribes to each one, on both the stalled +verdict and the trail. A stall in `resolving` is a blocked construction, not a silent validator; a +stall on a named validator is that validator's read. `LifecycleRequestsAlwaysReachAVerdictTest` +pins it: the silent validator's type name is in the verdict, and the assertion fails on the code +before the change, which said only `validators`. + **The deadline must release the hub after a prompt create.** Rx's absolute `Timeout` schedules a `LocalScheduler.WorkItem` that can remain in its priority queue until the deadline even after the subscription completes and disposes. Its fallback observable therefore captures only a small diff --git a/src/MeshWeaver.Mesh.Contract/MeshExtensions.cs b/src/MeshWeaver.Mesh.Contract/MeshExtensions.cs index 83c5dae0e0..74382b39c5 100644 --- a/src/MeshWeaver.Mesh.Contract/MeshExtensions.cs +++ b/src/MeshWeaver.Mesh.Contract/MeshExtensions.cs @@ -1292,7 +1292,13 @@ collision is not null return Observable.Return<(LocalizableText? Refusal, NodeCreationRejectionReason Reason)?>( (grantRejection, NodeCreationRejectionReason.ValidationFailed)); EnterCreateStage(CreateStageValidators); - return RunCreationValidatorsObs(hub, node, capturedRequest); + // 🚨 The stage is refined per validator (#6391): "validators" alone named + // every registered validator, and the core ones all answer synchronously + // for a System write, so a stalled verdict could not say which one — or whether the + // chain stalled before ANY validator was asked (resolving them). + return RunCreationValidatorsObs( + hub, node, capturedRequest, + onStage: detail => EnterCreateStage($"{CreateStageValidators}: {detail}")); }) .SelectMany(validationError => { @@ -6373,8 +6379,12 @@ internal static bool GovernedActivityExecuting( IMessageHub hub, MeshNode node, CreateNodeRequest request, - AccessContext? accessContext = null) + AccessContext? accessContext = null, + Action? onStage = null) { + // Named BEFORE the validators are resolved: resolving is DI construction of every scoped + // validator, and a construction that blocks would otherwise read as a silent validator. + onStage?.Invoke("resolving"); var accessService = hub.ServiceProvider.GetService(); var context = new NodeValidationContext { @@ -6392,8 +6402,17 @@ internal static bool GovernedActivityExecuting( if (validators.Count == 0) return Observable.Return<(LocalizableText?, NodeCreationRejectionReason)?>(null); + // Each validator is named at the moment Concat SUBSCRIBES to it (Defer), i.e. when it is + // the one the chain is waiting on — so a stalled verdict reads + // "validators: RlsNodeValidator (i/n)", never a bare stage name (#6391). return validators - .Select(v => v.Validate(context)) + .Select((v, i) => onStage is null + ? v.Validate(context) + : Observable.Defer(() => + { + onStage($"{v.GetType().Name} ({i + 1}/{validators.Count})"); + return v.Validate(context); + })) .Concat() .Where(result => !result.IsValid) .Select(result => diff --git a/test/MeshWeaver.Hosting.Test/LifecycleRequestsAlwaysReachAVerdictTest.cs b/test/MeshWeaver.Hosting.Test/LifecycleRequestsAlwaysReachAVerdictTest.cs index 5ffe779920..614690a2b6 100644 --- a/test/MeshWeaver.Hosting.Test/LifecycleRequestsAlwaysReachAVerdictTest.cs +++ b/test/MeshWeaver.Hosting.Test/LifecycleRequestsAlwaysReachAVerdictTest.cs @@ -76,6 +76,9 @@ public async Task ACreateWhoseValidatorNeverAnswers_IsAnsweredUnavailable_Naming response.Error.Should().Contain(path); response.Error.Should().Contain("validators", "the verdict names the stage that went silent, which is what the trail could not say"); + response.Error.Should().Contain($"validators: {nameof(SilentCreationValidator)} (", + "the verdict names the VALIDATOR the chain was waiting on — 'validators' alone named " + + "every registered validator at once (#6391)"); silent.Asked.Should().Contain(path, "the stall really was the armed validator"); (await StoredAt(path, ct)).Should().BeNull("the stall came before the write"); }