Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 74 additions & 1 deletion src/MeshWeaver.Data/DataExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -834,7 +834,7 @@ internal static void MergePatchRecursive(
/// Without base values (legacy / one-off senders) it falls back to the
/// <see cref="DropStaleMonotonicTriggers"/> guard + plain <see cref="MergePatchRecursive"/>.
/// </summary>
private static void ApplyMeshNodeMerge(
internal static void ApplyMeshNodeMerge(
System.Text.Json.Nodes.JsonObject currentNode,
System.Text.Json.Nodes.JsonObject patchNode,
bool isMeshNode,
Expand All @@ -850,6 +850,14 @@ private static void ApplyMeshNodeMerge(
&& System.Text.Json.Nodes.JsonNode.Parse(baseText)
is System.Text.Json.Nodes.JsonObject baseValues)
{
// 🚨 Monotonic-trigger resolution BEFORE the three-way merge. The generic merge
// REFUSES every conflicting scalar (base ≠ live) — correct for non-monotonic
// fields, but for the strictly-increasing RequestedReleaseAt control trigger the
// conflict IS mergeable (newest instant wins). Without this, a FORWARD trigger
// written off a stale mirror base was silently refused and the user's compile
// request LOST (memex-cloud 2026-07-20 GitSync burst: every explicit Store/Plugin
// compile trigger was dropped while mirrors lagged the owner).
RebaseMonotonicTriggers(currentNode, patchNode, baseValues, jsonOpts, logger, hubPath);
Serialization.MeshNodePatchMerge.Apply(currentNode, patchNode, baseValues,
onRefuse: key => logger?.LogWarning(
"[MergeGuard] {HubPath}: refused stale/reordered cross-hub write to '{Key}' "
Expand Down Expand Up @@ -909,6 +917,71 @@ private static long NextMeshNodeVersion(
/// intentional write — so dropping it is the correct merge, not a band-aid. Scoped to MeshNode by
/// the caller; the field key is resolved through the same naming policy used elsewhere here.</para>
/// </summary>
/// <summary>
/// 🚨 Monotonic-trigger resolution for the THREE-WAY merge path (base values carried).
/// <see cref="Serialization.MeshNodePatchMerge.Apply"/> refuses EVERY conflicting scalar
/// (live changed since the writer's base) — the right default for non-monotonic fields
/// (Status, IsDirty, …) where two concurrent writes are not mergeable. But for the ONE
/// strictly-increasing control trigger — <c>NodeTypeDefinition.RequestedReleaseAt</c>, which
/// every writer sets to <c>DateTimeOffset.UtcNow</c> and which contractually never moves
/// backward — the scalar conflict IS mergeable: the newest instant wins, regardless of what
/// base the writer diffed against. Refusing the FORWARD case silently swallows a legitimate
/// compile trigger whenever the writer's mirror lags the owner (guaranteed during a GitSync
/// burst) — the memex-cloud 2026-07-20 incident where every explicit Store/Plugin compile
/// trigger was dropped. Resolution, applied to the parsed patch/base before the merge:
/// <list type="bullet">
/// <item>Patch BEHIND live → drop the key from the patch (the flap guard — identical
/// outcome to the refusal, but with the precise monotonic log line).</item>
/// <item>Patch AHEAD of live with a stale base → REBASE the writer's base value to the
/// live value so the three-way merge sees "unchanged since base" and lands the newer
/// trigger. This is the "make the flip a legal write — rebased on current state" fix,
/// not a bypass: only this one contractually-monotonic field gets it.</item>
/// </list>
/// </summary>
Comment on lines +920 to +940
internal static void RebaseMonotonicTriggers(
System.Text.Json.Nodes.JsonObject currentNode,
System.Text.Json.Nodes.JsonObject patchNode,
System.Text.Json.Nodes.JsonObject baseValues,
System.Text.Json.JsonSerializerOptions jsonOpts,
ILogger? logger,
string hubPath)
{
var contentKey = jsonOpts.PropertyNamingPolicy?.ConvertName("Content") ?? "Content";
if (patchNode[contentKey] is not System.Text.Json.Nodes.JsonObject patchContent
|| currentNode[contentKey] is not System.Text.Json.Nodes.JsonObject liveContent)
return;

var triggerKey = jsonOpts.PropertyNamingPolicy?.ConvertName("RequestedReleaseAt")
?? "RequestedReleaseAt";
if (!TryReadDateTimeOffset(patchContent, triggerKey, out var patchAt)
|| !TryReadDateTimeOffset(liveContent, triggerKey, out var liveAt))
return;

if (patchAt < liveAt)
{
logger?.LogWarning(
"[MergeGuard] {HubPath}: dropping stale/reordered {Key} patch ({PatchAt:o} < live {LiveAt:o}) — "
+ "a strictly-increasing release trigger must not move backward; keeping the live value.",
hubPath, triggerKey, patchAt, liveAt);
patchContent.Remove(triggerKey);
return;
}

if (patchAt > liveAt
&& baseValues[contentKey] is System.Text.Json.Nodes.JsonObject baseContent
&& baseContent.TryGetPropertyValue(triggerKey, out var baseVal)
&& baseVal is not null
&& !System.Text.Json.Nodes.JsonNode.DeepEquals(baseVal, liveContent[triggerKey]))
{
logger?.LogInformation(
"[MergeGuard] {HubPath}: rebasing monotonic trigger {Key} — patch ({PatchAt:o}) is AHEAD of "
+ "live ({LiveAt:o}) but the writer's base is stale; a strictly-increasing trigger merges "
+ "monotonically (newest wins) instead of being refused as a scalar conflict.",
hubPath, triggerKey, patchAt, liveAt);
baseContent[triggerKey] = liveContent[triggerKey]!.DeepClone();
}
}

internal static void DropStaleMonotonicTriggers(
System.Text.Json.Nodes.JsonObject currentNode,
System.Text.Json.Nodes.JsonObject patchNode,
Expand Down
5 changes: 5 additions & 0 deletions src/MeshWeaver.Data/Workspace.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@
// (ReferenceEquals on Workspace._remoteStreamCache) legitimately open the raw
// single-node reduce — they test the mechanism itself, not mesh-node access.
[assembly: InternalsVisibleTo("MeshWeaver.Query.Test")]
// Owner-side merge internals (ApplyMeshNodeMerge / RebaseMonotonicTriggers): the
// monotonic-trigger merge semantics are pinned by deterministic unit tests
// (MonotonicTriggerMergeTest) against the exact internal seam the
// PatchDataRequest handler runs — not a re-implementation of the call sequence.
[assembly: InternalsVisibleTo("MeshWeaver.Data.Test")]

namespace MeshWeaver.Data;

Expand Down
19 changes: 19 additions & 0 deletions src/MeshWeaver.Graph/Configuration/CompilationCacheOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,23 @@ public class CompilationCacheOptions
/// Default: 2000ms
/// </summary>
public int LockMaxRetryDelayMs { get; set; } = 2000;

/// <summary>
/// 🚨 Bound on the compile pipeline's ONE-SHOT source-snapshot reads
/// (<c>ResolveSources(...).Take(1)</c> over the shared
/// <c>NodeSources.GetSources</c> synced query). In every healthy state the
/// snapshot arrives instantly (the query is <c>Replay(1)</c>-cached) or after a
/// single cold storage read — so this bound only trips when the query's Initial
/// is genuinely lost (a synced-query subscription that raced a source-update
/// burst and never received its Initial: memex-cloud 2026-07-20, Store/Plugin).
/// Without the bound that lost Initial parked the compile FOREVER at
/// <c>CompilationStatus=Compiling</c> with no error and no recovery path — the
/// absorbing wedge (the compile watcher needs a Pending transition, the release
/// watcher gates on a settled status, the recovery kickoff is activation-one-shot).
/// On timeout the compile FAILS terminally (Status=Error with a loud message
/// naming the dead source query) so the state machine settles and a fresh
/// trigger / the Compile button can retry.
/// Default: 30 seconds.
/// </summary>
public TimeSpan SourceSnapshotTimeout { get; set; } = TimeSpan.FromSeconds(30);
}
46 changes: 38 additions & 8 deletions src/MeshWeaver.Graph/Configuration/MeshNodeCompilationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -385,8 +385,7 @@ private IObservable<IEnumerable<MeshNode>> GetSourceCollection(
private IObservable<DateTimeOffset> DiscoverSourceMaxLastModified(
NodeTypeDefinition? ntDef, string selfPath,
IReadOnlyList<MeshNode>? sourcesOverride = null) =>
ResolveSources(ntDef, selfPath, sourcesOverride)
.Take(1)
SnapshotSources(ntDef, selfPath, sourcesOverride)
.Select(nodes => nodes.Aggregate(
DateTimeOffset.MinValue,
(acc, n) => n.LastModified > acc ? n.LastModified : acc));
Expand All @@ -410,8 +409,7 @@ private IObservable<DateTimeOffset> DiscoverSourceMaxLastModified(
public IObservable<ImmutableDictionary<string, long>> DiscoverSourceVersionSnapshot(
NodeTypeDefinition? ntDef, string selfPath,
IReadOnlyList<MeshNode>? sourcesOverride = null) =>
ResolveSources(ntDef, selfPath, sourcesOverride)
.Take(1)
SnapshotSources(ntDef, selfPath, sourcesOverride)
.Select(nodes => nodes
.Where(n => !string.IsNullOrEmpty(n.Path))
.Aggregate(
Expand All @@ -433,6 +431,40 @@ private IObservable<IEnumerable<MeshNode>> ResolveSources(
return GetSourceCollection(ntDef, selfPath);
}

/// <summary>
/// 🚨 The ONE-SHOT source snapshot every compile stage reads — <see cref="ResolveSources"/>
/// bounded to exactly one emission with a hard completion guarantee. The underlying
/// <c>NodeSources.GetSources</c> synced query is a LIVE, never-completing collection; a bare
/// <c>.Take(1)</c> on it means "wait for the first emission, potentially forever". When the
/// query's Initial is genuinely lost (a subscription that raced a source-update burst — the
/// memex-cloud 2026-07-20 Store/Plugin GitSync burst), that unbounded wait parked the compile
/// at <c>CompilationStatus=Compiling</c> with NO error, NO terminal write and NO recovery
/// path: the compile watcher needs a Pending TRANSITION, the release watcher gates on a
/// SETTLED status, and the recovery kickoff is activation-one-shot — one hung snapshot wedged
/// the NodeType absorbing-forever (every later trigger no-oped; only a recycle re-rolled the
/// dice). Bounding the snapshot turns the lost Initial into a LOUD terminal failure: the
/// timeout propagates through <c>RunCompile</c>'s Catch to a
/// <c>CompilationStatus=Error</c> write naming the dead query, the park registry counts a
/// bounded (non-deterministic) failure, and a fresh <c>RequestedReleaseAt</c> trigger — or
/// the Compile button — retries cleanly. In every healthy state the snapshot is instant
/// (Replay(1) cache) or one cold storage read, far below the bound.
/// </summary>
private IObservable<IEnumerable<MeshNode>> SnapshotSources(
NodeTypeDefinition? ntDef, string selfPath, IReadOnlyList<MeshNode>? sourcesOverride)
{
var bound = _cacheOptions.SourceSnapshotTimeout;
return ResolveSources(ntDef, selfPath, sourcesOverride)
.Take(1)
.Timeout(bound)
.Catch<IEnumerable<MeshNode>, TimeoutException>(ex =>
Observable.Throw<IEnumerable<MeshNode>>(new TimeoutException(
$"Source snapshot for '{selfPath}' did not emit within {bound.TotalSeconds:0}s — "
+ $"the shared synced source query ('{NodeSources.CacheId(selfPath)}') never produced "
+ "its Initial (a lost/raced synced-query subscription, not a source-code error). "
+ "The compile fails terminally instead of parking at Compiling forever; retry via "
+ "the Compile button / a fresh RequestedReleaseAt.", ex)));
}

/// <summary>
/// IObservable end-to-end. Source discovery rides the cached SyncedQuery
/// registered for this NodeType; @@ include resolution composes via
Expand Down Expand Up @@ -460,8 +492,7 @@ private IObservable<IEnumerable<MeshNode>> ResolveSources(
// could pick up the pre-update Initial emission and the V2 compile would
// silently consume V1 source — that was the V1↔V2 mismatch root cause in
// CodeEditRecompileTest.
var discoverCodeFiles = ResolveSources(ntDef, selfPath, sourcesOverride)
.Take(1)
var discoverCodeFiles = SnapshotSources(ntDef, selfPath, sourcesOverride)
.Select(matches =>
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
Expand Down Expand Up @@ -922,8 +953,7 @@ private static string BuildSourceDiscoveryReport(IReadOnlyList<string> executedQ
string selfPath = selfDef != null ? node.Path : node.NodeType;

return resolveDef.SelectMany(ntDef =>
ResolveSources(ntDef, selfPath, sourcesOverride)
.Take(1)
SnapshotSources(ntDef, selfPath, sourcesOverride)
.SelectMany(matches =>
{
var pairs = CollectSourcePairs(matches);
Expand Down
27 changes: 26 additions & 1 deletion src/MeshWeaver.Graph/Configuration/NodeTypeCompilationHelpers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1207,7 +1207,25 @@ curr.Content is NodeTypeDefinition def
.Take(1)
.Select(result => new CompileOutcome(result, null, pendingNode, resolvedActivityPath))
.Catch<CompileOutcome, Exception>(ex =>
Observable.Return(new CompileOutcome(null, ex, pendingNode, resolvedActivityPath))))
Observable.Return(new CompileOutcome(null, ex, pendingNode, resolvedActivityPath)))
// 🚨 TOTALITY guard — this subscription is the ONLY component that can
// settle the Compiling status it just flipped, so its termination MUST
// produce exactly one outcome. An EMPTY completion (an upstream stage
// completed without emitting — the silent twin of the hung-snapshot
// wedge) previously fell through Subscribe(onNext, onError) with NO
// terminal write: the NodeType stayed Compiling forever and every later
// trigger no-oped (memex-cloud 2026-07-20, Store/Plugin). Synthesize a
// terminal error outcome so the state machine ALWAYS settles — the same
// completion-guard contract MeshQuery.MergeProviderObservables enforces
// for provider Initials ("every Query observable must emit exactly one
// Initial" ⇒ "every dispatched compile reaches exactly one terminal
// status").
.DefaultIfEmpty(new CompileOutcome(null, new InvalidOperationException(
$"Compile pipeline for '{hubPath}' completed without producing an outcome — "
+ "an upstream stage (source snapshot / include resolution / Roslyn bridge) "
+ "terminated empty. This is a framework defect surfaced as a terminal "
+ "compile failure so the NodeType cannot wedge at Compiling; retry via the "
+ "Compile button / a fresh RequestedReleaseAt."), pendingNode, resolvedActivityPath)))
.SubscribeOn(System.Reactive.Concurrency.TaskPoolScheduler.Default))
.Subscribe(
outcome =>
Expand Down Expand Up @@ -1267,6 +1285,13 @@ curr.Content is NodeTypeDefinition def

releasePathObservable
.Take(1)
// 🚨 Same totality guard as the compile pipeline above: the terminal
// Status write below runs in THIS OnNext — a release-create observable
// that completed empty would silently skip it and wedge the NodeType at
// Compiling. TryCreateReleaseNode is bounded (null on timeout/fault) by
// contract; DefaultIfEmpty makes the write unconditional even if that
// contract is ever violated.
.DefaultIfEmpty()
.Subscribe(newReleasePath =>
{
// Terminal writes run under System — same rule as every other
Expand Down
Loading
Loading