Background services ran their work on the mesh router — issue it off-router

src/MeshWeaver.ContentCollections.Indexing.Graph/MeshDocumentSink.cs

@@ -59,7 +59,10 @@ public IObservable<Unit> WriteDocument(DocumentInfo doc)
 
         // Cold: the side effect runs on Subscribe (driven by ContentIndexingService.WriteDocumentBranch).
         // FirstAsync() completes after the single response; map to Unit per the IDocumentSink contract.
-        return hub.Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node))
+        // Off-router issuing: the indexing pipeline can hold the DI root mesh hub — a target-less
+        // CreateOrUpdateNodeRequest posted there runs on the router (ROUTER_TRAFFIC).
+        return hub.NodeOperationIssuingHub()
+            .Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node))
             .FirstAsync()
             .SelectMany(delivery =>
             {

src/MeshWeaver.ContentCollections/ContentImportExtensions.cs

@@ -305,7 +305,12 @@ public IObservable<ImportContentResponse> Post()
         // Typed request-response: pre-registers the response callback by message-id BEFORE posting
         // (canonical hub.Observe<TResponse> idiom) — no manual Post returning a nullable delivery.
         // Wrapped in Defer so the post still happens on Subscribe (cold), as before.
-        return Observable.Defer(() => _hub
+        // 🚨 Issued off the router: mesh-singleton callers (the plugin default-install seed) hold
+        // the DI root mesh hub, and an ImportContentRequest posted there addresses its response
+        // straight back at mesh/{id} — the production ROUTER_TRAFFIC line "ImportContentResponse
+        // has the mesh hub as target (sender: Agent…)". NodeOperationIssuingHub is a no-op for
+        // every non-router hub, so node/import/portal-hub callers are unchanged.
+        return Observable.Defer(() => _hub.NodeOperationIssuingHub()
             .Observe(request, o => o.WithTarget(address))
             .Select(d => d.Message)
             .Take(1));
@@ -389,7 +394,9 @@ public IObservable<ImportContentResponse> Post()
             SourceOwnedPaths = _sourceOwnedPaths,
         };
         var address = new Address(_nodePath);
-        return Observable.Defer(() => _hub
+        // Off-router issuing, same reason as ContentImportBuilder.Post: the router must be neither
+        // end of the request/response pair (ROUTER_TRAFFIC); a non-router hub gets itself back.
+        return Observable.Defer(() => _hub.NodeOperationIssuingHub()
             .Observe(request, o => o.WithTarget(address))
             .Select(d => d.Message)
             .Take(1));

src/MeshWeaver.Documentation/Data/WhatsNew/2026-08-10-router-stays-a-router.md

@@ -0,0 +1,19 @@
+---
+Name: Background services no longer run their work on the mesh router
+Category: Fix
+Description: Plugin installs, log-incident filing and one-shot node reads now run on a dedicated work hub instead of the mesh router, keeping routing responsive and removing a steady stream of router-traffic error logs.
+Icon: Sparkle
+Order: -20260810
+---
+
+# Background services no longer run their work on the mesh router
+
+The mesh router's only job is to route messages — but several background services (the plugin
+default-install seed, log-incident filing, GitHub sync writes, and one-shot node reads) were
+issuing their work directly on it. Every such operation competed with routing itself and produced
+a steady stream of router-traffic error logs (hundreds per day on a busy portal).
+
+All of these now issue their work on the dedicated off-router work hub, so routing stays
+responsive under load and the error stream disappears. The router-traffic detector also no longer
+flags the router's own undeliverable-message notices, so the remaining reports point at genuine
+problems only.

src/MeshWeaver.GitSync/GitHubCredentialService.cs

@@ -70,7 +70,10 @@ public IObservable<GitHubCredential> Save(string userId, GitHubToken token, stri
         logger?.LogInformation("Saving GitHub credential for {User} (login={Login}, keyFp={Fp})",
             userId, gitHubLogin, Fingerprint(token.AccessToken));
 
-        return hub.Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node))
+        // Off-router issuing: this service holds the DI root mesh hub — a target-less
+        // CreateOrUpdateNodeRequest posted there runs on the router (ROUTER_TRAFFIC).
+        return hub.NodeOperationIssuingHub()
+            .Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node))
             .FirstAsync()
             .Select(d => d.Message)
             .SelectMany(resp => resp.Success

src/MeshWeaver.GitSync/GitHubSyncService.cs

@@ -840,7 +840,9 @@ private IObservable<MeshNode> WriteConfig(
         };
         // Carry the caller's identity (captured in UpdateConfig before the async ReadConfig hop) on
         // the create — otherwise RLS denies Create on {space}/_GitSync when the AsyncLocal is gone.
-        return hub.Observe<CreateOrUpdateNodeResponse>(
+        // Off-router issuing (NodeOperationIssuingHub): the DI root mesh hub must be neither end
+        // of the exchange, nor execute the target-less request (ROUTER_TRAFFIC).
+        return hub.NodeOperationIssuingHub().Observe<CreateOrUpdateNodeResponse>(
                 new CreateOrUpdateNodeRequest(node),
                 o => ctx is null ? o : o.WithAccessContext(ctx))
             .FirstAsync()

src/MeshWeaver.GitSync/GitHubWebhookProcessor.cs

@@ -420,9 +420,12 @@ private IObservable<int> ProcessWorkflowRun(JsonElement payload)
             repoUrl, completion.Branch, headSha, completion.WorkflowName, completion.RunNumber, path);
 
         var accessService = hub.ServiceProvider.GetRequiredService<AccessService>();
+        // Off-router issuing: the webhook processor holds the DI root mesh hub — a target-less
+        // CreateOrUpdateNodeRequest posted there runs on the router (ROUTER_TRAFFIC).
         return Observable.Using(
                 () => accessService.ImpersonateAsSystem(),
-                _ => hub.Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node)).FirstAsync())
+                _ => hub.NodeOperationIssuingHub()
+                    .Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node)).FirstAsync())
             .SelectMany(d =>
             {
                 if (!d.Message.Success)
@@ -490,9 +493,11 @@ private IObservable<MeshNode> UpsertFromWebhook(string space, GitHubIssue issue,
                 Content = merged,
             };
             var accessService = hub.ServiceProvider.GetRequiredService<AccessService>();
+            // Off-router issuing — same reason as RecordBuildCompletion above.
             return Observable.Using(
                     () => accessService.ImpersonateAsSystem(),
-                    _ => hub.Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node)).FirstAsync())
+                    _ => hub.NodeOperationIssuingHub()
+                        .Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node)).FirstAsync())
                 .SelectMany(d => d.Message.Success
                     ? Observable.Return(d.Message.Node ?? node)
                     : Observable.Throw<MeshNode>(new InvalidOperationException(

src/MeshWeaver.GitSync/IssueService.cs

@@ -184,7 +184,10 @@ private IObservable<MeshNode> UpsertIssueNode(string spacePath, GitHubIssue issu
         // The write runs as the calling user: AccessContext is an AsyncLocal that ExecutionContext
         // carries across the IoPool's ConfigureAwait(false) hops, so it is still set here even after
         // the repoClient round-trip. Return the server-reconciled node (version/normalization) on success.
-        return hub.Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node))
+        // Off-router issuing: this service holds the DI root mesh hub — a target-less
+        // CreateOrUpdateNodeRequest posted there runs on the router (ROUTER_TRAFFIC).
+        return hub.NodeOperationIssuingHub()
+            .Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(node))
             .FirstAsync()
             .Select(d => d.Message)
             .SelectMany(resp => resp.Success

src/MeshWeaver.Hosting/MeshService.cs

@@ -67,10 +67,7 @@ internal sealed class MeshService(
     /// this scoped service's lifetime.</para>
     /// </summary>
     private IMessageHub? _issuingHub;
-    private IMessageHub IssuingHub => _issuingHub ??=
-        string.Equals(hub.Address.Type, AddressExtensions.MeshType, StringComparison.Ordinal)
-            ? hub.NodeOperationExecutionHub() ?? hub
-            : hub;
+    private IMessageHub IssuingHub => _issuingHub ??= hub.NodeOperationIssuingHub();
 
     /// <summary>
     /// Per-call timeout ceiling. Every CRUD observable is bounded by this so a lost response

src/MeshWeaver.InstanceSync/InstanceSyncWorker.cs

@@ -463,9 +463,12 @@ private IObservable<bool> PullOne(IRemoteMeshClient remote, InstanceSyncConfig c
     private IObservable<MeshNode> ApplyLocal(MeshNode payload)
     {
         var accessService = hub.ServiceProvider.GetRequiredService<AccessService>();
+        // Off-router issuing: this worker holds the DI root mesh hub — a target-less
+        // CreateOrUpdateNodeRequest posted there runs on the router (ROUTER_TRAFFIC).
         return Observable.Using(
                 () => accessService.ImpersonateAsSystem(),
-                _ => hub.Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(payload)).FirstAsync())
+                _ => hub.NodeOperationIssuingHub()
+                    .Observe<CreateOrUpdateNodeResponse>(new CreateOrUpdateNodeRequest(payload)).FirstAsync())
             .SelectMany(d => d.Message.Success
                 ? Observable.Return(d.Message.Node!)
                 : Observable.Throw<MeshNode>(new InvalidOperationException(

src/MeshWeaver.Mesh.Contract/MeshExtensions.cs

@@ -319,6 +319,39 @@ private static bool IsCentralNodeOperationHub(IMessageHub hub)
     private static Address NodeOperationFallbackTarget(IMessageHub hub)
         => hub.NodeOperationExecutionHub()?.Address ?? hub.GetMeshHub().Address;
 
+    /// <summary>
+    /// The hub a node-operation request/response exchange should be ISSUED ON. The caller's own hub
+    /// — except when that hub is the ROOT MESH HUB, the mesh's ROUTER: a request posted there makes
+    /// the router an END of the delivery in both directions (the request goes out stamped
+    /// <c>Sender = mesh/{id}</c> and its response is addressed straight back at <c>mesh/{id}</c>),
+    /// which is exactly what the <c>ROUTER_TRAFFIC</c> detector reports — and, for a target-less
+    /// request, EXECUTES the work on the router's single-threaded action block, starving the
+    /// routing it exists to do.
+    ///
+    /// <para>This is the one shared seam for every mesh-singleton service that takes the DI-injected
+    /// <see cref="IMessageHub"/> (which in the mesh's root container IS the router) and issues
+    /// request/response work on it: the plugin-catalog boot services, the log-incident ingest, the
+    /// content importers, one-shot <c>GetMeshNode</c> reads. They all hop onto
+    /// <see cref="NodeOperationExecutionHub"/> — the mesh's dedicated off-router execution hub,
+    /// routing-registered so responses land on it cross-silo, sharing the mesh's type registry and
+    /// permission evaluator. For any hub that is NOT the router this returns the hub unchanged, so
+    /// portal/session/import-hub callers keep their identity byte-for-byte.</para>
+    ///
+    /// <para>Identity is unaffected: the ambient <c>AccessService</c> is the mesh-wide singleton
+    /// every hosted hub's provider chains to, so an <c>ImpersonateAsSystem</c> (or any ambient
+    /// context) active at Subscribe time is read identically by this hub's post pipeline.</para>
+    ///
+    /// <para>Falls back to the hub itself only while the mesh is tearing down
+    /// (<see cref="NodeOperationExecutionHub"/> returns <c>null</c>) — the historical behaviour, at
+    /// a point where the operation is being dropped anyway.</para>
+    /// </summary>
+    /// <param name="hub">The hub the caller holds — returned unchanged unless it is the root mesh hub.</param>
+    /// <returns>The off-router issuing hub.</returns>
+    public static IMessageHub NodeOperationIssuingHub(this IMessageHub hub) =>
+        string.Equals(hub.Address.Type, AddressExtensions.MeshType, StringComparison.Ordinal)
+            ? hub.NodeOperationExecutionHub() ?? hub
+            : hub;
+
     /// <summary>
     /// Registers handlers for mesh node operations. Idempotent — calling twice on the
     /// same configuration is a no-op on the second call. Without this guard, every

src/MeshWeaver.Mesh.Contract/MeshNodeStreamExtensions.cs

@@ -1432,7 +1432,9 @@ public static IObservable<MeshNode> UpdateMeshNode(this IWorkspace workspace,
     /// subscribed (no <c>.Take(1)</c>). See <c>Doc/Architecture/AsynchronousCalls.md</c>.
     /// </para>
     /// </summary>
-    /// <param name="hub">The hub that posts the read.</param>
+    /// <param name="hub">The hub the caller holds. When it is the ROOT MESH HUB (the router), the
+    /// read is issued on <see cref="MeshExtensions.NodeOperationIssuingHub"/> instead — the router
+    /// must be neither end of a delivery (ROUTER_TRAFFIC).</param>
     /// <param name="path">The mesh path to read.</param>
     /// <param name="timeout">Wall-clock budget for the read; defaults to 10 seconds.</param>
     /// <param name="onTimeout">
@@ -1447,6 +1449,16 @@ public static IObservable<MeshNode> UpdateMeshNode(this IWorkspace workspace,
         ReadTimeoutBehavior onTimeout = ReadTimeoutBehavior.Throw)
         => Observable.Create<MeshNode?>(observer =>
         {
+            // 🚨 Never issue the read on the ROOT MESH HUB — the router. Mesh-singleton services
+            // (plugin-catalog boot, log-incident ingest, credential resolvers) hold the DI-injected
+            // root hub, and a GetDataRequest posted there makes the router an END of the delivery in
+            // both directions: the request reaches the per-node hub stamped Sender = mesh/{id}, and
+            // the GetDataResponse (or, for a missing node, the DeliveryFailure) is addressed straight
+            // back at mesh/{id} — both exactly what the ROUTER_TRAFFIC detector reports (production
+            // 2026-08: "GetDataResponse has the mesh hub as target (sender: Hosting/_Access/…)" /
+            // "DeliveryFailure … (sender: Plugins/_DefaultInstallLedger)"). Same seam MeshService
+            // uses for CRUD; a non-router caller gets itself back, unchanged.
+            var issuingHub = hub.NodeOperationIssuingHub();
             var budget = timeout ?? TimeSpan.FromSeconds(10);
             var started = Stopwatch.StartNew();
             var cts = new CancellationTokenSource(budget);
@@ -1489,7 +1501,9 @@ void EmitError(Exception error)
                 if (Volatile.Read(ref emitted) != 0) return;
                 var elapsed = started.Elapsed;
                 string diagnostics;
-                try { diagnostics = hub.GetPendingRequestDiagnostics(); }
+                // The pending-request snapshot must come from the ISSUING hub — that is where our
+                // GetDataRequest's callback is registered and where a lost reply shows as pending.
+                try { diagnostics = issuingHub.GetPendingRequestDiagnostics(); }
                 catch (Exception diagEx) { diagnostics = $"<diagnostics unavailable: {diagEx.GetType().Name}>"; }
                 // …and the OWNER's state, which is what actually decides the verdict. The reader's
                 // snapshot alone proves only that the reader is innocent (idle queues + our request
@@ -1550,7 +1564,7 @@ void EmitError(Exception error)
                 // reply was dropped and this read hung to its timeout. That was the intermittent bulk flake
                 // in WorkspaceCacheEvictionTest (ReadNode -> GetMeshNode), proven deterministically by
                 // GetMeshNode_WarmOwner_DropsResponse_WhenSubjectRegisteredAfterPost.
-                innerSubscription = hub
+                innerSubscription = issuingHub
                     .Observe<GetDataResponse>(
                         new GetDataRequest(new MeshNodeReference()),
                         o => o.WithTarget(new Address(path)))

src/MeshWeaver.Messaging.Contract/RouterTrafficRule.cs

@@ -27,8 +27,17 @@ public static class RouterTrafficRule
     /// </param>
     /// <param name="senderAddressType">Address type of the sender, if any.</param>
     /// <param name="message">The message being delivered.</param>
+    /// <param name="isResponse">Whether the delivery ANSWERS a request (it carries a request-id
+    /// correlation). A response the router posts is routing's own duty — the undeliverable-mail
+    /// NACK (<c>RoutingServiceBase.PostNotFound</c> / <c>NackRouteFailure</c> post their
+    /// <c>DeliveryFailure</c> from the mesh hub via <c>ResponseFor</c>, so its sender is honestly
+    /// <c>mesh/{id}</c>) — not work, and reporting it trains people to mute the channel. Coverage
+    /// is not lost: real work SENT TO the router is still reported at request time via the
+    /// <c>"target"</c> role, and the payload type is opaque at the detector anyway (a routed NACK
+    /// arrives packed as <c>RawJson</c>), so the correlation marker is the structural signal.</param>
     /// <returns><c>"target"</c>, <c>"sender"</c>, <c>"sender AND target"</c>, or <c>null</c>.</returns>
-    public static string? RoleOf(string? targetAddressType, string? senderAddressType, object? message)
+    public static string? RoleOf(string? targetAddressType, string? senderAddressType, object? message,
+        bool isResponse)
     {
         // Heartbeats ARE the router's own job — routing liveness, not work. Type check, not a name
         // match: a rename must not silently turn this exclusion off.
@@ -38,6 +47,12 @@ public static class RouterTrafficRule
         var targetIsRouter = string.Equals(targetAddressType, AddressExtensions.MeshType, StringComparison.Ordinal);
         var senderIsRouter = string.Equals(senderAddressType, AddressExtensions.MeshType, StringComparison.Ordinal);
 
+        // The router ANSWERING (and not also being the target) is the routing NACK — its own job.
+        // A response ADDRESSED AT the router still reports: it proves the router issued a request,
+        // which is the violation the issuing seam (NodeOperationIssuingHub) exists to remove.
+        if (senderIsRouter && !targetIsRouter && isResponse)
+            return null;
+
         return (targetIsRouter, senderIsRouter) switch
         {
             (true, true) => "sender AND target",
@@ -46,4 +61,16 @@ public static class RouterTrafficRule
             _ => null,
         };
     }
+
+    /// <summary>
+    /// Binary-compatible 3-argument form — the pre-<c>isResponse</c> public signature, kept so
+    /// assemblies compiled against it keep resolving (this is a shipped Contract package).
+    /// Treats the delivery as a non-response, i.e. applies no NACK exclusion.
+    /// </summary>
+    /// <param name="targetAddressType">Address type of the delivery's TARGET.</param>
+    /// <param name="senderAddressType">Address type of the sender, if any.</param>
+    /// <param name="message">The message being delivered.</param>
+    /// <returns><c>"target"</c>, <c>"sender"</c>, <c>"sender AND target"</c>, or <c>null</c>.</returns>
+    public static string? RoleOf(string? targetAddressType, string? senderAddressType, object? message)
+        => RoleOf(targetAddressType, senderAddressType, message, isResponse: false);
 }

src/MeshWeaver.Messaging.Hub/MessageHub.cs

@@ -1278,7 +1278,11 @@ private void ReportRouterTraffic(IMessageDelivery delivery)
     {
         // The rule itself is a pure predicate — see RouterTrafficRule, where it is unit-tested.
         // Target = where the delivery is ADDRESSED, not where it is currently being handled.
-        var role = RouterTrafficRule.RoleOf(delivery.Target?.Type, delivery.Sender?.Type, delivery.Message);
+        // isResponse: the RequestId correlation marks a delivery that ANSWERS a request — the shape
+        // of the routing layer's own undeliverable-mail NACK, which posts from the mesh hub via
+        // ResponseFor (see RouterTrafficRule.RoleOf's isResponse doc).
+        var role = RouterTrafficRule.RoleOf(delivery.Target?.Type, delivery.Sender?.Type, delivery.Message,
+            delivery.Properties.ContainsKey(PostOptions.RequestId));
         if (role is null)
             return;