fix(ci): the arm lane merges with an App token, so main's push lanes run (#2916)

.github/workflows/auto-arm.yml

@@ -17,6 +17,29 @@
 #     that is the intended way to say "not ready", rather than withholding the arm.
 #   * forks are skipped: arming a PR from a fork would let an outside branch merge itself the
 #     moment the required set passes.
+#
+# 🚨🚨 THE ARM CREDENTIAL IS LEAD, NOT TRIM — it decides whether main's push lanes run at all.
+# This lane armed with `secrets.GITHUB_TOKEN` for six hours on 2026-09-01, and the consequence was
+# measured on #2916: GitHub performs an auto-merge as the identity that ARMED it, so the merge
+# landed as `github-actions[bot]` — and a push created with `GITHUB_TOKEN` deliberately does not
+# trigger workflow runs. main's HEAD therefore got NO `push`-event run of anything:
+#
+#   8cc1cc6e  merged by rbuergi (human)   Build and Test ✅  Chart Gate ✅  Hosting Operator ✅
+#   19628536  merged by github-actions[bot]                 (nothing)
+#   b3e6ae65  merged by github-actions[bot]                 (nothing)
+#   04d25efa  merged by github-actions[bot]                 (nothing)
+#   dc6c5a45  merged by github-actions[bot]                 (nothing)
+#
+# No push run ⇒ no `Consolidate test results` on main's HEAD ⇒ `main-cd`'s readiness gate reads
+# `absent/none` and waits, on every tick, forever ⇒ NOTHING IS EVER PROMOTED and every
+# self-updating install stays on the previous image. Four commits' worth of delivery was lost
+# before anyone noticed, because the reconciler cannot tell "CI has not run YET" from "CI will
+# NEVER run" — both read `absent/none`. That is the AGENTS.md skip-trapdoor one level down: a
+# SUPPRESSED TRIGGER renders exactly like a slow one.
+#
+# So the arm runs on a minted GitHub App installation token, whose merges are ordinary pushes that
+# trigger everything. Never put `secrets.GITHUB_TOKEN` back here — see
+# `ArmedMergeMustTriggerMainsPushLanesGuard`, which fails the build if anyone does.
 name: Arm auto-merge
 
 on:
@@ -41,9 +64,68 @@ jobs:
       github.event.pull_request.draft == false &&
       github.event.pull_request.head.repo.full_name == github.repository
     steps:
+      # No `if:` on any step that DECIDES anything below. The assert, the mint and the arm run
+      # unconditionally: they assert and fail, they do not choose whether to run. The ONE exemption
+      # is on the job, and it is a property of the EVENT (draft, fork), never a question about
+      # whether a credential happens to be set — invariant #3.
+      #
+      # The single `if:` further down is on a step that only PRINTS, and it is conditioned on a
+      # failure having already happened. A diagnostic that can turn nothing green is not a
+      # trapdoor — the distinction that matters is whether an `if:` can make a gate not run, and
+      # that one cannot.
+      - name: The App credential, or a red run naming what to provision
+        env:
+          MESHWEAVER_APP_ID: ${{ secrets.MESHWEAVER_APP_ID }}
+          MESHWEAVER_APP_PRIVATE_KEY: ${{ secrets.MESHWEAVER_APP_PRIVATE_KEY }}
+        run: |
+          missing=()
+          [ -n "$MESHWEAVER_APP_ID" ]          || missing+=("secrets.MESHWEAVER_APP_ID           app id of the org's meshweaver-cloud GitHub App, installed on every repo in the fleet")
+          [ -n "$MESHWEAVER_APP_PRIVATE_KEY" ] || missing+=("secrets.MESHWEAVER_APP_PRIVATE_KEY  that App's private key (PEM); source of truth is Key Vault Systemorph/github-app-privatekey")
+          if [ ${#missing[@]} -gt 0 ]; then
+            echo "::error::auto-arm cannot run — provision the following, then re-run:"
+            for m in "${missing[@]}"; do echo "  • $m"; done
+            exit 1
+          fi
+          echo "App credential present."
+
+      # 🚨 `gh pr merge --auto` is the `enablePullRequestAutoMerge` GraphQL mutation, so the token
+      # needs `Pull requests: write`; the merge it later performs needs `Contents: write`. Both are
+      # requested explicitly, so an installation that is missing either fails HERE — loudly, on a
+      # step whose name says what it wanted — rather than degrading to a token that arms fine and
+      # then merges in a way that starts no CI. There is deliberately no fallback: falling back to
+      # `secrets.GITHUB_TOKEN` is precisely the outage this file's header documents.
+      - name: Mint an installation token whose merges actually trigger workflows
+        id: arm-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ secrets.MESHWEAVER_APP_ID }}
+          private-key: ${{ secrets.MESHWEAVER_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: ${{ github.event.repository.name }}
+          permission-contents: write
+          permission-pull-requests: write
+
+      # PRESENT is not GRANTED. The mint above fails with a bare "not permitted" when the App is
+      # installed but lacks one of the two permissions, and that message names neither the App nor
+      # the grant — so say it here, where a reader lands. Diagnostics on a failed step, not a gate:
+      # this cannot make a red run green.
+      # 🚨 Scoped to the MINT's own outcome, not to `failure()` alone. A bare `if: failure()` also
+      # fires when the secret-presence assert above failed — the mint never ran — and would then
+      # print "could not mint an installation token" over a run whose actual problem was a missing
+      # secret it had just named correctly. Diagnostics that describe a step that did not execute
+      # are worse than none: they send the reader to the wrong half of the lane.
+      - name: Name the grant, if the mint could not get it
+        if: failure() && steps.arm-token.outcome == 'failure'
+        run: |
+          echo "::error::could not mint an installation token with Contents:write + PullRequests:write."
+          echo "The meshweaver-cloud App must carry BOTH. Check:"
+          echo "  gh api orgs/${{ github.repository_owner }}/installations --jq '.installations[] | \"\(.app_slug) \(.permissions|tojson)\"'"
+          echo "Grant 'Pull requests: Read and write' in the App's settings, then ACCEPT the new"
+          echo "permission on the org installation — a requested-but-unaccepted permission is not granted."
+
       - name: Arm it
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ steps.arm-token.outputs.token }}
           PR: ${{ github.event.pull_request.number }}
           REPO: ${{ github.repository }}
         run: |

src/MeshWeaver.Documentation/Data/Architecture/ContinuousDeliveryContract.md

@@ -163,6 +163,49 @@ the target as the tip of *this* repo's `main` through the API and re-check `Cons
 on it. A fork's code can never be the tip of `Systemorph/MeshWeaver`'s `main`, so the safety property
 is preserved by a stronger check rather than by a proxy.
 
+### 🚨 The third state the reconciler cannot see — a SUPPRESSED trigger
+
+The paragraph above distinguishes two readings of an absent conclusion: *running* and *never ran*.
+There is a third, and it is indistinguishable from the second by any amount of polling —
+**the push that would have started CI was suppressed, so the check will never exist.**
+
+GitHub performs an auto-merge as the identity that ARMED it, and **a push created with
+`GITHUB_TOKEN` deliberately does not trigger workflow runs** — the recursion guard that stops
+automation from re-triggering itself. So a lane that arms auto-merge with `secrets.GITHUB_TOKEN`
+produces merges that land normally and start *nothing*: no `MeshWeaver Build and Test`, no
+`Chart Gate`, no `Hosting Operator`. Measured on 2026-09-01 (#2916):
+
+| main commit | merged by | `push`-event runs |
+|---|---|---|
+| `8cc1cc6e` | rbuergi (human) | Build and Test, Chart Gate, Hosting Operator |
+| `19628536` · `b3e6ae65` · `04d25efa` · `dc6c5a45` | `github-actions[bot]` | **none** |
+
+`gh api repos/Systemorph/MeshWeaver/commits/dc6c5a45d/check-runs` answered `0`. The reconciler read
+`absent/none`, correctly declined to publish an untested tree, and waited — for six hours, across
+four commits, while every install stayed on the previous image and every dashboard stayed green.
+
+**This is the skip-trapdoor rule one level down.** `AGENTS.md` warns that GitHub paints a skipped
+job the same colour as a passed one; here a *suppressed trigger* renders exactly like a slow one.
+The absence is the symptom, and absence is the one thing polling cannot age out of, because
+"not yet" and "never" produce byte-identical readings.
+
+**The cure is at the credential, not at the gate.** Do not teach the reconciler to accept weaker
+evidence — the green-tree marker (`refs/ci-green/<tree>/<epoch>`) would let CD publish, but
+`Chart Gate` and `Hosting Operator` would stay silently dead, so that trades one invisible hole for
+two. The merge must simply be performed by an identity whose pushes trigger workflows: a minted
+GitHub App installation token (`actions/create-github-app-token`, requesting
+`permission-contents: write` **and** `permission-pull-requests: write` — arming is the
+`enablePullRequestAutoMerge` mutation, the merge itself writes to the branch). `auto-arm.yml` does
+this, fails RED naming the grant when the App lacks it, and never falls back;
+`ArmedMergeMustTriggerMainsPushLanesGuard` fails the build if anyone reintroduces `GITHUB_TOKEN`
+there.
+
+That guard reads configuration, which is normally the weak shape — deliberately, and the file says
+so. The outcome ("did the merge start main's push lanes?") is unobservable from a pull-request run
+by construction: it can only be seen on main, after the merge, and what you would inspect is
+precisely what is missing. The credential is the only place a pre-merge check can stand, and it is
+causal rather than correlated — with `GITHUB_TOKEN` the trigger is suppressed 100% of the time.
+
 ## The standing trap — verify the IMAGE, never the tick
 
 CD's `workflow_run` trigger reacts to a **real push**, and — more exactly — to one that

src/MeshWeaver.Documentation/Data/WhatsNew/2026-09-02-updates-reach-your-installation-again.md

@@ -0,0 +1,47 @@
+---
+Name: Updates reach your installation again
+Category: Fix
+Description: For six hours, every merge to the platform's main branch landed without starting the checks that release depends on, so nothing new was ever published and self-updating installations quietly stayed on an older build. Merges now start those checks again.
+Icon: ArrowSyncCheckmark
+Order: -20260902
+---
+
+# Updates reach your installation again
+
+An installation that keeps itself up to date does it by watching for a new published build. On
+2026-09-01, for about six hours, there were none — not because anything failed, but because nothing
+was ever offered.
+
+Work was landing normally the whole time. Changes were reviewed, tested and merged; the branch moved
+forward four times. What did not happen was the step *after* the merge: the platform's build-and-test
+run, which is the evidence the release process waits for before it publishes anything. The release
+process asked for that evidence on every check, did not find it, and did the correct thing — it
+waited. It waited on evidence that was never going to arrive.
+
+The cause was a credential, and it is worth stating plainly because the symptom points somewhere
+else entirely. A lane had recently been added to merge a change automatically once it went green, so
+that finished work would stop sitting around waiting for someone to press the button. That lane
+signed its merges with the automation's own built-in identity — and merges made with that identity
+deliberately do not start further automation. It is a sensible rule in general: it stops automation
+from triggering itself in a loop. Here it meant the merge landed and the checks that should have
+followed it simply never began.
+
+Nothing was red. There was no failed run to find, no error, no warning — only an absence, which is
+the hardest thing to notice. From the outside it looked exactly like a build that had not started
+*yet*.
+
+**The merge lane now signs with an identity whose merges behave like anyone else's**, so the checks
+run, the evidence appears, and publication proceeds. Three things were changed together so this
+cannot come back quietly:
+
+- If that identity is ever missing or under-privileged, the lane now **fails loudly and names
+  exactly what to provision** — rather than falling back to the credential that caused this.
+- A build-time check refuses any future attempt to merge with the built-in identity, with the
+  explanation attached.
+- The release process's own account of what it needs is written down, including how to tell "the
+  checks have not run yet" apart from "the checks will never run" — the distinction that made six
+  hours of silence look normal.
+
+If your installation looked stuck on an older version yesterday, it was not stuck: there was
+genuinely nothing newer to take. It will pick up the current build on its next check, with no action
+from you.

test/MeshWeaver.Documentation.Test/ArmedMergeMustTriggerMainsPushLanesGuard.cs

@@ -0,0 +1,138 @@
+#pragma warning disable CS1591
+
+using System;
+using System.IO;
+using System.Linq;
+using Xunit;
+
+namespace MeshWeaver.Documentation.Test;
+
+/// <summary>
+/// 🚨 A workflow that MERGES a pull request — or arms auto-merge on one — must never do it with
+/// <c>secrets.GITHUB_TOKEN</c>. GitHub performs an auto-merge as the identity that armed it, and a
+/// push created with <c>GITHUB_TOKEN</c> deliberately does not trigger workflow runs. The merge
+/// lands, and <b>main's entire <c>push</c> lane silently does not exist for that commit</b>.
+///
+/// <para><b>The measured outage (#2916, 2026-09-01).</b> <c>auto-arm.yml</c> shipped armed with
+/// <c>GITHUB_TOKEN</c>. Four consecutive merges landed as <c>github-actions[bot]</c> and produced
+/// no <c>push</c>-event run of anything — not <c>MeshWeaver Build and Test</c>, not
+/// <c>Chart Gate</c>, not <c>Hosting Operator</c>. No push run means no
+/// <c>Consolidate test results</c> check on main's HEAD, and <c>main-cd</c>'s readiness gate reads
+/// that as <c>absent/none</c> and waits — on every tick, forever. Nothing was promoted for six
+/// hours and every self-updating install stayed on the previous image, while every dashboard was
+/// green: the last commit that reached the image jobs was the last one a human had merged.</para>
+///
+/// <para><b>Why this guard reads CONFIGURATION, which is normally the weak shape.</b> AGENTS.md is
+/// right that a check asserting configuration cannot see its outcome fail, and this file is a
+/// deliberate exception rather than an oversight. The outcome here — "does merging this PR start
+/// main's push lanes?" — is unobservable from a pull-request run BY CONSTRUCTION: it can only be
+/// observed on main, after the merge, at which point the evidence that would have caught it is the
+/// very thing that is missing. There is no run to inspect, no skipped job, no pending check; the
+/// absence is the symptom. The credential is the only place a pre-merge check can stand. So the
+/// guard is honest about measuring the input, and the input is causal rather than correlated: with
+/// <c>GITHUB_TOKEN</c> the trigger is suppressed 100% of the time, by documented design.</para>
+///
+/// <para>The companion assertion is that the arm step still names a token at all — a step that
+/// simply dropped its <c>GH_TOKEN</c> would fall back to whatever the runner has and reintroduce
+/// the same failure by omission.</para>
+/// </summary>
+public class ArmedMergeMustTriggerMainsPushLanesGuard
+{
+    /// <summary>
+    /// The <c>gh</c> invocations that merge a PR or arm it to merge later. Each produces a commit
+    /// on the default branch attributed to whoever the token belongs to.
+    /// </summary>
+    private static readonly string[] MergingCommands =
+    [
+        "gh pr merge",
+        "enablePullRequestAutoMerge",
+    ];
+
+    private static string WorkflowsDir() =>
+        Path.Combine(FindRepoRoot(), ".github", "workflows");
+
+    /// <summary>
+    /// Executable lines only. A guard that matches comment prose is measuring the documentation,
+    /// not the workflow — and this repository's workflows explain themselves at length, including
+    /// by quoting the very token this file forbids.
+    /// </summary>
+    private static string[] ExecutableLines(string text) =>
+        text.Split('\n')
+            .Select(l => l.Trim())
+            .Where(l => l.Length > 0 && !l.StartsWith("#", StringComparison.Ordinal))
+            .ToArray();
+
+    [Fact]
+    public void NoWorkflowMergesOrArmsAPullRequestWithTheDefaultToken()
+    {
+        var offenders = Directory
+            .EnumerateFiles(WorkflowsDir(), "*.yml")
+            .Select(path => (path, lines: ExecutableLines(File.ReadAllText(path))))
+            .Where(w => w.lines.Any(l => MergingCommands.Any(c => l.Contains(c, StringComparison.Ordinal))))
+            .Where(w => w.lines.Any(l =>
+                l.Contains("GH_TOKEN", StringComparison.Ordinal) &&
+                l.Contains("secrets.GITHUB_TOKEN", StringComparison.Ordinal)))
+            .Select(w => Path.GetFileName(w.path))
+            .OrderBy(n => n, StringComparer.Ordinal)
+            .ToArray();
+
+        Assert.True(
+            offenders.Length == 0,
+            $"These workflows merge (or arm auto-merge on) a pull request using secrets.GITHUB_TOKEN: "
+            + $"{string.Join(", ", offenders)}.\n"
+            + "GitHub performs the merge as the arming identity, and a push created with GITHUB_TOKEN "
+            + "does not trigger workflow runs — so main's HEAD gets NO push-event run, no "
+            + "'Consolidate test results' check, and main-cd waits for evidence that can never "
+            + "arrive. Nothing is ever promoted and every self-updating install freezes (#2916).\n"
+            + "Mint a GitHub App installation token instead (actions/create-github-app-token, "
+            + "permission-contents: write + permission-pull-requests: write) and use that as GH_TOKEN.");
+    }
+
+    /// <summary>
+    /// The arm lane must name its credential explicitly. Deleting the <c>GH_TOKEN</c> line entirely
+    /// would pass the check above while recreating the outage — <c>gh</c> would fall back to
+    /// whatever the runner exposes.
+    /// </summary>
+    [Fact]
+    public void TheArmLaneNamesAMintedInstallationTokenExplicitly()
+    {
+        var path = Path.Combine(WorkflowsDir(), "auto-arm.yml");
+        Assert.True(File.Exists(path), $"{path} is missing — the arm lane is the subject of this guard.");
+
+        var lines = ExecutableLines(File.ReadAllText(path));
+
+        Assert.True(
+            lines.Any(l => l.Contains("actions/create-github-app-token", StringComparison.Ordinal)),
+            "auto-arm.yml no longer mints a GitHub App installation token. Its merges are then "
+            + "attributed to whatever identity remains, and if that is github-actions[bot] the "
+            + "resulting push starts no CI at all (#2916).");
+
+        Assert.True(
+            lines.Any(l =>
+                l.Contains("GH_TOKEN", StringComparison.Ordinal) &&
+                l.Contains("steps.arm-token.outputs.token", StringComparison.Ordinal)),
+            "auto-arm.yml's arm step does not pass the minted token as GH_TOKEN. Without it gh "
+            + "falls back to the runner's default credential — the #2916 failure, by omission "
+            + "rather than by choice.");
+
+        foreach (var permission in new[] { "permission-contents: write", "permission-pull-requests: write" })
+        {
+            Assert.True(
+                lines.Any(l => l.Contains(permission, StringComparison.Ordinal)),
+                $"auto-arm.yml's token mint no longer requests '{permission}'. Arming is the "
+                + "enablePullRequestAutoMerge mutation (Pull requests: write) and the merge it "
+                + "performs writes to the branch (Contents: write); requesting both explicitly is "
+                + "what makes a missing grant fail at the mint instead of somewhere downstream.");
+        }
+    }
+
+    private static string FindRepoRoot()
+    {
+        var dir = new DirectoryInfo(AppContext.BaseDirectory);
+        while (dir is not null && !Directory.Exists(Path.Combine(dir.FullName, ".github", "workflows")))
+            dir = dir.Parent;
+
+        Assert.NotNull(dir);
+        return dir!.FullName;
+    }
+}