fix(#3461): the publication pointer never moves BACKWARDS

.github/scripts/publish-bake-bundles.sh

@@ -1069,6 +1069,69 @@ publish_one_target() { # <account> <share> <dest-dir> <resealing>
 # sealed; the pointer is written AFTER that, precisely because it is what CHANGES once the set is
 # proven whole. Stamping it would claim it was covered by a postcondition that, by construction, ran
 # before it existed.
+# 🚨 THE POINTER MUST NEVER MOVE BACKWARDS, AND THE DECISION THAT LET THIS RUN PUBLISH IS ~90 s OLD.
+#
+# `publish_to_target` resolves LIVE once and asks the never-seal-backwards question there, against
+# the publication that was live THEN. Under the flat layout that staleness was covered: a sibling
+# that sealed in between wrote the SAME directory, so the byte-level postcondition found another
+# run's bytes and refused. Under the generation layout the two runs write DISJOINT directories, so
+# there is no mix to refuse — and `move_pointer` would then hand consumers this run's older content:
+#
+#   A (older content, slow)  resolve LIVE ──────── upload ──────── seal GA ── move _current → GA
+#   B (newer content, fast)      resolve LIVE ── upload ── seal GB ── _current → GB
+#                                                                        ↑ B is live … then A takes it back
+#
+# Readers never see a torn directory — they see a COMPLETE publication of older bytes, which is the
+# one failure this layout exists to prevent, reached the other way round. So the question is asked
+# AGAIN, here, immediately before the one-line write that changes what is served.
+#
+# 🚨 This is a postcondition, not mutual exclusion — the same thing `verify_publication` says about
+# itself, and for the same reason: there is no lease command under either storage group. What it
+# buys is that the window shrinks from a whole publication (~90 s) to the gap between this read and
+# the pointer upload, and that losing THAT race is no worse than the flat layout's behaviour today.
+#
+# An unorderable pair keeps today's behaviour (move) and SAYS so, exactly as the decision-time guard
+# does — refusing on an unanswerable comparison would mean a run whose content cannot be ordered
+# never becomes live at all.
+#
+# Returns 0 (true) when a NEWER publication became live while this run was in flight.
+pointer_moved_past_us() { # <account> <share> <dest>
+  local account="$1" share="$2" dest="$3" live_now published_now order
+  resolve_publication_dir "$account" "$share" "$dest"
+  live_now="$RESOLVED_DIR"
+  # No pointer at all (this run is migrating the prefix), or it already names THIS run's
+  # generation: there is nothing that could be newer, so there is nothing to order against.
+  [ "$live_now" != "$dest" ] || return 1
+  [ "${live_now##*/}" != "$PUBLICATION" ] || return 1
+  if [ -z "${SOURCE_SHA:-}" ]; then
+    # A framework-repo producer has no content identity, so "newer" is not defined for it; the
+    # framework identity IS the key and two runs under one identity publish the same surface.
+    return 1
+  fi
+  published_now=$(az storage file download --account-name "$account" --share-name "$share" \
+    --path "$live_now/$SOURCE_MARKER" --dest "$SENTINEL_LOCAL_DIR/pointer-$SOURCE_MARKER" \
+    --auth-mode login --backup-intent --only-show-errors > /dev/null 2>&1 \
+    && tr -d '[:space:]' < "$SENTINEL_LOCAL_DIR/pointer-$SOURCE_MARKER" || echo "")
+  rm -f "$SENTINEL_LOCAL_DIR/pointer-$SOURCE_MARKER"
+  if [ -z "$published_now" ] || [ "$published_now" = "unknown" ]; then
+    echo "::warning::$account/$share: $dest/$POINTER moved to '${live_now##*/}' while this run was in flight, but that generation records no readable $SOURCE_MARKER — this run's pointer move cannot be ordered against it, so it proceeds as before. A generation with no source marker cannot be protected from being replaced."
+    return 1
+  fi
+  order=""
+  if [ -n "${GH_TOKEN:-}" ] && [ -n "${BAKE_CONTENT_REPOSITORY:-}" ]; then
+    if ! order=$(gh api "repos/$BAKE_CONTENT_REPOSITORY/compare/$SOURCE_SHA...$published_now" --jq .status 2>"$SENTINEL_LOCAL_DIR/pointer-compare.err"); then
+      echo "::warning::the compare API refused to order $SOURCE_SHA against $published_now: $(tail -c 400 "$SENTINEL_LOCAL_DIR/pointer-compare.err" | tr '\n' ' ')"
+      order=""
+    fi
+  fi
+  if [ "$order" = "ahead" ]; then
+    echo "::notice title=Not moving the pointer backwards::$account/$share: while this run was publishing, '${live_now##*/}' became the live publication of $published_now, which is NEWER than this bake's $SOURCE_SHA and contains it. This run's generation $PUBLICATION is sealed and complete on the share, and is deliberately NOT pointed at — moving $POINTER to it would serve older bytes to every reader (MeshWeaver#3461, phase 4). Nothing is lost: the newer publication is live, and this content is contained in it."
+    return 0
+  fi
+  [ -n "$order" ] || echo "::warning::$account/$share: $dest/$POINTER moved to '${live_now##*/}' ($published_now) while this run was in flight and the pair could not be ordered against this bake's $SOURCE_SHA (no GH_TOKEN / BAKE_CONTENT_REPOSITORY, or the compare API refused) — moving the pointer as before. If that publication was newer, this run has just taken it back."
+  return 1
+}
+
 move_pointer() { # <account> <share> <dest>
   local account="$1" share="$2" dest="$3"
   printf '%s\n' "$PUBLICATION" > "$SENTINEL_LOCAL_DIR/$POINTER"
@@ -1128,6 +1191,19 @@ publish_publication() { # <account> <share> <dest> <live-was-sealed>
     --path "$dest/$SENTINEL" --auth-mode login --backup-intent --query exists -o tsv \
     --only-show-errors 2>/dev/null || echo false)
   if [ "$flat_resealing" != "true" ]; then flat_resealing=false; fi
+  # 🚨 Asked again here, ~90 seconds after `publish_to_target` asked it: a newer publication may
+  # have become live while this generation was being uploaded and verified. Pointing at this one
+  # would then serve OLDER bytes, whole and sealed — see pointer_moved_past_us.
+  if pointer_moved_past_us "$account" "$share" "$dest"; then
+    # The generation stays on the share, sealed and complete, named by nothing. That is exactly the
+    # state retention is defined over (unreachable ⇒ collectable once past the window), so it costs
+    # storage for at most the retention window and is never served.
+    # 🚨 The flat compatibility copy is deliberately NOT refreshed either: writing this run's older
+    # bytes there would do to pre-pointer readers precisely what the pointer refusal just declined
+    # to do to pointer-following ones.
+    echo superseded >> "$OUTCOMES"
+    return 0
+  fi
   move_pointer "$account" "$share" "$dest"
   # Recorded before the compatibility copy: the publication IS live at this point, for every reader
   # that follows the pointer. A refusal below leaves that true and still fails the target.
@@ -1366,6 +1442,12 @@ MARKERS=$(awk '/^marker$/ { c++ } END { print c + 0 }' "$OUTCOMES")
 # claims a seal this run did not write — and printed on every run, including zero, so the number
 # is a denominator rather than an occasional line.
 CONVERGED=$(awk '/^converged$/ { c++ } END { print c + 0 }' "$OUTCOMES")
+# Targets where a NEWER publication became live while this run was uploading, so this run's sealed
+# generation was deliberately not pointed at (MeshWeaver#3461, phase 4). Counted separately and
+# printed on every run, including zero, for the same reason `converged` is: the summary must never
+# report a serve this run did not make, and a number that only appears when non-zero is a number
+# nobody can use as a denominator.
+SUPERSEDED=$(awk '/^superseded$/ { c++ } END { print c + 0 }' "$OUTCOMES")
 if [ "${#FAILED[@]}" -gt 0 ]; then
   echo "::error::bake publication FAILED on ${#FAILED[@]} of $(printf '%s\n' $BAKE_PUBLISH_TARGETS | wc -l | tr -d ' ') target(s): ${FAILED[*]} — identity=$IDENTITY source=$SOURCE. Every OTHER target above was published and sealed; these were not. A target that no longer exists (a torn-down instance's share) belongs OUT of BAKE_PUBLISH_TARGETS — remove it, never route around it."
   exit 1
@@ -1406,4 +1488,4 @@ if [ -n "${BAKE_PUBLICATION_DIR:-}" ]; then
   materialise_publication "$BAKE_PUBLICATION_DIR"
 fi
 
-echo "bake published: identity=$IDENTITY arch=$BAKE_ARCHITECTURE source=$SOURCE source-sha=${SOURCE_SHA:-unknown} bundles=${#BUNDLES[@]} surface=$HAS_SURFACE targets-published=$PUBLISHED targets-converged=$CONVERGED release=${RELEASE_VERSION:-none} release-markers=$MARKERS"
+echo "bake published: identity=$IDENTITY arch=$BAKE_ARCHITECTURE source=$SOURCE source-sha=${SOURCE_SHA:-unknown} bundles=${#BUNDLES[@]} surface=$HAS_SURFACE targets-published=$PUBLISHED targets-converged=$CONVERGED targets-superseded=$SUPERSEDED release=${RELEASE_VERSION:-none} release-markers=$MARKERS"

.github/scripts/test-publish-bake-overlap.py

@@ -245,6 +245,45 @@ def emit(v):
 '''
 
 
+# ────────────────────────────── the stub `gh` (the compare API only) ──────────────────────────────
+# The publisher orders two source commits through `gh api repos/<repo>/compare/<base>...<head>` in
+# TWO places: the decision-time never-seal-backwards guard, and the pointer-move guard that repeats
+# it ~90 seconds later (MeshWeaver#3461 phase 4). Without a stub, `gh` is either absent or the real
+# CLI against a real repository, so both guards took their "could not order" arm and the cases below
+# would have measured the fallback rather than the guard.
+#
+# MOCK_GH_AHEAD holds the HEAD-side shas that are to be reported as NEWER than (and containing) the
+# base — i.e. `.status == "ahead"`, the real API's answer. MOCK_GH_FAIL makes the call fail the way
+# a token without access to a private repository does, which is the arm that measured 5 of 25 core
+# re-seals on 2026-09-13.
+STUB_GH = r'''#!/usr/bin/env python3
+import os, sys
+
+argv = sys.argv[1:]
+if not argv or argv[0] != "api":
+    sys.stderr.write("stub gh: only `gh api` is implemented\n")
+    sys.exit(2)
+if os.environ.get("MOCK_GH_FAIL"):
+    sys.stderr.write("gh: Not Found (HTTP 404)\n")
+    sys.exit(1)
+path = argv[1] if len(argv) > 1 else ""
+if "/compare/" not in path:
+    sys.stderr.write(f"stub gh: unsupported api path {path!r}\n")
+    sys.exit(2)
+pair = path.split("/compare/", 1)[1]
+base, _, head = pair.partition("...")
+ahead = os.environ.get("MOCK_GH_AHEAD", "").split()
+if head in ahead and head != base:
+    print("ahead")
+elif base in ahead and head != base:
+    print("behind")
+elif head == base:
+    print("identical")
+else:
+    print("diverged")
+'''
+
+
 # ────────────────────────────── the fake share backend ──────────────────────────────
 # Loaded by publish-bake-files.py from PUBLISH_BAKE_FAKE_SHARE_BACKEND, over the SAME filesystem
 # share and the SAME `.meta` sidecars the stub `az` reads and writes. Every second-publisher hook
@@ -457,6 +496,9 @@ def __init__(self, script: Path, workdir: Path):
         az = self.bin / "az"
         az.write_text(STUB_AZ)
         az.chmod(0o755)
+        gh = self.bin / "gh"
+        gh.write_text(STUB_GH)
+        gh.chmod(0o755)
         self.backend = workdir / "fake_share_backend.py"
         self.backend.write_text(FAKE_BACKEND)
 
@@ -1087,6 +1129,72 @@ def run_cases(script: Path, work: Path, expect_defect: bool) -> None:
               and len(s.under(s.pointer()).files()) == EXPECTED_FILES,
               f"_current={s.pointer()!r}: {denominator(s.under(s.pointer()))}")
 
+    # ── 🚨 AND THE POINTER MUST NOT MOVE BACKWARDS WHEN THE FINISH ORDER INVERTS THE CONTENT ORDER.
+    # ── The case above deliberately tolerates either generation winning, because for two runs of
+    # ── UNORDERED content either answer is whole and correct. This one is the ordered pair, and it
+    # ── is the one thing the generation layout can get silently wrong that the flat layout could
+    # ── not: under `flat` an overlap writes ONE directory, so the byte-level postcondition finds
+    # ── the other run's bytes and refuses; under `generation` the two write disjoint directories,
+    # ── there is no mix to refuse, and whichever run finishes LAST moves `_current` — which, when
+    # ── that is the run carrying OLDER content, hands every reader a complete, sealed publication
+    # ── of older bytes with nothing red anywhere.
+    # ──
+    # ── The decision-time never-seal-backwards guard cannot see it: it reads the publication that
+    # ── was live ~90 seconds earlier, before the newer run sealed. So the question is asked again
+    # ── immediately before the pointer write.
+    print("\ngeneration layout — the pointer never moves BACKWARDS (the newer run sealed first):")
+    h.reset()
+    older = Bake(work, "older-content", "e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1e1")
+    newer = Bake(work, "newer-content", "f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2")
+    tok_old, tok_new = "Systemorph-MeshWeaver-3701-1", "Systemorph-MeshWeaver-3702-1"
+    # The NEWER content publishes (and moves the pointer) while the OLDER run is still uploading.
+    inner = h.publish_command(newer, "Systemorph/MeshWeaver", "3702", {
+        **gen,
+        "BAKE_CONTENT_REPOSITORY": "Systemorph/MeshWeaver.Plugins",
+        "GH_TOKEN": "stub", "MOCK_GH_AHEAD": newer.source_sha,
+    })
+    r = h.publish(older, "Systemorph/MeshWeaver", "3701", {
+        "MOCK_AZ_HOOK_ON": f"{DEST}/{tok_old}/{BUNDLES[1]}",
+        "MOCK_AZ_HOOK_WHEN": "before",
+        "MOCK_AZ_HOOK_ONCE": work / "fired-backwards",
+        "MOCK_AZ_HOOK_CMD": inner,
+        "BAKE_CONTENT_REPOSITORY": "Systemorph/MeshWeaver.Plugins",
+        "GH_TOKEN": "stub", "MOCK_GH_AHEAD": newer.source_sha,
+        **gen,
+    })
+    s = h.shelf()
+    check("the fixture really did invert the order (both generations exist — not vacuous)",
+          sorted(s.generations()) == sorted([tok_old, tok_new]),
+          f"generations={s.generations()}")
+    check("…and the newer run really did seal first, so there was something to take back",
+          s.under(tok_new).sealed(), f"{tok_new}: {denominator(s.under(tok_new))}")
+    if expect_defect:
+        check("PRE-FIX: the older run moved the pointer onto its own publication, so every reader "
+              "serves a COMPLETE, SEALED publication of older bytes and nothing is red",
+              r.returncode == 0 and s.pointer() == tok_old,
+              f"rc={r.returncode}, _current={s.pointer()!r}")
+    else:
+        check("the pointer is left on the NEWER publication",
+              s.pointer() == tok_new and s.under(tok_new).sealed()
+              and len(s.under(tok_new).files()) == EXPECTED_FILES,
+              f"_current={s.pointer()!r}: {denominator(s.under(tok_new))}")
+        check("…and the older run still SUCCEEDS — its content is contained in what is live, so "
+              "this is not a failure to report",
+              r.returncode == 0, f"rc={r.returncode}")
+        check("…and says so, naming both commits rather than leaving a silent no-op",
+              "Not moving the pointer backwards" in r.stdout
+              and older.source_sha in r.stdout and newer.source_sha in r.stdout,
+              "a pointer that was deliberately not moved must be readable as a decision")
+        check("…and the summary counts it, on every run, as its own denominator",
+              "targets-superseded=1" in r.stdout,
+              "a run that published nothing readers see must never be counted as having published")
+        check("the older run's generation is sealed and complete, just named by nothing",
+              s.under(tok_old).sealed() and len(s.under(tok_old).files()) == EXPECTED_FILES,
+              f"{tok_old}: {denominator(s.under(tok_old))} — retention collects it once it is past the window")
+        check("the FLAT compatibility copy was not refreshed with the older bytes either",
+              set(s.bakes_present()) - {"<marker>"} != {"older-content"},
+              f"flat: {denominator(s)} — pre-pointer readers must not be handed what the pointer refused")
+
     # ── The writer must decide "already published" from the POINTED-TO directory, not the prefix.
     # Getting this wrong is the silent one: the writer would read the flat compatibility copy while
     # portals served the generation, and republish (or skip) against a publication nobody serves.

.github/workflow-shell.allow

@@ -27,3 +27,17 @@
 # $LIVE — the generation that currently applies — rather than the prefix. Same call, same
 # diagnosis; only the line changed. MeshWeaver#3461.)
 swallow .github/scripts/publish-bake-bundles.sh 465523efa176
+
+# publish-bake-bundles.sh, pointer_moved_past_us: the SAME read as the entry above, taken a second
+# time immediately before the pointer write (MeshWeaver#3461, phase 4 — the pointer must never move
+# backwards). It has the same three normal absences: a generation sealed before source-marker
+# recording, a marker recorded as `unknown`, and a pointer that moved to a directory being written
+# right now. Letting the stderr through would print an Azure "ResourceNotFound" on every publish of
+# a prefix carrying such a generation.
+#
+# 🚨 And the empty answer is NOT authoritative in the permissive direction either — it is read as
+# "this pair cannot be ordered", which takes the SAME arm the decision-time guard takes and prints
+# a ::warning:: naming the generation and saying the pointer is being moved anyway. So an unreadable
+# marker degrades to today's behaviour, loudly, never to a silent skip and never to a silent
+# backwards move: the two lines below the read test for empty/unknown explicitly and warn.
+swallow .github/scripts/publish-bake-bundles.sh 6d43be64a517