IanFrelinger · Sep 29, 2026 · Sep 29, 2026
@@ -138,10 +138,13 @@ jobs:
             exit 1
           fi
 
+  # Ordered AFTER nuget on purpose. GHCR tags can be re-pointed or deleted; nuget.org packages
+  # can only be unlisted. Running the two in parallel let `:0.2.0` go permanent on one side
+  # while the other failed - which is how v0.1.2 carried a semver image tag and a red NuGet job.
   images:
     name: Container images (GHCR)
-    needs: [validate]
-    if: needs.validate.result == 'success'
+    needs: [validate, nuget]
+    if: needs.validate.result == 'success' && needs.nuget.result == 'success'
     uses: ./.github/workflows/reusable-container-publish.yml
     secrets: inherit
     with:
@@ -248,10 +251,17 @@ jobs:
   github-release:
     name: Draft GitHub Release
     needs: [validate, images, nuget]
+    # always(), gated on publication having BEGUN rather than on everything having succeeded.
+    # When a post-push check failed, `success()` skipped this job and the only copy of the
+    # nupkgs and nuget-publish-manifest.json was a 90-day Actions artifact - for packages that
+    # are permanent. That is how the v0.1.2 release page came to be assembled by hand. The
+    # release is created as a DRAFT, so running here after a failure publishes nothing.
     if: >-
-      success() &&
+      always() &&
       github.ref_type == 'tag' &&
-      vars.RELEASE_CREATE_GITHUB_RELEASE != 'false'
+      vars.RELEASE_CREATE_GITHUB_RELEASE != 'false' &&
+      needs.validate.result == 'success' &&
+      needs.nuget.outputs.publish_began == 'true'
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -266,6 +276,10 @@ jobs:
         shell: bash
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NUGET_RESULT: ${{ needs.nuget.result }}
+          IMAGES_RESULT: ${{ needs.images.result }}
+          CLI_SHA_TAG: ${{ needs.images.outputs.cli_sha_tag }}
+          API_SHA_TAG: ${{ needs.images.outputs.api_sha_tag }}
         run: |
           TAG="${GITHUB_REF_NAME}"
           if gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1; then
@@ -277,11 +291,21 @@ jobs:
           SNIP="$(mktemp)"
           bash scripts/changelog-snippet-for-release.sh > "$SNIP" || true
           {
+            if [ "${NUGET_RESULT}" != "success" ]; then
+              echo "> **This release did not complete cleanly.** NuGet publication began, but the"
+              echo "> nuget job ended \`${NUGET_RESULT}\` - see the run below. Packages may already be"
+              echo "> public on nuget.org. Do NOT publish this draft until that is resolved."
+              echo ""
+            fi
             echo "Automated draft for **${TAG}**."
             echo ""
             echo "- Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
             echo "- NuGet: \`Ashlar.Hosting.Bundle\` **${VER}** (+ graph). Packages, symbols, \`nuget-publish-manifest.json\` and per-package \`.sha256.txt\` are attached as release assets (also in the \`nuget-packages-${VER}\` workflow artifact)."
-            echo "- Images: \`${{ needs.images.outputs.cli_sha_tag }}\`, \`${{ needs.images.outputs.api_sha_tag }}\`"
+            if [ "${IMAGES_RESULT}" = "success" ]; then
+              echo "- Images: \`${CLI_SHA_TAG}\`, \`${API_SHA_TAG}\`"
+            else
+              echo "- Images: **not published** (images job ended \`${IMAGES_RESULT}\`; it runs only after NuGet succeeds)."
+            fi
             echo ""
             cat "$SNIP"
             echo ""

@@ -12,6 +12,9 @@ on:
       package_version:
         description: "Resolved semver without v prefix"
         value: ${{ jobs.pack-and-publish.outputs.package_version }}
+      publish_began:
+        description: "true once a push to nuget.org was attempted - i.e. something may now be public"
+        value: ${{ jobs.pack-and-publish.outputs.publish_began }}
 
 permissions:
   contents: read
@@ -22,6 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       package_version: ${{ steps.resolve.outputs.package_version }}
+      publish_began: ${{ steps.publish_began.outputs.began }}
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -184,6 +188,15 @@ jobs:
         with:
           user: ${{ secrets.NUGET_USER }}
 
+      # Recorded BEFORE the first push. The push loops below run per-file under -e, so a failure
+      # half way leaves some packages public; the caller uses this to keep the release artifacts
+      # durable in exactly that case, instead of treating "the job failed" as "nothing shipped".
+      - name: Record that irreversible publication is beginning
+        id: publish_began
+        if: vars.NUGET_PUBLISH_MODE == 'oidc' || vars.NUGET_PUBLISH_MODE == 'apikey'
+        shell: bash
+        run: echo "began=true" >> "$GITHUB_OUTPUT"
+
       - name: dotnet nuget push packages (OIDC)
         if: vars.NUGET_PUBLISH_MODE == 'oidc'
         shell: bash
@@ -234,6 +247,7 @@ jobs:
         env:
           ASHLAR_NUGET_VERIFY_VERSION: ${{ env.PACKAGE_VERSION }}
           ASHLAR_NUGET_VERIFY_PACKAGE_IDS: ${{ vars.NUGET_POST_PUSH_VERIFY_PACKAGE_IDS }}
+          ASHLAR_MANIFEST_JSON: ${{ runner.temp }}/nuget-release/nuget-publish-manifest.json
           ASHLAR_NUGET_VERIFY_ATTEMPTS: ${{ vars.NUGET_POST_PUSH_ATTEMPTS || '40' }}
           ASHLAR_NUGET_VERIFY_SLEEP_SEC: ${{ vars.NUGET_POST_PUSH_SLEEP_SEC || '15' }}
         run: bash scripts/verify-nuget-org-packages-visible.sh

@@ -79,6 +79,15 @@ Trusted Publishing: register **`release.yml`** and **`release-nuget.yml`** as ne
    alone — they stay true.
 
 7. **Promote the public API**: review each stable-tier project's `PublicAPI.Unshipped.txt`, move its lines into `PublicAPI.Shipped.txt`, commit on the release commit (`docs/SdkCompatibilityPolicy.md`, "Release step"). After the tag those lines are the promise.
+8. **Rehearse the publish with a prerelease before tagging.** Dispatch `release-nuget.yml` with
+   version `X.Y.Z-rc1` from the release commit. Both guards accept it -
+   `assert_dispatch_version_allowed` permits a prerelease of the canonical version, and
+   `assert_consumer_pin_matches` compares `base_semver`, so `0.2.0-rc1` matches a pin of `0.2.0`.
+   It runs pack, both pre-publish gates, the real OIDC push and every post-push check against
+   nuget.org. **Cost: one permanent `X.Y.Z-rc1` prerelease**, which no `X.Y.Z` pin resolves to.
+   Skip this and the tag is the first time the post-push chain runs against a live feed, on the
+   one trigger where a failure cannot be taken back. The staging-feed path does not substitute: the
+   post-push scripts target nuget.org, and `NUGET_STAGING_FEED_URL` is unset.
 
 ## After `release.yml`
 

@@ -2,7 +2,8 @@
 # Poll nuget.org flat container until all listed package versions return HTTP 200 (index lag after push).
 #
 # ASHLAR_NUGET_VERIFY_VERSION — required semver (no v prefix)
-# ASHLAR_NUGET_VERIFY_PACKAGE_IDS — comma-separated ids (default: Ashlar.Hosting.Bundle,Ashlar.Hosting,Ashlar.Sdk,Ashlar.CLI)
+# ASHLAR_NUGET_VERIFY_PACKAGE_IDS — comma-separated ids; overrides everything below
+# ASHLAR_MANIFEST_JSON — when set, poll EVERY id in the release manifest (the default in CI)
 # ASHLAR_NUGET_VERIFY_PACKAGE_ID — if set and ASHLAR_NUGET_VERIFY_PACKAGE_IDS unset, a single id (backward compat)
 # ASHLAR_NUGET_VERIFY_ATTEMPTS / ASHLAR_NUGET_VERIFY_SLEEP_SEC — optional (defaults 40 / 15)
 # v0.1.2: 12×15s timed out after every nupkg was pushed; nuget.org listed them later.
@@ -25,7 +26,14 @@ if [[ -n "${ASHLAR_NUGET_VERIFY_PACKAGE_IDS:-}" && "${ASHLAR_NUGET_VERIFY_PACKAG
   mapfile -t IDS < <(echo "${ASHLAR_NUGET_VERIFY_PACKAGE_IDS}" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | grep -v '^$' || true)
 elif [[ -n "${ASHLAR_NUGET_VERIFY_PACKAGE_ID:-}" ]]; then
   IDS=("${ASHLAR_NUGET_VERIFY_PACKAGE_ID}")
+elif [[ -n "${ASHLAR_MANIFEST_JSON:-}" && -f "${ASHLAR_MANIFEST_JSON}" ]]; then
+  # Every package this release pushed, not a sample of them. Polling four ids and then letting the
+  # next step download all ~22 is how the rest were still 404 on index lag when they were fetched -
+  # the failure that ended the v0.1.2 release after its packages were already public.
+  mapfile -t IDS < <(python3 -c 'import json,sys
+for r in json.load(open(sys.argv[1], encoding="utf-8")): print(r["id"])' "${ASHLAR_MANIFEST_JSON}" | tr -d '\r')
 else
+  # Last resort, for a local run with no manifest: the four packages a stranger restores first.
   IDS=(Ashlar.Hosting.Bundle Ashlar.Hosting Ashlar.Sdk Ashlar.CLI)
 fi
 

@@ -25,7 +25,7 @@ trap 'rm -rf "${WORKDIR}"' EXIT
 LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/lib" && pwd)"
 
 python3 - "$MAN" "$VER" "$WORKDIR" "$LIB_DIR" <<'PY'
-import json, os, sys, urllib.request
+import json, os, sys, time, urllib.error, urllib.request
 
 manifest_path, ver, workdir, lib_dir = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
 sys.path.insert(0, lib_dir)
@@ -35,6 +35,42 @@ with open(manifest_path, encoding="utf-8") as f:
     rows = json.load(f)
 os.makedirs(workdir, exist_ok=True)
 
+# nuget.org indexes packages one at a time after a push, so a package pushed seconds ago can
+# still 404 from the flat container. The visibility poll that runs before this step waits for a
+# handful of ids, not all of them, so the rest can legitimately be missing on the first request.
+# That is exactly the lag that failed v0.1.2's release. A bare urlretrieve raised HTTPError on the
+# first such 404 and ended the step with a traceback - after every package was already public.
+ATTEMPTS = int(os.environ.get("ASHLAR_NUGET_VERIFY_ATTEMPTS") or 40)
+SLEEP_SEC = float(os.environ.get("ASHLAR_NUGET_VERIFY_SLEEP_SEC") or 15)
+REQUEST_TIMEOUT_SEC = 60
+# Overridable so the retry path can be exercised against a local server, and so a staging feed can
+# be verified with the same code that verifies nuget.org rather than a copy of it.
+FLAT_CONTAINER = (os.environ.get("ASHLAR_NUGET_FLAT_CONTAINER") or "https://api.nuget.org/v3-flatcontainer").rstrip("/")
+RETRYABLE_STATUS = {404, 408, 429, 500, 502, 503, 504}
+
+
+def fetch(url, dest):
+    """Download url to dest, riding out index lag. Returns None on success, else a reason."""
+    last = "no attempt made"
+    for attempt in range(1, ATTEMPTS + 1):
+        try:
+            with urllib.request.urlopen(url, timeout=REQUEST_TIMEOUT_SEC) as resp:
+                data = resp.read()
+            with open(dest, "wb") as fh:
+                fh.write(data)
+            return None
+        except urllib.error.HTTPError as exc:
+            last = f"HTTP {exc.code}"
+            if exc.code not in RETRYABLE_STATUS:
+                return f"{last} (not retryable)"
+        except (urllib.error.URLError, TimeoutError, OSError) as exc:
+            last = f"{type(exc).__name__}: {exc}"
+        if attempt < ATTEMPTS:
+            print(f"  attempt {attempt}/{ATTEMPTS}: {last}; retrying in {SLEEP_SEC:g}s", file=sys.stderr)
+            time.sleep(SLEEP_SEC)
+    return f"{last} after {ATTEMPTS} attempts"
+
+
 checked = 0
 failures = 0
 for row in rows:
@@ -54,10 +90,19 @@ for row in rows:
         continue
 
     id_lc, v_lc = pid.lower(), v.lower()
-    url = f"https://api.nuget.org/v3-flatcontainer/{id_lc}/{v_lc}/{id_lc}.{v_lc}.nupkg"
+    url = f"{FLAT_CONTAINER}/{id_lc}/{v_lc}/{id_lc}.{v_lc}.nupkg"
     out = os.path.join(workdir, row["fileName"])
     print(f"download {url}")
-    urllib.request.urlretrieve(url, out)
+    problem = fetch(url, out)
+    if problem is not None:
+        print(
+            f"::error::could not download {pid} {v} from nuget.org: {problem}. The package was "
+            f"pushed before this step ran, so it is public; this is a failure to VERIFY it, not a "
+            f"failure to publish it. Re-run this job to verify once indexing catches up.",
+            file=sys.stderr,
+        )
+        failures += 1
+        continue
 
     got = content_digest(out)
     if got.lower() != expected.lower():

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+# Tests that the post-push verifier survives nuget.org index lag, against a local HTTP server.
+#
+# WHY THIS EXISTS. scripts/verify-nuget-published-sha256-matches-manifest.sh runs AFTER
+# `dotnet nuget push`. It downloaded each package with a bare urllib.request.urlretrieve - no
+# timeout, no retry, no HTTPError handling. nuget.org indexes packages one at a time after a push,
+# and the visibility poll before this step waits for only four ids, so the other ~18 can still 404
+# on the first request. That raised HTTPError and ended the step with a traceback, with every
+# package already public - and `github-release`, then gated on success(), skipped attaching the
+# release artifacts. It is the same index lag that failed the v0.1.2 release. (github-release is
+# now gated on publication having begun, so the artifacts survive a failure here.)
+#
+# This drives the REAL script, not a copy of its logic, against a server that answers 404 for a
+# while and then serves the package. It asserts four things, and a retry loop that satisfied only
+# the first would still be broken:
+#   1. a package that 404s and then appears is verified          (lag is ridden out)
+#   2. a package that never appears is a named failure            (not a traceback)
+#   3. the loop CONTINUES past it and verifies what follows       (failures are reported together)
+#   4. a non-retryable status fails fast                          (403 is not index lag)
+#
+# Run:  bash tests/scripts/nuget-verify-retry.test.sh
+# Pure python + bash: no network beyond 127.0.0.1, no dotnet, no container.
+
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+VERIFIER="${ROOT}/scripts/verify-nuget-published-sha256-matches-manifest.sh"
+
+PASS=0
+FAIL=0
+EXPECTED_ASSERTIONS=8
+
+ok()  { PASS=$((PASS + 1)); echo "  ok   — $1"; }
+bad() { FAIL=$((FAIL + 1)); echo "  FAIL — $1"; echo "         $2"; }
+
+# Probe by RUNNING each candidate. On Windows `command -v python3` finds an App Execution Alias that
+# exits 49 without executing anything; every assertion below would then pass having run nothing.
+PY_BIN=""
+for cand in python3 python py; do
+  if "${cand}" -c 'print(1)' >/dev/null 2>&1; then PY_BIN="${cand}"; break; fi
+done
+[[ -n "${PY_BIN}" ]] || { echo "FAIL - no working python interpreter"; exit 1; }
+
+# The verifier itself invokes `python3`. Where that name is the Windows stub, put a shim first on
+# PATH so the script under test runs the interpreter that works. CI is Linux and never needs it.
+WORK="$(mktemp -d)"
+trap 'rm -rf "${WORK}"' EXIT
+if ! python3 -c 'print(1)' >/dev/null 2>&1; then
+  mkdir -p "${WORK}/bin"
+  printf '#!/bin/sh\nexec %s "$@"\n' "${PY_BIN}" > "${WORK}/bin/python3"
+  chmod +x "${WORK}/bin/python3"
+  export PATH="${WORK}/bin:${PATH}"
+fi
+
+# Hand python the bash that is running THIS script. A bare "bash" from Windows python resolves to
+# System32ash.exe - the WSL launcher - which fails before the verifier starts, so the server
+# receives nothing and every assertion about its requests is vacuous.
+RESULT="$("${PY_BIN}" - "${ROOT}" "${VERIFIER}" "${WORK}" "${BASH}" <<'PY'
+import http.server, json, os, subprocess, sys, threading, zipfile
+
+root, verifier, work, bash_bin = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
+sys.path.insert(0, os.path.join(root, "scripts", "lib"))
+from nupkg_content_digest import content_digest
+
+VER = "9.9.9"
+pkgs = {}
+for pid in ("Pkg.Lagging", "Pkg.Missing", "Pkg.Present", "Pkg.Forbidden"):
+    path = os.path.join(work, f"{pid}.{VER}.nupkg")
+    with zipfile.ZipFile(path, "w") as z:
+        z.writestr(f"{pid}.nuspec", f"<package><metadata><id>{pid}</id></metadata></package>")
+        z.writestr("lib/net8.0/x.dll", f"payload-{pid}")
+        z.writestr(".signature.p7s", b"repository-signature")
+    pkgs[pid] = path
+
+# Behaviour per package, by lowercase id as the flat container spells it.
+lag_left = {"pkg.lagging": 2}      # 404 twice, then serve
+hits = {}
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    def log_message(self, *a): pass
+    def do_GET(self):
+        pid_lc = self.path.strip("/").split("/")[0]
+        hits[pid_lc] = hits.get(pid_lc, 0) + 1
+        if pid_lc == "pkg.missing":
+            self.send_response(404); self.end_headers(); return
+        if pid_lc == "pkg.forbidden":
+            self.send_response(403); self.end_headers(); return
+        if lag_left.get(pid_lc, 0) > 0:
+            lag_left[pid_lc] -= 1
+            self.send_response(404); self.end_headers(); return
+        real = next(p for k, p in pkgs.items() if k.lower() == pid_lc)
+        data = open(real, "rb").read()
+        self.send_response(200); self.send_header("Content-Length", str(len(data))); self.end_headers()
+        self.wfile.write(data)
+
+srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+threading.Thread(target=srv.serve_forever, daemon=True).start()
+
+manifest = os.path.join(work, "manifest.json")
+json.dump([{"id": pid, "version": VER, "fileName": os.path.basename(p),
+            "sha256": "unused", "contentSha256": content_digest(p)} for pid, p in pkgs.items()],
+          open(manifest, "w"))
+
+env = dict(os.environ,
+           ASHLAR_MANIFEST_JSON=manifest,
+           ASHLAR_NUGET_VERIFY_VERSION=VER,
+           ASHLAR_NUGET_FLAT_CONTAINER=f"http://127.0.0.1:{srv.server_address[1]}",
+           ASHLAR_NUGET_VERIFY_ATTEMPTS="4",
+           ASHLAR_NUGET_VERIFY_SLEEP_SEC="0")
+proc = subprocess.run([bash_bin, verifier], env=env, capture_output=True, text=True)
+srv.shutdown()
+
+out = proc.stdout + proc.stderr
+def emit(name, cond, detail=""):
+    print(f"{name}\t{'PASS' if cond else 'FAIL'}\t{detail}")
+
+emit("the run fails overall, because one package never appears", proc.returncode != 0, f"exit {proc.returncode}")
+emit("a lagging package is retried and then verified", "Pkg.Lagging" not in proc.stderr.split("could not download")[-1] and hits.get("pkg.lagging") == 3,
+     f"hits={hits.get('pkg.lagging')}")
+emit("a package that never appears is a NAMED failure", "could not download Pkg.Missing" in out, "")
+emit("and not an uncaught traceback", "Traceback" not in out, out[-400:] if "Traceback" in out else "")
+emit("the missing package is retried the configured number of times", hits.get("pkg.missing") == 4, f"hits={hits.get('pkg.missing')}")
+emit("the loop continues past the failure and verifies what follows", hits.get("pkg.present", 0) >= 1 and "could not download Pkg.Present" not in out, f"hits={hits.get('pkg.present')}")
+emit("a non-retryable status fails fast", hits.get("pkg.forbidden") == 1, f"hits={hits.get('pkg.forbidden')}")
+emit("and says it was not retryable", "not retryable" in out, "")
+PY
+)"
+
+if [[ -z "${RESULT}" ]]; then
+  echo "FAIL - the python harness produced no results"
+  exit 1
+fi
+
+echo "== post-push verifier under index lag =="
+while IFS=$'\t' read -r name verdict detail; do
+  [[ -z "${name}" ]] && continue
+  if [[ "${verdict}" == "PASS" ]]; then ok "${name}"; else bad "${name}" "${detail}"; fi
+done <<< "${RESULT}"
+
+echo
+echo "passed: ${PASS}   failed: ${FAIL}"
+
+RAN=$((PASS + FAIL))
+if [[ "${RAN}" -ne "${EXPECTED_ASSERTIONS}" ]]; then
+  echo "FAIL - ran ${RAN} assertions, expected ${EXPECTED_ASSERTIONS}. A partial run is not a pass."
+  exit 1
+fi
+[[ "${FAIL}" -eq 0 ]] || exit 1