meshweaver-cloud · Sep 14, 2026 · Sep 14, 2026
diff --git a/.github/workflows/hosting-operator.yml b/.github/workflows/hosting-operator.yml
@@ -196,7 +196,7 @@ jobs:
         # through to a REAL az/kubectl on a runner that has one.
         run: |
           set -euo pipefail
-          for f in deploy/aks/operator/test/stubs/pull-secret/az deploy/aks/operator/test/stubs/pull-secret/kubectl deploy/aks/operator/test/stubs/pv-resize/kubectl deploy/aks/operator/test/stubs/inline-env/kubectl deploy/aks/operator/test/stubs/kv-rotate/az deploy/aks/operator/test/stubs/kv-ensure/az deploy/aks/operator/test/stubs/kv-copy/az deploy/aks/operator/test/stubs/registry-register/az deploy/aks/operator/test/stubs/signin-app/az deploy/aks/operator/test/stubs/image-mirror/az deploy/aks/operator/test/stubs/image-mirror/crane; do
+          for f in deploy/aks/operator/test/stubs/pull-secret/az deploy/aks/operator/test/stubs/pull-secret/kubectl deploy/aks/operator/test/stubs/pv-resize/kubectl deploy/aks/operator/test/stubs/inline-env/kubectl deploy/aks/operator/test/stubs/kv-rotate/az deploy/aks/operator/test/stubs/kv-ensure/az deploy/aks/operator/test/stubs/kv-copy/az deploy/aks/operator/test/stubs/registry-register/az deploy/aks/operator/test/stubs/signin-app/az deploy/aks/operator/test/stubs/image-mirror/az deploy/aks/operator/test/stubs/image-mirror/crane deploy/aks/operator/test/stubs/deploy/az; do
             git ls-files --error-unmatch "$f" >/dev/null 2>&1 || { echo "::error::${f} is not tracked"; git check-ignore -v "$f" || true; exit 1; }
             [ "$(git ls-files -s "$f" | awk '{print $1}')" = "100755" ] || { echo "::error::${f} is not committed executable (mode 100755)"; exit 1; }
           done

diff --git a/deploy/aks/operator/bin/hosting-deploy b/deploy/aks/operator/bin/hosting-deploy
@@ -1,16 +1,37 @@
 #!/usr/bin/env bash
-# hosting-deploy --namespace <ns> --release <name> --database <db> --values <file> --config-file <file>
+# hosting-deploy --namespace <ns> --release <name> --database <db> --values <file> [--vault <name>] [--image <ref>]
 #
 # Install (or upgrade) the portal release for ONE instance.
 #
 # 🚨 WHERE THE PER-INSTANCE VALUES COME FROM, because this is the piece a fleet gets wrong. There is
 # NO checked-in `values.<newinstance>.yaml`, and there must not be: a self-service instance is
-# created after the repo was last edited. So the shape is base + derived:
+# created after the repo was last edited. So the shape is TWO layers, in this order:
 #
-#   --values <file>   the FLEET TEMPLATE — everything every instance shares. Reviewed, in git.
-#   --set …           this instance's IDENTITY only: namespace, host, database, TLS secret, the
-#                     Key Vault prefix. Derived from the deployment record, which is itself the
-#                     reviewed artefact (it arrives in the config repo by write-through PR).
+#   helm-values-<release>   the Key Vault VALUES HALF — `parameters:`, `pgbackrest:`, `secrets:` —
+#                           captured from the live release by the config repo's `helm-release
+#                           capture` and held in no repository. Layered FIRST, from the vault the
+#                           record names (--vault), only when the record declares that the chart's
+#                           own Secret is fed from it (`vaultValuesKeys`).
+#   --values <file>         the record's RENDER (HelmValues in the Hosting plugin) — secret-free by
+#                           construction. Layered LAST, so the reviewed record wins on every key it
+#                           declares; the `--set` flags after it carry only this instance's identity.
+#
+# 🚨 WHY THE VAULT HALF IS NOT OPTIONAL WHERE THE RECORD DECLARES IT (MeshWeaver#3780). The chart's
+# `memex-portal-secrets` / `memex-migration-secrets` render from `secrets.<half>.*`, and for an
+# instance on an external Postgres those values live ONLY in the vault half. helm REPLACES a Secret
+# wholesale on every upgrade, so an upgrade fed the record alone re-rendered both Secrets from the
+# chart's in-cluster defaults — `ConnectionStrings__orleans` became `Host=memex-postgres-service;…`,
+# a Service the release does not render — and every new pod died at silo start:
+#
+#     'MembershipTableManager' failed to start due to errors at stage 'RuntimeGrainServices (8000)'.
+#     Npgsql.NpgsqlException: Name or service not known
+#
+# Measured twice on the control instance: revision 44 (2026-09-09 02:35Z) and revision 55
+# (2026-09-14 20:14Z), both record-driven Reconciles; the lane-driven deploys in between, which
+# layer `-f vault-values.yaml -f <overlay>`, rolled the same image 2/2 Ready. The two paths had two
+# answers to one question; this is the disagreement resolved in favour of the one that kept the
+# instance up. The values half is read by the operator's own identity and is never echoed — only
+# its size is logged.
 #
 # The chart ships in this image at /opt/hosting/chart, so the operator image version IS the chart
 # version it deploys. That pinning is deliberate: the scripts and the templates they --set against
@@ -22,14 +43,15 @@ HOSTING_CMD="hosting-deploy"
 
 CHART="${HOSTING_CHART:-/opt/hosting/chart}"
 
-namespace="" release="" database="" values="" image=""
+namespace="" release="" database="" values="" image="" vault=""
 while [ $# -gt 0 ]; do
   case "$1" in
     --namespace)   namespace="${2:-}";   shift 2 ;;
     --release)     release="${2:-}";     shift 2 ;;
     --database)    database="${2:-}";    shift 2 ;;
     --values)      values="${2:-}";      shift 2 ;;
     --image)       image="${2:-}";       shift 2 ;;
+    --vault)       vault="${2:-}";       shift 2 ;;
     *) hosting::die "unknown argument '$1'" ;;
   esac
 done
@@ -40,13 +62,15 @@ hosting::need_flag database "$database"
 hosting::safe_name namespace "$namespace"
 hosting::safe_name release "$release"
 hosting::safe_name database "$database"
+[ -z "$vault" ] || hosting::safe_name vault "$vault"
 
 [ -d "$CHART" ] || hosting::die "no chart at ${CHART}. It is baked into the operator image (COPY helm/ /opt/hosting/chart) precisely so a run cannot deploy a chart nobody reviewed — an image built without it is broken, not degraded."
 # 🚨 THE VALUES FILE IS RENDERED FROM THE DEPLOYMENT RECORD (HelmValues in the Hosting plugin)
 # and materialised by run.sh from HOSTING_VALUES. There is no fleet template and no per-environment
-# overlay any more: the record is the only source. A missing or empty file is a refusal, because
-# helm would fall through to CHART DEFAULTS — ghcr.io/…:latest and an in-cluster Postgres this
-# fleet does not run — and report success.
+# overlay any more: the record is the only REPOSITORY source (the vault half above is the other
+# layer, and it is not a file anyone edits). A missing or empty file is a refusal, because helm
+# would fall through to CHART DEFAULTS — ghcr.io/…:latest and an in-cluster Postgres this fleet
+# does not run — and report success.
 [ -n "$values" ] && [ -f "$values" ] \
   || hosting::die "the rendered values file '${values:-<unset>}' does not exist. It is rendered from the Deployment record by the Hosting plugin and passed to the job as HOSTING_VALUES; without it the release would fall through to CHART DEFAULTS, which point at ghcr.io/…:latest and an in-cluster Postgres this fleet does not run."
 [ -s "$values" ] \
@@ -98,11 +122,38 @@ if [ -n "$pg_host" ]; then
   )
 fi
 
+# ── the Key Vault values half, layered FIRST ────────────────────────────────────────────────────
+# 🚨 Read only when the record asked for it (--vault), and then REQUIRED: a record that declares
+# `vaultValuesKeys` is stating that the chart's own Secret carries keys nothing in the record
+# renders. Proceeding without the half would not deploy "less" — it would replace those keys with
+# the chart's in-cluster defaults, which is exactly the boot failure in the header. The same rule
+# as the config repo's lane: "Key Vault holds no values for <release> — run capture first".
+#
+# The file is 0600 in the job's private /tmp, removed on exit, and NEVER printed: the adoption
+# render below holds the values in memory, and the only thing this step logs is the byte count.
+vault_values=""
+if [ -n "$vault" ]; then
+  vault_values="$(mktemp)"
+  chmod 600 "$vault_values"
+  # shellcheck disable=SC2064  # expand the path NOW: the trap must name this file, not a later $vault_values
+  trap "rm -f '$vault_values'" EXIT
+  hosting::log "vault     ${vault}: reading the values half helm-values-${release}"
+  kv_err="$(hosting::read az keyvault secret download --vault-name "$vault" --name "helm-values-${release}" --file "$vault_values" --output none 2>&1 >/dev/null)" \
+    || hosting::die "Key Vault ${vault} holds no values half for release ${release} (helm-values-${release}): ${kv_err}. The record declares vaultValuesKeys — keys the chart's own Secret must carry — and nothing in the record renders them, so an upgrade without this file would re-render memex-portal-secrets from the chart's in-cluster defaults and every new pod would fail at silo start (MeshWeaver#3780). Run the config repo's helm-release workflow with action=capture for this release first, or clear vaultValuesKeys on the record if the chart's Secret genuinely carries nothing."
+  [ -s "$vault_values" ] \
+    || hosting::die "the values half helm-values-${release} in Key Vault ${vault} is EMPTY — an empty vault half is not 'no secrets', it is a capture that wrote nothing. Re-run helm-release capture for this release (MeshWeaver#3780)."
+  hosting::log "vault     helm-values-${release} ($(wc -c < "$vault_values") bytes) — layered FIRST; the record's render wins on every key it declares"
+  hosting::say vault_values_bytes "$(wc -c < "$vault_values" | tr -d ' ')"
+fi
+
 # The catalog config the mesh composed (plugin mounts + pre-install patterns) is a values FILE, so
 # it layers like any other and stays visible in `helm get values`.
-# ONE values file. The catalog wiring, the module policy, the envelope and the volumes all ride in
-# it, in the chart's own shape — there is no second file to forget.
-files=( -f "$values" )
+# TWO values files at most, in a fixed order: the vault half (when the record declares one), then
+# the record's render. The catalog wiring, the module policy, the envelope and the volumes all ride
+# in the render, in the chart's own shape — there is no third file to forget.
+files=()
+[ -z "$vault_values" ] || files+=( -f "$vault_values" )
+files+=( -f "$values" )
 hosting::log "values    ${values} ($(wc -c < "$values") bytes, rendered from the record)"
 
 hosting::log "chart     ${CHART}"

diff --git a/deploy/aks/operator/test/run-tests.sh b/deploy/aks/operator/test/run-tests.sh
@@ -1596,6 +1596,82 @@ else
 fi
 rm -rf "$_ap_dir"
 
+# ── hosting-deploy layers the Key Vault VALUES HALF first, when the record declares one ─────────
+# 🚨 MeshWeaver#3780, measured twice on the control instance (helm revisions 44 and 55). The chart's
+# own Secrets render from `secrets.<half>.*`, which on an external database live ONLY in the vault
+# half `helm-values-<release>` — and this script fed helm the record's secret-free render alone.
+# helm replaces a Secret wholesale, so `ConnectionStrings__orleans` became the chart's in-cluster
+# default and every new pod died at silo start. The config repo's lane had layered
+# `-f vault-values.yaml -f <overlay>` all along. These cases pin the shape that agrees with it:
+# the half is read from the vault the record names, layered FIRST so the render wins, never
+# echoed, REQUIRED where asked for (an absent half is a refusal before helm, naming capture), and
+# absent altogether when the record declares none (a provisioned instance's shape).
+echo
+echo "── hosting-deploy: the Key Vault values half (#3780) ────────────"
+_vh_dir="$(mktemp -d)"; cp -R "$DP_FIXTURES/." "$_vh_dir/"; _vh_log="$_vh_dir/calls.log"; : > "$_vh_log"
+mkdir -p "$_vh_dir/vault"
+printf 'secrets:\n  memex_portal:\n    ConnectionStrings__orleans: "Host=pg.example.test;Password=SENTINEL-NEVER-PRINTED"\n' > "$_vh_dir/vault/helm-values-memex"
+_vh_vals="$_vh_dir/values.yaml"; printf '# GENERATED from the Hosting/Deployment record by HelmValues\nreplicas:\n  portal: 2\n' > "$_vh_vals"
+_vh_run() { env PATH="$DP_STUBS:$PATH" HOSTING_CHART=/tmp HOSTING_DEPLOY_FIXTURE="$_vh_dir" HOSTING_DEPLOY_STUB_LOG="$_vh_log" \
+  hosting-deploy --namespace memex --release memex --database memex --values "$_vh_vals" --image cr.example.test/memex-portal-ai:1 "$@" 2>&1; }
+_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$?
+[ "$_vh_rc" -eq 0 ] && ok "deploy succeeds when the vault holds the values half" || bad "deploy succeeds when the vault holds the values half" "exited ${_vh_rc}: ${_vh_out}"
+grep -q '^az keyvault secret download --vault-name kv-test --name helm-values-memex --file ' "$_vh_log" \
+  && ok "the half is read from the vault the record names, under helm-values-<release>" \
+  || bad "the half is read from the vault the record names" "$(cat "$_vh_log")"
+# ORDER IS THE CONTRACT: vault half first, the record's render LAST — on the upgrade AND on the
+# adoption render, or the two would disagree about what the release is.
+_vh_layers() { sed -n 's/.* -f \([^ ]*\) -f \([^ ]*\).*/\1 \2/p' <<< "$1"; }
+_vh_up="$(_vh_layers "$(grep '^helm upgrade' "$_vh_log" | head -1)")"
+_vh_tp="$(_vh_layers "$(grep '^helm template' "$_vh_log" | head -1)")"
+_vh_first="${_vh_up%% *}"; _vh_second="${_vh_up##* }"
+if [ -n "$_vh_up" ] && [ "$_vh_first" != "$_vh_vals" ] && [ "$_vh_second" = "$_vh_vals" ]; then
+  ok "helm upgrade layers the vault half FIRST and the record's render LAST"
+else
+  bad "helm upgrade layers the vault half first and the record last" "layers: '${_vh_up}' in: $(cat "$_vh_log")"
+fi
+[ -n "$_vh_tp" ] && [ "$_vh_tp" = "$_vh_up" ] \
+  && ok "the adoption render sees the same two layers in the same order" \
+  || bad "the adoption render sees the same two layers" "template: '${_vh_tp}' upgrade: '${_vh_up}'"
+case "$_vh_out" in *SENTINEL-NEVER-PRINTED*) bad "the values half is never echoed" "the secret value reached the log: ${_vh_out}" ;;
+  *) ok "the values half is never echoed" ;; esac
+case "$_vh_out" in *"::hosting:: vault_values_bytes="*) ok "only the half's SIZE is reported (::hosting:: vault_values_bytes=)" ;;
+  *) bad "only the half's size is reported" "said: ${_vh_out}" ;; esac
+[ -n "$_vh_first" ] && [ ! -e "$_vh_first" ] \
+  && ok "the half's temp file is removed when the script exits" \
+  || bad "the half's temp file is removed on exit" "'${_vh_first}' still exists"
+# 🚨 Declared but absent is a REFUSAL, before helm, naming the way forward — never a deploy that
+# quietly falls through to the chart's defaults, which is the #3780 boot failure itself.
+rm -f "$_vh_dir/vault/helm-values-memex"; : > "$_vh_log"
+_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$?
+if [ "$_vh_rc" -ne 0 ] && printf '%s' "$_vh_out" | grep -q 'helm-values-memex' \
+   && printf '%s' "$_vh_out" | grep -q 'capture' && printf '%s' "$_vh_out" | grep -q '3780' \
+   && ! grep -q '^helm ' "$_vh_log"; then
+  ok "a declared half the vault does not hold is refused before helm, naming capture and #3780"
+else
+  bad "a declared half the vault does not hold is refused before helm" "rc=${_vh_rc} out: ${_vh_out} log: $(cat "$_vh_log")"
+fi
+: > "$_vh_dir/vault/helm-values-memex"; : > "$_vh_log"
+_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$?
+if [ "$_vh_rc" -ne 0 ] && printf '%s' "$_vh_out" | grep -q 'EMPTY' && ! grep -q '^helm ' "$_vh_log"; then
+  ok "an EMPTY half is refused too — a capture that wrote nothing is not 'no secrets'"
+else
+  bad "an empty half is refused" "rc=${_vh_rc} out: ${_vh_out}"
+fi
+# A record that declares NO vault half (vaultValuesKeys empty — every provisioned instance) reads
+# nothing from any vault, and the record is the only layer.
+: > "$_vh_log"
+_vh_out="$(_vh_run)"; _vh_rc=$?
+_vh_fs="$(grep '^helm upgrade' "$_vh_log" | head -1 | grep -o ' -f ' | wc -l | tr -d ' ')"
+if [ "$_vh_rc" -eq 0 ] && ! grep -q '^az ' "$_vh_log" && [ "$_vh_fs" = "1" ]; then
+  ok "without --vault nothing is read from any vault and the record is the only layer"
+else
+  bad "without --vault the record is the only layer" "rc=${_vh_rc} -f count=${_vh_fs} log: $(cat "$_vh_log")"
+fi
+refuses_hard "a vault name that is not a plain name is refused before anything runs" "not a plain name" \
+  env HOSTING_DRY_RUN=true HOSTING_CHART=/tmp hosting-deploy --namespace memex --release memex --database memex --values "$_vh_vals" --vault 'kv;rm -rf /'
+rm -rf "$_vh_dir"
+
 # ── hosting-deploy refuses BEFORE helm when the identity cannot write a rendered kind ───────────
 # Measured 2026-09-09 01:59Z on memex: helm died on `poddisruptionbudgets.policy is forbidden`
 # and its --atomic rollback erred too. The preflight asks `kubectl auth can-i` per rendered kind

diff --git a/deploy/aks/operator/test/stubs/deploy/az b/deploy/aks/operator/test/stubs/deploy/az
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# A stand-in `az` for hosting-deploy's values-half tests (MeshWeaver#3780). Only ONE call is
+# answered — `keyvault secret download --vault-name V --name N --file F` — and it is answered from
+# a fixture, not from anything az does: the bytes of $HOSTING_DEPLOY_FIXTURE/vault/<N> are copied to
+# F when that file exists, else the real CLI's SecretNotFound shape goes to stderr with exit 1.
+# Every invocation is recorded in $HOSTING_DEPLOY_STUB_LOG so the tests can assert WHICH vault and
+# WHICH secret name were asked for. It asserts nothing about Key Vault itself.
+set -uo pipefail
+log="${HOSTING_DEPLOY_STUB_LOG:?the az stub needs HOSTING_DEPLOY_STUB_LOG}"
+fixture="${HOSTING_DEPLOY_FIXTURE:?the az stub needs HOSTING_DEPLOY_FIXTURE}"
+printf 'az %s\n' "$*" >> "$log"
+if [ "${1:-} ${2:-} ${3:-}" != "keyvault secret download" ]; then
+  echo "az stub: unexpected invocation: $*" >&2; exit 1
+fi
+shift 3
+name="" file=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --name) name="${2:-}"; shift 2 ;;
+    --file) file="${2:-}"; shift 2 ;;
+    *) shift ;;
+  esac
+done
+src="$fixture/vault/$name"
+if [ ! -f "$src" ]; then
+  echo "ERROR: (SecretNotFound) A secret with (name/id) ${name} was not found in this key vault. If you recently deleted this secret you may be able to recover it using the correct recovery command." >&2
+  exit 1
+fi
+cp "$src" "$file"