diff --git a/deploy/aks/operator/bin/hosting-deploy b/deploy/aks/operator/bin/hosting-deploy index 4493d24d8c..99ed0e5367 100755 --- a/deploy/aks/operator/bin/hosting-deploy +++ b/deploy/aks/operator/bin/hosting-deploy @@ -172,6 +172,40 @@ fi # # The file is 0600 in the job's private /tmp, removed on exit, and NEVER printed: the adoption # render below holds the values in memory, and the only thing this step logs is the byte count. +# +# vault_top_level_keys — the TOP-LEVEL KEY NAMES of the values half, one per line, and never a +# value. The half is written by the config repo's `capture` (PyYAML safe_dump, block style), so a +# top-level key is a line at column 0 ending its name with `:`; a JSON object (az stores what it is +# given) is read with jq, which this image carries. Anything else — a flow mapping, a bare scalar, +# a list — returns 1: the caller refuses what it cannot read rather than guessing it is clean. +# Pure bash for the YAML branch, for the reason values_carry_portal_image gives above. +vault_top_level_keys() { + local line key first="" + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in ''|' '*|$'\t'*|'#'*|'---'*) continue ;; esac + first="$line"; break + done < "$1" + case "$first" in + '{'*) jq -er 'if type == "object" then keys[] else error("not an object") end' "$1" 2>/dev/null || return 1 + return 0 ;; + '') return 0 ;; # comments only: no keys (the families check refuses it) + '-'*|'['*) return 1 ;; # the document itself is a list, which is not helm values + esac + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + ''|' '*|$'\t'*|'#'*|'---'*|'...'*) continue ;; + # PyYAML writes a sequence under a key INDENTLESS (`extraVolumes:` then `- name: …` at column + # 0), so a column-0 `- ` after the first key is that key's item, not a key of its own. + '- '*|'-') continue ;; + '{'*|'['*) return 1 ;; + esac + key="${line%%:*}" + [ "$key" != "$line" ] || return 1 # a column-0 line with no `:` is not a mapping key + key="${key%\"}"; key="${key#\"}"; key="${key%\'}"; key="${key#\'}" + printf '%s\n' "$key" + done < "$1" +} + vault_values="" if [ -n "$vault" ]; then vault_values="$(mktemp)" @@ -188,7 +222,39 @@ if [ -n "$vault" ]; then || hosting::die "Key Vault ${vault} holds no values half for release ${release} (helm-values-${release}): ${kv_err}. The record declares vaultValuesKeys — keys the chart's own Secret must carry — and nothing in the record renders them, so an upgrade without this file would re-render memex-portal-secrets from the chart's in-cluster defaults and every new pod would fail at silo start (MeshWeaver#3780). Run the config repo's helm-release workflow with action=capture for this release first, or clear vaultValuesKeys on the record if the chart's Secret genuinely carries nothing." [ -s "$vault_values" ] \ || hosting::die "the values half helm-values-${release} in Key Vault ${vault} is EMPTY — an empty vault half is not 'no secrets', it is a capture that wrote nothing. Re-run helm-release capture for this release (MeshWeaver#3780)." - hosting::log "vault helm-values-${release} ($(wc -c < "$vault_values") bytes) — layered FIRST; the record's render wins on every key it declares" + # 🚨 THE HALF CARRIES SECRET FAMILIES ONLY — STRUCTURE IS REFUSED, NEVER LAYERED (Memex#295). + # "The record's render wins on every key it declares" is true only for keys the render DECLARES. + # helm REPLACES a list the later file names and LEAVES one it omits, and HelmValues omits an empty + # list (no `extraVolumes:` at all when the record mounts nothing) — so a legacy list sitting in the + # half DEPLOYS alongside whatever the record renders for the same wiring. Measured 2026-09-25 on + # memex, the V58 Reconcile: + # Job.batch "memex-migration-62" is invalid: … Duplicate value: "kv-secrets" + # — the half still carried extraVolumes/extraVolumeMounts/extraEnvFrom for the SecretProviderClass + # the record declares under keyVaultSecretClasses, and the chart rendered the volume twice. The + # same shape failed memex-cloud's helm revision 42 on 2026-09-12 through the config repo's lane. + # A map merges instead, so a structural map in the half silently supplies every leaf the render + # does not name — configuration nobody reviews, decided by a capture of whatever was live. + # The families are an ALLOW-list, the same one the config repo's `capture` stores and its lane's + # guard reads (scripts/vault-values.py VAULT_FAMILIES): `secrets`, `parameters`, `pgbackrest`. + # Everything else is structure and has one home, the record. Only the KEY NAMES are read — never + # a value — and the refusal names them, with the one remedy: re-capture the half filtered to its + # families (the previous version stays in the vault as the rollback). + if ! vault_keys="$(vault_top_level_keys "$vault_values")"; then + hosting::die "the values half helm-values-${release} in Key Vault ${vault} is not a block-style YAML mapping or a JSON object — its top-level keys cannot be read, so whether it carries structure is unknown and it is refused rather than layered (Memex#295). Re-capture it: a governed HelmRelease action with helmAction: capture stores the families as block YAML." + fi + structural="" families=0 + while IFS= read -r key; do + [ -n "$key" ] || continue + case "$key" in + secrets|parameters|pgbackrest) families=$((families+1)) ;; + *) structural="${structural} ${key}" ;; + esac + done <<< "$vault_keys" + [ -z "$structural" ] \ + || hosting::die "the values half helm-values-${release} in Key Vault ${vault} carries STRUCTURE — top-level key(s):${structural}. The half holds secret families only (secrets, parameters, pgbackrest); structure lives on the Deployment record. Layered first, a structural list the record's render omits DEPLOYS beside the record's own wiring (memex 2026-09-25: Job memex-migration-62 invalid, duplicate volume \"kv-secrets\"), and a structural map supplies every leaf the render does not name. Refused before helm, nothing changed. Remedy: a governed HelmRelease action for ${release} with helmAction: capture — it re-stores the half filtered to its families and keeps the previous version as the rollback. Before capturing, confirm every leaf the half alone supplies is inert or declared on the record (Memex#295)." + [ "$families" -gt 0 ] \ + || hosting::die "the values half helm-values-${release} in Key Vault ${vault} holds none of the families (secrets, parameters, pgbackrest) — a record that declares vaultValuesKeys needs the chart's Secrets fed from it, so this half would deploy no secrets at all (MeshWeaver#3780). Re-run the HelmRelease capture for this release." + hosting::log "vault helm-values-${release} ($(wc -c < "$vault_values") bytes, families only) — layered FIRST; the record's render wins on every key it declares" hosting::say vault_values_bytes "$(wc -c < "$vault_values" | tr -d ' ')" fi diff --git a/deploy/aks/operator/test/run-tests.sh b/deploy/aks/operator/test/run-tests.sh index e3869622ae..669f1a81c8 100755 --- a/deploy/aks/operator/test/run-tests.sh +++ b/deploy/aks/operator/test/run-tests.sh @@ -1788,6 +1788,54 @@ if [ "$_vh_rc" -eq 0 ] && ! grep -q '^az ' "$_vh_log" && [ "$_vh_fs" = "1" ]; th else bad "without --vault the record is the only layer" "rc=${_vh_rc} -f count=${_vh_fs} log: $(cat "$_vh_log")" fi +# 🚨 Memex#295: the half carries secret FAMILIES only. A structural key in it is refused BEFORE helm +# and named — never layered, because a list the render omits deploys beside the record's own wiring +# (memex 2026-09-25, Job memex-migration-62: duplicate volume "kv-secrets"). Names only, never values. +printf 'extraVolumes:\n- name: kv-secrets\n csi:\n driver: secrets-store.csi.k8s.io\npgbackrest:\n enabled: false\nsecrets:\n memex_portal:\n ConnectionStrings__orleans: "SENTINEL-NEVER-PRINTED"\n' > "$_vh_dir/vault/helm-values-memex" +: > "$_vh_log" +_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$? +if [ "$_vh_rc" -ne 0 ] && printf '%s' "$_vh_out" | grep -q 'STRUCTURE' && printf '%s' "$_vh_out" | grep -q ' extraVolumes' \ + && printf '%s' "$_vh_out" | grep -q 'helmAction: capture' && ! grep -q '^helm ' "$_vh_log"; then + ok "a half carrying a structural key is refused before helm, naming the key and the capture remedy (Memex#295)" +else + bad "a structural half is refused before helm" "rc=${_vh_rc} out: ${_vh_out} log: $(cat "$_vh_log")" +fi +case "$_vh_out" in *SENTINEL-NEVER-PRINTED*|*secrets-store.csi*) bad "the structural refusal names keys, never values" "a value reached the log: ${_vh_out}" ;; + *) ok "the structural refusal names keys, never values" ;; esac +printf 'config:\n memex_portal:\n Foo: "bar"\nsecrets:\n memex_portal: {}\n' > "$_vh_dir/vault/helm-values-memex" +: > "$_vh_log" +_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$? +[ "$_vh_rc" -ne 0 ] && printf '%s' "$_vh_out" | grep -q ' config' && ! grep -q '^helm ' "$_vh_log" \ + && ok "a structural MAP is refused too — it would silently supply every leaf the render omits" \ + || bad "a structural map is refused" "rc=${_vh_rc} out: ${_vh_out}" +# All three families, quoted keys and comments: accepted and layered. +printf '# captured\n"parameters": {}\npgbackrest:\n enabled: false\nsecrets:\n memex_portal:\n X: "y"\n' > "$_vh_dir/vault/helm-values-memex" +: > "$_vh_log" +_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$? +[ "$_vh_rc" -eq 0 ] && grep -q '^helm upgrade' "$_vh_log" \ + && ok "a half holding only secrets/parameters/pgbackrest is layered" \ + || bad "a families-only half is layered" "rc=${_vh_rc} out: ${_vh_out}" +# A JSON object is read by its keys (jq): families pass, structure is refused. +printf '{"secrets":{"memex_portal":{"X":"y"}}}\n' > "$_vh_dir/vault/helm-values-memex" +: > "$_vh_log" +_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$? +[ "$_vh_rc" -eq 0 ] && ok "a JSON half holding only a family is layered" || bad "a JSON families-only half is layered" "rc=${_vh_rc} out: ${_vh_out}" +printf '{"secrets":{},"ingress":{"host":"h"}}\n' > "$_vh_dir/vault/helm-values-memex" +: > "$_vh_log" +_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$? +[ "$_vh_rc" -ne 0 ] && printf '%s' "$_vh_out" | grep -q ' ingress' && ! grep -q '^helm ' "$_vh_log" \ + && ok "a JSON half carrying structure is refused" || bad "a JSON structural half is refused" "rc=${_vh_rc} out: ${_vh_out}" +# What cannot be read is refused, never assumed clean; a half with no family at all is refused. +printf -- '- secrets\n' > "$_vh_dir/vault/helm-values-memex" +: > "$_vh_log" +_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$? +[ "$_vh_rc" -ne 0 ] && printf '%s' "$_vh_out" | grep -q 'cannot be read' && ! grep -q '^helm ' "$_vh_log" \ + && ok "a half whose top level is not a mapping is refused as unreadable" || bad "an unreadable half is refused" "rc=${_vh_rc} out: ${_vh_out}" +printf '# nothing but a comment\n' > "$_vh_dir/vault/helm-values-memex" +: > "$_vh_log" +_vh_out="$(_vh_run --vault kv-test)"; _vh_rc=$? +[ "$_vh_rc" -ne 0 ] && printf '%s' "$_vh_out" | grep -q 'none of the families' && ! grep -q '^helm ' "$_vh_log" \ + && ok "a half holding none of the families is refused" || bad "a family-less half is refused" "rc=${_vh_rc} out: ${_vh_out}" refuses_hard "a vault name that is not a plain name is refused before anything runs" "not a plain name" \ env HOSTING_DRY_RUN=true HOSTING_CHART=/tmp hosting-deploy --namespace memex --release memex --database memex --values "$_vh_vals" --vault 'kv;rm -rf /' rm -rf "$_vh_dir" diff --git a/src/MeshWeaver.Documentation/Data/Architecture/DeploymentEnvLayers.md b/src/MeshWeaver.Documentation/Data/Architecture/DeploymentEnvLayers.md index ca02fec00a..e0ea069340 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/DeploymentEnvLayers.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/DeploymentEnvLayers.md @@ -387,6 +387,31 @@ Three things changed, so that neither path can produce that render again: host (invariant 16) that simply did not exist — so `check-chart-invariants.sh` now carries a **refusal control**: a fixture of exactly the #3780 render that must fail to template. +### The half carries secret families only — structure is refused (Memex#295) + +"The record wins on every key it declares" holds only for keys the render DECLARES. helm REPLACES a +list the later `-f` file names and LEAVES one it omits, and `HelmValues` omits an empty list — a +record that mounts nothing renders no `extraVolumes:` key at all, where a committed overlay may +write `extraVolumes: []`. So a legacy list left in the half by an old whole-release capture deploys +BESIDE whatever the record renders for the same wiring. Measured on memex, 2026-09-25, the V58 +migration Reconcile: the half still carried `extraVolumes`/`extraVolumeMounts`/`extraEnvFrom` for +the SecretProviderClass the record declares under `keyVaultSecretClasses`, the chart rendered the +volume twice, and `helm upgrade` refused with +`Job memex-migration-62 invalid: duplicate volume "kv-secrets"`. A structural MAP fails the other +way — silently: it merges, supplying every leaf the render does not name. + +`hosting-deploy` therefore reads the half's top-level KEY NAMES (never a value) before anything +else runs and refuses, naming them, any key outside the allow-list `secrets`, `parameters`, +`pgbackrest` — the same list the config repo's `capture` stores and its lane's guard reads +(`scripts/vault-values.py`, `VAULT_FAMILIES`). A half it cannot read as a mapping, and a half with +none of the families, are refused too. The remedy it names is a governed `HelmRelease` action with +`helmAction: capture`, which re-stores the half filtered to its families and keeps the previous +version as the rollback. Before capturing, check that every leaf the half ALONE supplies is inert +or declared on the record: on memex-cloud (measured 2026-09-25 against a local `HelmValues.Render` +of the live record) those leaves were `image.*`, `ingress.tls.*`, `portal.image` and +`migration.image` — the chart reads neither of the first two, and `hosting-deploy` `--set`s the +last two — so nothing the portal reads left the render. + What is deliberately *not* gated the same way: the mesh string's chart-side copy on a record-driven instance is a shadowed placeholder by design (layer 3 supplies the real one), so `helm template` still accepts it. That left one known consequence for the first operator image carrying the #4173