From 6b7419c7051c94b508c715ac1acf8c65a1a54d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:47:59 +0200 Subject: [PATCH] chore(operator): the hosting operator leaves core for MeshWeaver.Plugins (delete half) Removes deploy/aks/operator, .github/workflows/hosting-operator.yml and the operator's entries in manual-keyvault.allow and .gitignore; they live in MeshWeaver.Plugins deploy/hosting-operator now, with its own hosting-operator.yml publishing hosting-operator:main. - memex-local's Linux gate, which rode in hosting-operator.yml, stays here unchanged as memex-local.yml. - release-channels: a writer binding may name another repository ("repo"); the hosting-operator:main writer now names MeshWeaver.Plugins, so the label stays declared and protected. Self-tests for both directions. - Doc/Architecture/OperatorCredentialsByName moved to Hosting/ in Plugins; inbound links and path mentions repointed; the carving map records the move. Pairs-with: Systemorph/MeshWeaver.Plugins#4137 Refs core slimming. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/new-deployment/SKILL.md | 2 +- .github/manual-keyvault.allow | 26 +- .github/release-channels.json | 5 +- .github/scripts/check-workflow-shell.py | 4 +- .github/scripts/lock-pinned-digests.py | 6 +- .github/scripts/release-channels.py | 30 +- .github/workflows/homebrew.yml | 2 +- .github/workflows/hosting-operator.yml | 457 -- .github/workflows/memex-local.yml | 69 + .github/workflows/s1-extract-image.yml | 2 +- .gitignore | 11 - deploy/aks/INSTANCE-LIFECYCLE.md | 12 +- .../aks/manifests/hosting-operator/README.md | 17 +- deploy/aks/operator/Dockerfile | 79 - deploy/aks/operator/bin/_audit.jq | 208 - deploy/aks/operator/bin/_common.sh | 477 -- deploy/aks/operator/bin/_dump_analyze_pod.sh | 114 - deploy/aks/operator/bin/_storage_prune_pod.sh | 246 - deploy/aks/operator/bin/_values_half.py | 292 - deploy/aks/operator/bin/hosting-aks-upgrade | 481 -- .../operator/bin/hosting-artifact-store-bind | 229 - deploy/aks/operator/bin/hosting-audit | 201 - deploy/aks/operator/bin/hosting-backup | 100 - deploy/aks/operator/bin/hosting-chart-drift | 145 - deploy/aks/operator/bin/hosting-db-release | 154 - deploy/aks/operator/bin/hosting-db-reset | 245 - deploy/aks/operator/bin/hosting-deploy | 718 --- deploy/aks/operator/bin/hosting-dns | 91 - deploy/aks/operator/bin/hosting-dump-analyze | 396 -- deploy/aks/operator/bin/hosting-export | 64 - deploy/aks/operator/bin/hosting-federate | 85 - deploy/aks/operator/bin/hosting-image-mirror | 163 - .../operator/bin/hosting-inline-env-retire | 304 - deploy/aks/operator/bin/hosting-inventory | 122 - deploy/aks/operator/bin/hosting-kv-copy | 112 - deploy/aks/operator/bin/hosting-kv-ensure | 389 -- deploy/aks/operator/bin/hosting-kv-purge | 83 - deploy/aks/operator/bin/hosting-kv-rotate | 236 - deploy/aks/operator/bin/hosting-kv-set | 226 - deploy/aks/operator/bin/hosting-kv-state | 84 - deploy/aks/operator/bin/hosting-kv-status | 93 - deploy/aks/operator/bin/hosting-kv-synced | 166 - deploy/aks/operator/bin/hosting-kv-values-set | 197 - .../aks/operator/bin/hosting-manifest-apply | 226 - deploy/aks/operator/bin/hosting-migrate | 265 - deploy/aks/operator/bin/hosting-pull-secret | 138 - deploy/aks/operator/bin/hosting-pv-purge | 141 - deploy/aks/operator/bin/hosting-pv-resize | 180 - deploy/aks/operator/bin/hosting-redirect | 82 - .../aks/operator/bin/hosting-registry-coupon | 146 - deploy/aks/operator/bin/hosting-registry-key | 188 - .../operator/bin/hosting-registry-register | 204 - deploy/aks/operator/bin/hosting-restore | 84 - deploy/aks/operator/bin/hosting-signin-app | 135 - deploy/aks/operator/bin/hosting-storage-prune | 353 -- deploy/aks/operator/bin/hosting-tls | 133 - deploy/aks/operator/bin/hosting-verify | 103 - deploy/aks/operator/bin/hosting-verify-backup | 61 - .../aks/operator/bin/hosting-verify-catalog | 63 - .../aks/operator/bin/hosting-verify-restore | 75 - deploy/aks/operator/bin/run.sh | 261 - .../test/check-chart-kinds-granted.sh | 89 - .../aks/operator/test/check-rbac-coverage.sh | 158 - .../operator/test/check-stderr-discarded.sh | 174 - .../fixtures/aks-upgrade/deployments.json | 56 - .../fixtures/aks-upgrade/ingress-memex.json | 13 - .../fixtures/aks-upgrade/manifests/cron.yml | 2 - .../fixtures/aks-upgrade/manifests/pdb.yaml | 7 - .../test/fixtures/aks-upgrade/metrics.txt | 4 - .../test/fixtures/aks-upgrade/nodes.json | 84 - .../test/fixtures/aks-upgrade/pdbs.json | 14 - .../test/fixtures/aks-upgrade/pods-memex.json | 50 - .../fixtures/aks-upgrade/pools.after.json | 68 - .../test/fixtures/aks-upgrade/pools.json | 68 - .../test/fixtures/aks-upgrade/runner.json | 12 - .../test/fixtures/aks-upgrade/show.after.json | 19 - .../test/fixtures/aks-upgrade/show.json | 19 - .../test/fixtures/aks-upgrade/upgrades.json | 39 - .../test/fixtures/aks-upgrade/versions.json | 73 - .../existing-pv-delete.json | 1 - .../existing-pv-other.json | 1 - .../artifact-store-bind/existing-pv-same.json | 1 - .../existing-pv-shared-handle.json | 1 - .../existing-pv-stale-secret.json | 1 - .../existing-pv-unlabelled.json | 1 - .../existing-pv-writable.json | 1 - .../existing-pvc-bound.json | 1 - .../existing-pvc-other.json | 1 - .../owner-pv-custom-secret.json | 1 - .../artifact-store-bind/owner-pv-disk.json | 1 - .../owner-pv-explicit-attrs.json | 1 - .../artifact-store-bind/owner-pv.json | 3 - .../owner-pvc-pending.json | 1 - .../artifact-store-bind/owner-pvc.json | 1 - .../test/fixtures/audit/clean/hooks.json | 1 - .../fixtures/audit/clean/live/configmap.json | 28 - .../fixtures/audit/clean/live/deployment.json | 96 - .../fixtures/audit/clean/live/ingress.json | 14 - .../test/fixtures/audit/clean/live/job.json | 14 - .../clean/live/persistentvolumeclaim.json | 33 - .../audit/clean/live/scaledobject.notype | 0 .../fixtures/audit/clean/live/secret.json | 40 - .../fixtures/audit/clean/live/service.json | 20 - .../audit/clean/live/serviceaccount.json | 13 - .../test/fixtures/audit/clean/manifest.json | 161 - .../test/fixtures/audit/clean/status.json | 28 - .../test/fixtures/audit/drift/hooks.json | 1 - .../fixtures/audit/drift/live/configmap.json | 19 - .../fixtures/audit/drift/live/cronjob.json | 6 - .../fixtures/audit/drift/live/deployment.json | 172 - .../fixtures/audit/drift/live/ingress.json | 7 - .../test/fixtures/audit/drift/live/job.json | 7 - .../drift/live/persistentvolumeclaim.json | 49 - .../fixtures/audit/drift/live/secret.json | 11 - .../audit/drift/live/secretproviderclass.json | 6 - .../fixtures/audit/drift/live/service.json | 7 - .../audit/drift/live/serviceaccount.json | 7 - .../test/fixtures/audit/drift/manifest.json | 54 - .../test/fixtures/audit/drift/status.json | 37 - .../audit/forbidden/live/configmap.json | 28 - .../audit/forbidden/live/deployment.json | 96 - .../audit/forbidden/live/role.forbidden | 0 .../fixtures/audit/forbidden/manifest.json | 139 - .../test/fixtures/audit/forbidden/status.json | 8 - .../test/fixtures/audit/missing/.keep | 0 .../live/configmap_memex-portal-config.json | 1 - ...s.secrets-store.csi.x-k8s.io_memex-kv.json | 1 - .../operator/test/fixtures/deploy/names.txt | 3 - .../test/fixtures/deploy/rendered.yaml | 15 - .../operator/test/fixtures/deploy/status.json | 1 - .../dump-analyze/deployments-claim.json | 50 - .../dump-analyze/deployments-data.json | 53 - .../dump-analyze/deployments-emptydir.json | 38 - .../dump-analyze/deployments-noroot.json | 28 - .../fixtures/dump-analyze/pod-partial.txt | 35 - .../fixtures/dump-analyze/pod-refused.txt | 2 - .../test/fixtures/dump-analyze/pod.txt | 55 - .../inline-env/absent/deployment.json | 95 - .../base/configmap_memex-portal-config.json | 9 - .../inline-env/base/deployment.after.json | 95 - .../fixtures/inline-env/base/deployment.json | 60 - .../base/secret_memex-portal-keyvault.json | 10 - .../base/secret_memex-portal-secrets.json | 9 - .../differ/secret_memex-portal-keyvault.json | 13 - .../inline-env/paused/deployment.json | 109 - .../inline-env/rolling/deployment.json | 108 - .../inline-env/rotated/deployment.json | 95 - .../rotated/secret_memex-portal-keyvault.json | 10 - .../inline-env/stuck/deployment.after.json | 60 - .../inline-env/valuefrom/deployment.json | 112 - .../test/fixtures/migrate/manifest.yaml | 2 - .../test/fixtures/migrate/rendered.json | 9 - .../operator/test/fixtures/pv-purge/pvs.json | 12 - .../pv-resize/absent/storageclass.json | 9 - .../test/fixtures/pv-resize/already/pvc.json | 25 - .../pv-resize/already/storageclass.json | 9 - .../filesystem-pending/pvc.after.json | 31 - .../pv-resize/filesystem-pending/pvc.json | 25 - .../filesystem-pending/storageclass.json | 9 - .../fixtures/pv-resize/grows/pvc.after.json | 25 - .../test/fixtures/pv-resize/grows/pvc.json | 25 - .../pv-resize/grows/storageclass.json | 9 - .../fixtures/pv-resize/no-expansion/pvc.json | 25 - .../pv-resize/no-expansion/storageclass.json | 9 - .../test/fixtures/pv-resize/pending/pvc.json | 25 - .../pv-resize/pending/storageclass.json | 9 - .../fixtures/pv-resize/stuck/pvc.after.json | 25 - .../test/fixtures/pv-resize/stuck/pvc.json | 25 - .../pv-resize/stuck/storageclass.json | 9 - .../storage-prune/deployments-emptydir.json | 3 - .../storage-prune/deployments-plain.json | 3 - .../storage-prune/deployments-shadowed.json | 6 - .../storage-prune/deployments-subpath.json | 3 - .../fixtures/storage-prune/pod-partial.txt | 3 - .../fixtures/storage-prune/pod-refused.txt | 1 - deploy/aks/operator/test/run-tests.sh | 4994 ----------------- .../aks/operator/test/stderr-discarded.allow | 26 - deploy/aks/operator/test/stubs/aks-upgrade/az | 46 - .../aks/operator/test/stubs/aks-upgrade/curl | 7 - .../operator/test/stubs/aks-upgrade/kubectl | 30 - .../test/stubs/artifact-store-bind/kubectl | 41 - deploy/aks/operator/test/stubs/az-session/az | 36 - .../aks/operator/test/stubs/az-session/curl | 26 - .../operator/test/stubs/az-session/pg_dump | 16 - .../aks/operator/test/stubs/db-release/helm | 6 - .../operator/test/stubs/db-release/kubectl | 68 - deploy/aks/operator/test/stubs/db-reset/az | 16 - deploy/aks/operator/test/stubs/db-reset/psql | 69 - deploy/aks/operator/test/stubs/deploy/az | 51 - deploy/aks/operator/test/stubs/deploy/helm | 43 - .../test/stubs/deploy/hosting-migrate | 14 - deploy/aks/operator/test/stubs/deploy/kubectl | 53 - .../operator/test/stubs/dump-analyze/dotnet | 57 - .../operator/test/stubs/dump-analyze/kubectl | 37 - deploy/aks/operator/test/stubs/helm | 22 - .../aks/operator/test/stubs/image-mirror/az | 19 - .../operator/test/stubs/image-mirror/crane | 38 - .../operator/test/stubs/inline-env/kubectl | 54 - .../test/stubs/interlock/hosting-migrate | 13 - .../aks/operator/test/stubs/interlock/kubectl | 15 - deploy/aks/operator/test/stubs/inventory/psql | 17 - deploy/aks/operator/test/stubs/kubectl | 28 - deploy/aks/operator/test/stubs/kv-copy/az | 44 - deploy/aks/operator/test/stubs/kv-ensure/az | 63 - deploy/aks/operator/test/stubs/kv-rotate/az | 19 - deploy/aks/operator/test/stubs/kv-set/az | 43 - deploy/aks/operator/test/stubs/kv-set/kubectl | 24 - deploy/aks/operator/test/stubs/kv-status/az | 39 - .../aks/operator/test/stubs/kv-synced/kubectl | 46 - .../aks/operator/test/stubs/kv-values-set/az | 50 - .../test/stubs/manifest-apply/kubectl | 29 - deploy/aks/operator/test/stubs/migrate/helm | 11 - .../aks/operator/test/stubs/migrate/kubectl | 61 - deploy/aks/operator/test/stubs/pg/az | 35 - deploy/aks/operator/test/stubs/pg/pg_dump | 12 - deploy/aks/operator/test/stubs/pg/pg_restore | 8 - deploy/aks/operator/test/stubs/pg/psql | 15 - deploy/aks/operator/test/stubs/pull-secret/az | 22 - .../operator/test/stubs/pull-secret/kubectl | 73 - deploy/aks/operator/test/stubs/pv-purge/az | 3 - .../aks/operator/test/stubs/pv-purge/kubectl | 18 - .../aks/operator/test/stubs/pv-resize/kubectl | 29 - .../operator/test/stubs/registry-coupon/curl | 38 - .../operator/test/stubs/registry-register/az | 56 - deploy/aks/operator/test/stubs/registry/curl | 120 - deploy/aks/operator/test/stubs/run/az | 17 - deploy/aks/operator/test/stubs/signin-app/az | 49 - .../operator/test/stubs/storage-prune/kubectl | 44 - deploy/aks/operator/test/stubs/verify/curl | 8 - deploy/aks/operator/test/stubs/verify/openssl | 18 - .../Data/Architecture.md | 2 +- .../Architecture/CarvingProjectsOutOfCore.md | 27 + .../Data/Architecture/ChartDriftSemantics.md | 2 +- .../Architecture/ContainerRegistryInMemex.md | 2 +- .../Architecture/DebuggingNativeCrashes.md | 4 +- .../Data/Architecture/DeniedIsNotAbsent.md | 4 +- .../Data/Architecture/DeploymentAKS.md | 6 +- .../Architecture/OperatorCredentialsByName.md | 230 - .../Data/Architecture/PinnedImageRetention.md | 4 +- .../PlanningADatabaseMigration.md | 4 +- .../Data/Architecture/ReleaseEditions.md | 2 +- .../Architecture/RemovingAKeyFromTheRecord.md | 2 +- .../Architecture/SecretsWriteOnlyEntry.md | 2 +- .../Data/Architecture/SelfUpdateSchemaWall.md | 2 +- 244 files changed, 172 insertions(+), 20409 deletions(-) delete mode 100644 .github/workflows/hosting-operator.yml create mode 100644 .github/workflows/memex-local.yml delete mode 100644 deploy/aks/operator/Dockerfile delete mode 100644 deploy/aks/operator/bin/_audit.jq delete mode 100644 deploy/aks/operator/bin/_common.sh delete mode 100644 deploy/aks/operator/bin/_dump_analyze_pod.sh delete mode 100644 deploy/aks/operator/bin/_storage_prune_pod.sh delete mode 100755 deploy/aks/operator/bin/_values_half.py delete mode 100755 deploy/aks/operator/bin/hosting-aks-upgrade delete mode 100755 deploy/aks/operator/bin/hosting-artifact-store-bind delete mode 100755 deploy/aks/operator/bin/hosting-audit delete mode 100755 deploy/aks/operator/bin/hosting-backup delete mode 100755 deploy/aks/operator/bin/hosting-chart-drift delete mode 100755 deploy/aks/operator/bin/hosting-db-release delete mode 100755 deploy/aks/operator/bin/hosting-db-reset delete mode 100755 deploy/aks/operator/bin/hosting-deploy delete mode 100755 deploy/aks/operator/bin/hosting-dns delete mode 100755 deploy/aks/operator/bin/hosting-dump-analyze delete mode 100755 deploy/aks/operator/bin/hosting-export delete mode 100755 deploy/aks/operator/bin/hosting-federate delete mode 100755 deploy/aks/operator/bin/hosting-image-mirror delete mode 100755 deploy/aks/operator/bin/hosting-inline-env-retire delete mode 100755 deploy/aks/operator/bin/hosting-inventory delete mode 100755 deploy/aks/operator/bin/hosting-kv-copy delete mode 100755 deploy/aks/operator/bin/hosting-kv-ensure delete mode 100755 deploy/aks/operator/bin/hosting-kv-purge delete mode 100755 deploy/aks/operator/bin/hosting-kv-rotate delete mode 100755 deploy/aks/operator/bin/hosting-kv-set delete mode 100755 deploy/aks/operator/bin/hosting-kv-state delete mode 100755 deploy/aks/operator/bin/hosting-kv-status delete mode 100755 deploy/aks/operator/bin/hosting-kv-synced delete mode 100755 deploy/aks/operator/bin/hosting-kv-values-set delete mode 100755 deploy/aks/operator/bin/hosting-manifest-apply delete mode 100755 deploy/aks/operator/bin/hosting-migrate delete mode 100755 deploy/aks/operator/bin/hosting-pull-secret delete mode 100755 deploy/aks/operator/bin/hosting-pv-purge delete mode 100755 deploy/aks/operator/bin/hosting-pv-resize delete mode 100755 deploy/aks/operator/bin/hosting-redirect delete mode 100755 deploy/aks/operator/bin/hosting-registry-coupon delete mode 100755 deploy/aks/operator/bin/hosting-registry-key delete mode 100755 deploy/aks/operator/bin/hosting-registry-register delete mode 100755 deploy/aks/operator/bin/hosting-restore delete mode 100755 deploy/aks/operator/bin/hosting-signin-app delete mode 100755 deploy/aks/operator/bin/hosting-storage-prune delete mode 100755 deploy/aks/operator/bin/hosting-tls delete mode 100755 deploy/aks/operator/bin/hosting-verify delete mode 100755 deploy/aks/operator/bin/hosting-verify-backup delete mode 100755 deploy/aks/operator/bin/hosting-verify-catalog delete mode 100755 deploy/aks/operator/bin/hosting-verify-restore delete mode 100755 deploy/aks/operator/bin/run.sh delete mode 100755 deploy/aks/operator/test/check-chart-kinds-granted.sh delete mode 100755 deploy/aks/operator/test/check-rbac-coverage.sh delete mode 100755 deploy/aks/operator/test/check-stderr-discarded.sh delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/deployments.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/ingress-memex.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/manifests/cron.yml delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/manifests/pdb.yaml delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/metrics.txt delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/nodes.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/pdbs.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/pods-memex.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/pools.after.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/pools.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/runner.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/show.after.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/show.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/upgrades.json delete mode 100644 deploy/aks/operator/test/fixtures/aks-upgrade/versions.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pv-delete.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pv-other.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pv-same.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pv-shared-handle.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pv-stale-secret.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pv-unlabelled.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pv-writable.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pvc-bound.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/existing-pvc-other.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/owner-pv-custom-secret.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/owner-pv-disk.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/owner-pv-explicit-attrs.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/owner-pv.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/owner-pvc-pending.json delete mode 100644 deploy/aks/operator/test/fixtures/artifact-store-bind/owner-pvc.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/hooks.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/configmap.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/ingress.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/job.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/persistentvolumeclaim.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/scaledobject.notype delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/secret.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/service.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/live/serviceaccount.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/manifest.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/clean/status.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/hooks.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/configmap.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/cronjob.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/ingress.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/job.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/persistentvolumeclaim.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/secret.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/secretproviderclass.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/service.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/live/serviceaccount.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/manifest.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/drift/status.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/forbidden/live/configmap.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/forbidden/live/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/forbidden/live/role.forbidden delete mode 100644 deploy/aks/operator/test/fixtures/audit/forbidden/manifest.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/forbidden/status.json delete mode 100644 deploy/aks/operator/test/fixtures/audit/missing/.keep delete mode 100644 deploy/aks/operator/test/fixtures/deploy/live/configmap_memex-portal-config.json delete mode 100644 deploy/aks/operator/test/fixtures/deploy/live/secretproviderclass.secrets-store.csi.x-k8s.io_memex-kv.json delete mode 100644 deploy/aks/operator/test/fixtures/deploy/names.txt delete mode 100644 deploy/aks/operator/test/fixtures/deploy/rendered.yaml delete mode 100644 deploy/aks/operator/test/fixtures/deploy/status.json delete mode 100644 deploy/aks/operator/test/fixtures/dump-analyze/deployments-claim.json delete mode 100644 deploy/aks/operator/test/fixtures/dump-analyze/deployments-data.json delete mode 100644 deploy/aks/operator/test/fixtures/dump-analyze/deployments-emptydir.json delete mode 100644 deploy/aks/operator/test/fixtures/dump-analyze/deployments-noroot.json delete mode 100644 deploy/aks/operator/test/fixtures/dump-analyze/pod-partial.txt delete mode 100644 deploy/aks/operator/test/fixtures/dump-analyze/pod-refused.txt delete mode 100644 deploy/aks/operator/test/fixtures/dump-analyze/pod.txt delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/absent/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/base/configmap_memex-portal-config.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/base/deployment.after.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/base/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/base/secret_memex-portal-keyvault.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/base/secret_memex-portal-secrets.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/differ/secret_memex-portal-keyvault.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/paused/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/rolling/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/rotated/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/rotated/secret_memex-portal-keyvault.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/stuck/deployment.after.json delete mode 100644 deploy/aks/operator/test/fixtures/inline-env/valuefrom/deployment.json delete mode 100644 deploy/aks/operator/test/fixtures/migrate/manifest.yaml delete mode 100644 deploy/aks/operator/test/fixtures/migrate/rendered.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-purge/pvs.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/absent/storageclass.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/already/pvc.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/already/storageclass.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/filesystem-pending/pvc.after.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/filesystem-pending/pvc.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/filesystem-pending/storageclass.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/grows/pvc.after.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/grows/pvc.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/grows/storageclass.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/no-expansion/pvc.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/no-expansion/storageclass.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/pending/pvc.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/pending/storageclass.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/stuck/pvc.after.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/stuck/pvc.json delete mode 100644 deploy/aks/operator/test/fixtures/pv-resize/stuck/storageclass.json delete mode 100644 deploy/aks/operator/test/fixtures/storage-prune/deployments-emptydir.json delete mode 100644 deploy/aks/operator/test/fixtures/storage-prune/deployments-plain.json delete mode 100644 deploy/aks/operator/test/fixtures/storage-prune/deployments-shadowed.json delete mode 100644 deploy/aks/operator/test/fixtures/storage-prune/deployments-subpath.json delete mode 100644 deploy/aks/operator/test/fixtures/storage-prune/pod-partial.txt delete mode 100644 deploy/aks/operator/test/fixtures/storage-prune/pod-refused.txt delete mode 100755 deploy/aks/operator/test/run-tests.sh delete mode 100644 deploy/aks/operator/test/stderr-discarded.allow delete mode 100755 deploy/aks/operator/test/stubs/aks-upgrade/az delete mode 100755 deploy/aks/operator/test/stubs/aks-upgrade/curl delete mode 100755 deploy/aks/operator/test/stubs/aks-upgrade/kubectl delete mode 100755 deploy/aks/operator/test/stubs/artifact-store-bind/kubectl delete mode 100755 deploy/aks/operator/test/stubs/az-session/az delete mode 100755 deploy/aks/operator/test/stubs/az-session/curl delete mode 100755 deploy/aks/operator/test/stubs/az-session/pg_dump delete mode 100755 deploy/aks/operator/test/stubs/db-release/helm delete mode 100755 deploy/aks/operator/test/stubs/db-release/kubectl delete mode 100755 deploy/aks/operator/test/stubs/db-reset/az delete mode 100755 deploy/aks/operator/test/stubs/db-reset/psql delete mode 100755 deploy/aks/operator/test/stubs/deploy/az delete mode 100755 deploy/aks/operator/test/stubs/deploy/helm delete mode 100755 deploy/aks/operator/test/stubs/deploy/hosting-migrate delete mode 100755 deploy/aks/operator/test/stubs/deploy/kubectl delete mode 100755 deploy/aks/operator/test/stubs/dump-analyze/dotnet delete mode 100755 deploy/aks/operator/test/stubs/dump-analyze/kubectl delete mode 100755 deploy/aks/operator/test/stubs/helm delete mode 100755 deploy/aks/operator/test/stubs/image-mirror/az delete mode 100755 deploy/aks/operator/test/stubs/image-mirror/crane delete mode 100755 deploy/aks/operator/test/stubs/inline-env/kubectl delete mode 100755 deploy/aks/operator/test/stubs/interlock/hosting-migrate delete mode 100755 deploy/aks/operator/test/stubs/interlock/kubectl delete mode 100755 deploy/aks/operator/test/stubs/inventory/psql delete mode 100755 deploy/aks/operator/test/stubs/kubectl delete mode 100755 deploy/aks/operator/test/stubs/kv-copy/az delete mode 100755 deploy/aks/operator/test/stubs/kv-ensure/az delete mode 100755 deploy/aks/operator/test/stubs/kv-rotate/az delete mode 100755 deploy/aks/operator/test/stubs/kv-set/az delete mode 100755 deploy/aks/operator/test/stubs/kv-set/kubectl delete mode 100755 deploy/aks/operator/test/stubs/kv-status/az delete mode 100755 deploy/aks/operator/test/stubs/kv-synced/kubectl delete mode 100755 deploy/aks/operator/test/stubs/kv-values-set/az delete mode 100755 deploy/aks/operator/test/stubs/manifest-apply/kubectl delete mode 100755 deploy/aks/operator/test/stubs/migrate/helm delete mode 100755 deploy/aks/operator/test/stubs/migrate/kubectl delete mode 100755 deploy/aks/operator/test/stubs/pg/az delete mode 100755 deploy/aks/operator/test/stubs/pg/pg_dump delete mode 100755 deploy/aks/operator/test/stubs/pg/pg_restore delete mode 100755 deploy/aks/operator/test/stubs/pg/psql delete mode 100755 deploy/aks/operator/test/stubs/pull-secret/az delete mode 100755 deploy/aks/operator/test/stubs/pull-secret/kubectl delete mode 100755 deploy/aks/operator/test/stubs/pv-purge/az delete mode 100755 deploy/aks/operator/test/stubs/pv-purge/kubectl delete mode 100755 deploy/aks/operator/test/stubs/pv-resize/kubectl delete mode 100755 deploy/aks/operator/test/stubs/registry-coupon/curl delete mode 100755 deploy/aks/operator/test/stubs/registry-register/az delete mode 100755 deploy/aks/operator/test/stubs/registry/curl delete mode 100755 deploy/aks/operator/test/stubs/run/az delete mode 100755 deploy/aks/operator/test/stubs/signin-app/az delete mode 100755 deploy/aks/operator/test/stubs/storage-prune/kubectl delete mode 100755 deploy/aks/operator/test/stubs/verify/curl delete mode 100755 deploy/aks/operator/test/stubs/verify/openssl delete mode 100644 src/MeshWeaver.Documentation/Data/Architecture/OperatorCredentialsByName.md diff --git a/.claude/skills/new-deployment/SKILL.md b/.claude/skills/new-deployment/SKILL.md index 3cfeb314a4..14842326fb 100644 --- a/.claude/skills/new-deployment/SKILL.md +++ b/.claude/skills/new-deployment/SKILL.md @@ -89,7 +89,7 @@ Every one of these is a **prerequisite the Provision REFUSES without**, so do th ## 2. What the operator does — the Provision plan, in order Composed by `InstanceActionPlan.ProvisionSteps` (MeshWeaver.Plugins, in-mesh — it compiles in the -portal, no CI type-checks it), run by `deploy/aks/operator/bin/run.sh` in a Job in `memex-ops`, +portal, no CI type-checks it), run by `MeshWeaver.Plugins/deploy/hosting-operator/bin/run.sh` in a Job in `memex-ops`, fail-fast, the failing STEP named on the node: 1. `az postgres flexible-server db create` — control plane, workload identity. (Memex#132 — diff --git a/.github/manual-keyvault.allow b/.github/manual-keyvault.allow index aae7fc9cb1..ccfad53c00 100644 --- a/.github/manual-keyvault.allow +++ b/.github/manual-keyvault.allow @@ -7,28 +7,10 @@ # 🚨 "TRANSITIONAL" marks a governed verb that still READS a value (or checks existence with a value # read). Migration step 3 in the design page rewrites each so the writer identity never needs `get`; # the line goes when that rewrite lands. - -# The writer's own verbs: set with tags, status from metadata, lifecycle. None reads a value. -deploy/aks/operator/bin/hosting-kv-set — writer verb: writes through --file, confirms by list-versions. TRANSITIONAL (policy secret-actions-interim-actions-lane): its --generate mode also runs as hosting-operator on the aks-ops lane until the writer identity is provisioned and every secret action runs in the in-cluster Job under it -deploy/aks/operator/bin/hosting-kv-state — writer verb: set-attributes / delete / recover / purge, no value -deploy/aks/operator/bin/hosting-kv-purge — teardown verb: deletes an instance's objects by prefix, no value read - -# Governed provisioning verbs that still run as hosting-operator. -deploy/aks/operator/bin/_common.sh — TRANSITIONAL: hosting::pg_password reads the server admin password to compose connection strings -deploy/aks/operator/bin/hosting-kv-ensure — TRANSITIONAL: the admin-password read for the connection string (existence is checked with `list`), and the download of an existing helm-values- to filter it to the record's declared keys (policy one-values-half-per-release) -deploy/aks/operator/bin/hosting-kv-values-set — TRANSITIONAL: the governed key-level writer of helm-values- downloads the half to change named declared keys and keep the rest (a read-modify-write); retired with the vault half -deploy/aks/operator/bin/hosting-kv-copy — TRANSITIONAL: copyFrom reads the source value to write the per-prefix copy -deploy/aks/operator/bin/hosting-kv-rotate — TRANSITIONAL: reads the vaulted registry key to compare before rotating -deploy/aks/operator/bin/hosting-registry-register — TRANSITIONAL: reads the held instance key and the bootstrap key to call the registry -deploy/aks/operator/bin/hosting-registry-coupon — TRANSITIONAL: reads the registry's coupon-mint credential to call the registry; writes the minted coupon through --file, existence checked with `list` -deploy/aks/operator/bin/hosting-signin-app — writes the minted Entra client secret through --file; existence is checked with `list`, no value read -deploy/aks/operator/bin/hosting-image-mirror — TRANSITIONAL: reads the registry publisher password to push the mirror -deploy/aks/operator/bin/hosting-pull-secret — TRANSITIONAL: reads the registry pull key to render the image pull Secret -deploy/aks/operator/bin/hosting-deploy — TRANSITIONAL: downloads the helm values half (helm-values-); retired with the vault half - -# Their behaviour suite and its stand-in binaries (they answer the commands, they do not run them). -deploy/aks/operator/test/run-tests.sh — asserts the verbs write through --file, never print a value and never read one back -deploy/aks/operator/test/stubs/*/az — stand-in az binaries for that suite +# +# The hosting operator's governed `hosting-kv-*` verbs moved to Systemorph/MeshWeaver.Plugins with the +# operator ("core slimming"); their entries live in its deploy/hosting-operator/manual-keyvault.allow, +# and its hosting-operator.yml runs this gate over that directory. # This gate, which has to spell the commands it refuses. .github/scripts/check-manual-keyvault.py — the gate itself diff --git a/.github/release-channels.json b/.github/release-channels.json index e6c65927a9..a6980eb619 100644 --- a/.github/release-channels.json +++ b/.github/release-channels.json @@ -321,14 +321,15 @@ }, { "workflow": "hosting-operator.yml", - "job": "publish", + "repo": "Systemorph/MeshWeaver.Plugins", + "job": "operator", "repositories": [ "hosting-operator" ], "registries": [ "meshweaver.azurecr.io" ], - "state": "every merge that touches the operator image" + "state": "every merge there that touches the operator, and hourly when core's chart or drift scripts moved (the operator moved to MeshWeaver.Plugins)" } ] }, diff --git a/.github/scripts/check-workflow-shell.py b/.github/scripts/check-workflow-shell.py index c6e8289dbd..b4f3a80b98 100755 --- a/.github/scripts/check-workflow-shell.py +++ b/.github/scripts/check-workflow-shell.py @@ -64,8 +64,8 @@ WORKFLOW_GLOBS = [".github/workflows/*.yml", ".github/workflows/*.yaml"] SHELL_GLOBS = [".github/scripts/*.sh"] -# shellcheck severity. Matches the level the operator-scripts gate already runs at -# (hosting-operator.yml), so one repo does not carry two different bars for the same tool. +# shellcheck severity. Matches the level memex-local.yml runs at (and the operator scripts' gate, which +# moved to MeshWeaver.Plugins), so one repo does not carry two different bars for the same tool. SHELLCHECK_SEVERITY = "warning" # A `${{ … }}` expression is not shell. Substituting a LITERAL for it makes shellcheck diff --git a/.github/scripts/lock-pinned-digests.py b/.github/scripts/lock-pinned-digests.py index d6c73807b7..e7812c2188 100755 --- a/.github/scripts/lock-pinned-digests.py +++ b/.github/scripts/lock-pinned-digests.py @@ -80,8 +80,8 @@ Axis 2 is a different subject with a different shape (a TAG, in a helm overlay, not a digest in a workflow) and it necessarily has its own reader. Its scope is deliberately narrow, and the boundary is measured rather than guessed: `values*.yml` / `values*.yaml` under a `deploy/` or `deployments/` -path segment. Widening it to every YAML/JSON under `deploy/` was tried and rejected — core's -`deploy/aks/operator/test/fixtures/**` carry image references to tags that never existed +path segment. Widening it to every YAML/JSON under `deploy/` was tried and rejected — the hosting +operator's `deploy/hosting-operator/test/fixtures/**` (MeshWeaver.Plugins; core's `deploy/aks/operator` until it moved) carry image references to tags that never existed (`memex-portal-ai:3.0.0-rc8.ci.5000`), which would red this job nightly over test fixtures, and `deploy/aks/manifests/observability/log-watcher.yaml` names the FLOATING tag `:latest`, which has no fixed manifest to protect. @@ -5490,7 +5490,7 @@ def check(condition: bool, message: str) -> None: "ARM 13: Memex's real overlay path was rejected") check(is_overlay_path("deploy/whisper/helm/values.yaml"), "ARM 13: this repository's whisper chart values file was rejected") - check(not is_overlay_path("deploy/aks/operator/test/fixtures/audit/clean/manifest.json"), + check(not is_overlay_path("deploy/hosting-operator/test/fixtures/audit/clean/manifest.json"), "ARM 13: a TEST FIXTURE was accepted as a deployment overlay — those name tags that " "never existed and would red this job nightly") check(not is_overlay_path("deploy/aks/manifests/observability/log-watcher.yaml"), diff --git a/.github/scripts/release-channels.py b/.github/scripts/release-channels.py index 1d7fd706eb..de7bb25deb 100644 --- a/.github/scripts/release-channels.py +++ b/.github/scripts/release-channels.py @@ -46,6 +46,14 @@ PLACEHOLDERS = ("major", "minor", "patch") REQUIRED_FIELDS = ("id", "tag", "edition", "meaning", "writers", "notation", "status") BINDING_FIELDS = ("workflow", "job", "repositories") +# The repository whose workflows this file's bindings name by default. A binding may name ANOTHER +# repository's workflow with `"repo": "/"` — the hosting operator's image is published +# by Systemorph/MeshWeaver.Plugins (its hosting-operator.yml), and its `main` label must stay +# declared, and so protected, here. Such a binding is shape-validated, but its workflow cannot be +# read from this tree, so `check` does not open it; that repository's lane spells the tag and the +# repository itself. +THIS_REPO = "Systemorph/MeshWeaver" +REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") # A label must look like a tag and must NEVER be SELECTABLE as a version. The self-updater # (memex/Memex.Portal.Shared/SelfUpdate/VersionSelect.cs) keeps every tag matching # `PlatformVersionTag` = ^\d+\.\d+\.\d+([-+].*)?$ EXCEPT one ending in `-latest` (`MovingPointer`), so @@ -186,6 +194,8 @@ def validate(doc: dict) -> list[str]: problems.append(f"{at}: `repositories` must be a list of repository names") if binding.get("notation", "any") not in NOTATIONS: problems.append(f"{at}: notation {binding.get('notation')!r} is not one of {NOTATIONS}") + if "repo" in binding and not (isinstance(binding["repo"], str) and REPO_RE.match(binding["repo"])): + problems.append(f"{at}: `repo` must be '/', got {binding['repo']!r}") return problems @@ -280,6 +290,8 @@ def check(doc: dict, root: Path, platform_version=None, is_floating=None) -> lis workflows = root / ".github" / "workflows" for entry in channels(doc): for binding in entry["writers"]: + if binding.get("repo", THIS_REPO) != THIS_REPO: + continue # another repository's workflow: shape-validated above, not readable here wf = binding["workflow"] path = workflows / wf if not path.is_file(): @@ -451,8 +463,22 @@ def line_pointers(version: str) -> list[str]: # The publishers outside main-cd/release.yml are declared too (regression: they were omitted). expect("edge is protected (edge-images.yml writes it for portal and migration)", is_protected(real, "edge") and tags_for(real, "3.1.10050", "edge-images.yml", repository="memex-migration") == ["edge"]) - expect("hosting-operator:main is declared (hosting-operator.yml#publish)", - tags_for(real, "3.1.10050", "hosting-operator.yml", job="publish", repository="hosting-operator") == ["main"]) + expect("hosting-operator:main is declared (MeshWeaver.Plugins hosting-operator.yml#operator)", + tags_for(real, "3.1.10050", "hosting-operator.yml", job="operator", repository="hosting-operator") == ["main"]) + expect("the hosting-operator writer names the repository that publishes it", + any(b.get("repo") == "Systemorph/MeshWeaver.Plugins" and b["workflow"] == "hosting-operator.yml" + for e in real["channels"] if e["id"] == "main" for b in e["writers"])) + # A binding into ANOTHER repository is not opened here; one into THIS repository still is. + elsewhere = json.loads(json.dumps(real)) + elsewhere["channels"][3]["writers"][0].update(workflow="no-such-workflow.yml", repo="Systemorph/Elsewhere") + expect("a writer in another repository is not looked for in this tree", + not any("no-such-workflow.yml" in p for p in check(elsewhere, root, FakePV, always_floating))) + here = json.loads(json.dumps(real)) + here["channels"][3]["writers"][0].update(workflow="no-such-workflow.yml", repo=THIS_REPO) + expect("a writer that names THIS repository explicitly is still looked for", + any("does not exist" in p for p in check(here, root, FakePV, always_floating))) + expect("validate refuses a malformed `repo`", + bool(broken(lambda d: d["channels"][3]["writers"][0].update(repo="not a repo")))) expect("memex-portal-ai-base:latest is declared (base-image-acr.yml#base)", tags_for(real, "3.1.10050", "base-image-acr.yml", job="base", repository="memex-portal-ai-base") == ["latest"]) patterns = {p["id"]: p["repositories"] for p in protected_patterns(real)} diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 7c4e0db689..aaf9834358 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -1,7 +1,7 @@ name: Homebrew (memex-local) # memex-local is delivered through Homebrew — the tap Systemorph/homebrew-memex — and until this -# workflow existed nothing ever INSTALLED the formula: the Linux gate in hosting-operator.yml +# workflow existed nothing ever INSTALLED the formula: the Linux gate in memex-local.yml # parses the CLI, shellchecks it and runs its behaviour tests against stub tools, which is exactly # the layer where most of its defects lived, but `brew install` itself, the vendored chart layout, # the wrapper's MEMEX_LOCAL_SHARE/MEMEX_CHART_DIR plumbing and macOS's bash 3.2 were all asserted by diff --git a/.github/workflows/hosting-operator.yml b/.github/workflows/hosting-operator.yml deleted file mode 100644 index a62585a70d..0000000000 --- a/.github/workflows/hosting-operator.yml +++ /dev/null @@ -1,457 +0,0 @@ -name: Hosting Operator - -# The operator scripts are the executable half of instance lifecycle — they create namespaces, -# drop databases and delete DNS records — and nothing in CI opened them until now. -# -# 🚨 WHAT THIS GATE CAN AND CANNOT PROVE, stated here so a green tick is not read as more than it is. -# These scripts drive az/kubectl/helm/psql against a live Azure estate; a runner has none of those, -# and mocking them would only assert that the mocks agree with themselves. So this proves the layer -# where the dangerous bugs actually live — argument handling, the REFUSALS that stand in front of -# every destructive act, the ::hosting:: contract the mesh parses, and run.sh's ordering — and it -# proves the image still builds. Whether `az network dns record-set a add-record` behaves as -# expected is settled by the first real provision, which is why the rollout is staged. -# -# 🛡️ NO SKIP-TRAPDOOR (AGENTS.md → "A gate NEVER tests its own inputs"). Every input is in this -# repository. There is no secret, no cluster and no registry, so there is no condition under which -# this may decline to run and still render a tick: no `continue-on-error`, no input-shaped `if:`, -# and no path filter (a path filter turns "unrelated PR" into a skipped run, which is the same grey -# tick from the other side). - -on: - pull_request: - push: - branches: [main] - merge_group: - workflow_dispatch: - -permissions: - contents: read - -# One live run per ref: a newer push to the same PR branch cancels the run it supersedes; main and -# the merge queue never cancel (each main push is its own group, keyed on the commit). -concurrency: - group: hosting-operator-${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && github.sha || github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} - -jobs: - scripts: - name: Operator scripts (shellcheck + behaviour) - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v7 - # The values-half key check (bin/_values_half.py, policy `one-values-half-per-release`) is - # python3 + PyYAML — the operator image installs both; here they are installed explicitly - # rather than relied on from the runner image. - - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - run: pip install pyyaml - - # shellcheck ships on hosted runners, which is exactly why it is asserted rather than assumed: - # a dependency satisfied by luck of the runner image breaks on an image bump, as a "command - # not found" unrelated to the code being gated. - - name: Assert the tools this gate needs - run: | - set -euo pipefail - missing=() - command -v shellcheck >/dev/null 2>&1 || missing+=("shellcheck — apt-get install shellcheck") - command -v bash >/dev/null 2>&1 || missing+=("bash") - if [ ${#missing[@]} -gt 0 ]; then - echo "::error::the operator gate cannot run — provide:" - for m in "${missing[@]}"; do echo " • $m"; done - exit 1 - fi - shellcheck --version - - # 🚨 The regression guard for the trap that kept these scripts out of the repo entirely: - # .gitignore's `/deploy/**/bin/` (build output) also matched THIS bin/, so `git add` on them - # exited 0 and did nothing. The Dockerfile shipped COPYing a directory git refused to track. - # A checkout is the only place that can prove the re-include still holds. - - name: The operator scripts are actually TRACKED - run: | - set -euo pipefail - tracked=$(git ls-files deploy/aks/operator/bin/ | wc -l | tr -d ' ') - if [ "$tracked" -lt 46 ]; then - echo "::error::only ${tracked} file(s) tracked under deploy/aks/operator/bin/ — expected 46 (40 commands + run.sh + _common.sh + _audit.jq + _values_half.py + _dump_analyze_pod.sh + _storage_prune_pod.sh). The floor is the REAL count — raise it with every command added, or a lost file passes." - echo "::error::Almost certainly .gitignore matching this bin/ again. `git add` on an ignored path exits 0 and does NOTHING, so the loss is silent until a fresh checkout builds an image with no scripts in it." - git check-ignore -v deploy/aks/operator/bin/run.sh || true - exit 1 - fi - # Executability is part of the contract: the Dockerfile chmods, but run.sh is the Job's - # ENTRYPOINT and a non-executable command is a CrashLoop, not a warning. - nonexec=$(git ls-files -s deploy/aks/operator/bin/hosting-* deploy/aks/operator/bin/run.sh | awk '$1 != "100755"' | wc -l | tr -d ' ') - [ "$nonexec" -eq 0 ] || { echo "::error::${nonexec} operator command(s) are not committed executable (mode 100755)"; exit 1; } - echo "${tracked} files tracked, all commands executable" - - - name: Every script parses - run: | - set -euo pipefail - cd deploy/aks/operator/bin - for f in _common.sh run.sh hosting-*; do - bash -n "$f" || { echo "::error::${f} is not valid bash"; exit 1; } - done - echo "all $(ls _common.sh run.sh hosting-* | wc -l) scripts parse" - - - name: shellcheck - run: shellcheck -S warning deploy/aks/operator/bin/_common.sh deploy/aks/operator/bin/run.sh deploy/aks/operator/bin/hosting-* - - # 🚨 Every hosting-* command the mesh's plan can emit must EXIST. A plan step naming a script - # nobody wrote fails inside a Job, halfway through a lifecycle action, as "command not found" - # — which for a teardown means after the quiesce and possibly after the backup. - - name: Every command the plan can emit exists - run: | - set -euo pipefail - plan_src=$(mktemp) - # The plan lives in the Hosting plugin repo; the command names it emits are also listed - # here so this repo can gate them without a cross-repo checkout. Keep in step with - # MeshWeaver.Plugins/Hosting/InstanceAction/Source/InstanceActionPlan.cs. - cat > "$plan_src" <<'EOF' - hosting-aks-upgrade - hosting-artifact-store-bind - hosting-audit - hosting-backup - hosting-deploy - hosting-dns - hosting-export - hosting-federate - hosting-image-mirror - hosting-inline-env-retire - hosting-kv-copy - hosting-kv-ensure - hosting-kv-purge - hosting-kv-rotate - hosting-kv-set - hosting-pull-secret - hosting-pv-purge - hosting-pv-resize - hosting-redirect - hosting-registry-coupon - hosting-registry-key - hosting-registry-register - hosting-restore - hosting-signin-app - hosting-storage-prune - hosting-tls - hosting-verify - hosting-verify-backup - hosting-verify-catalog - hosting-verify-restore - EOF - missing=() - while read -r cmd; do - cmd="$(echo "$cmd" | tr -d '[:space:]')" - [ -n "$cmd" ] || continue - [ -x "deploy/aks/operator/bin/${cmd}" ] || missing+=("$cmd") - done < "$plan_src" - if [ ${#missing[@]} -gt 0 ]; then - echo "::error::the plan can emit commands this image does not carry: ${missing[*]}" - echo "::error::A missing one fails INSIDE a running lifecycle Job — for a teardown, after the quiesce." - exit 1 - fi - echo "all $(grep -c . "$plan_src") plan commands present and executable" - - # 🚨 THE REVERSE DIRECTION, and it is the half that actually rots. - # - # The check above asks "does every command the plan names exist?" — it can only fail when - # someone deletes a script. It is structurally blind to the opposite drift: a script that - # SHIPS in the image while no plan entry names it. That one is silent, because a list - # nobody compares against its subject simply stops describing it, and the gate goes on - # passing while its coverage narrows. Measured on 2026-09-01: `hosting-kv-rotate` had - # shipped, had zero behaviour tests, and was absent from the list above — so nothing in CI - # knew it existed. - # - # So every hosting-* that ships must be EITHER claimed by the plan above OR named here, - # deliberately, with the reason it is not yet wired. Adding a script without touching one - # of the two lists is now a red build rather than an unnoticed hole. - not_wired=$(mktemp) - cat > "$not_wired" <<'EOF' - # command why it ships without a plan entry - # - # (hosting-kv-rotate stood here until it moved up into the plan list: - # InstanceActionPlan.RotateRegistryKeySteps emits it.) - # - # 🚨 An entry here is NOT the transitional shape of scripts/*.allow — it does not expire - # with the pull request that adds it. It stands until the command becomes REACHABLE, and - # the change that makes it reachable is the one that moves the line up into the plan list. - # - # hosting-db-release ships from MeshWeaver#4436 (the operator image carries bin/ whole) and - # is EXERCISED: the behaviour suite drives it against test/stubs/db-release, so deleting - # the script still reds this gate rather than passing unnoticed. What it is not yet is - # REACHABLE — no step in MeshWeaver.Plugins' InstanceActionPlan.cs emits it, so no - # InstanceAction can run it on a cluster. Naming it in the plan list above would assert a - # wiring that does not exist and make that list's "keep in step with InstanceActionPlan.cs" - # false in the direction the list cannot check. - # - # BEFORE IT IS WIRED, both must be true: MeshWeaver.Plugins#1937 ("Deploy database - # release") merges, which is the change that adds the step emitting this command — it is - # held as a draft until #4436 merges and seals; and the widened ClusterRole its - # platform-layer preflight reads (nodes + customresourcedefinitions, MeshWeaver#4722) has - # reached the cluster through Systemorph/Memex's helm-release lane, since a Job never - # widens its own RBAC. The line moves into the plan list above in that same change. - hosting-db-release #4436 — ships and is behaviour-tested, but no plan step emits it yet: the plan half is draft Plugins#1937 - # - # hosting-migrate is the same shape (policy `roll-migrates-first`): it ships from - # MeshWeaver#5148 and the behaviour suite drives it against test/stubs/migrate, so deleting - # it reds this gate. The Roll plan step that emits it is draft Plugins#2219, held until an - # operator image carrying this command is on hosting-operator:main — a plan that emits it - # against an older image stops at "command not found" BEFORE the image moves. The line - # moves up into the plan list in the change that merges that plan step. - hosting-migrate #5148 — ships and is behaviour-tested, but no plan step emits it yet: the plan half is draft Plugins#2219 - # - # hosting-kv-status and hosting-kv-state are the write-only secret GUI's read and - # lifecycle verbs (policy `secrets-write-only-entry`, Doc/Architecture/SecretsWriteOnlyEntry): - # status from vault METADATA only, and enable/disable/delete/recover/purge, all run as the - # writer identity that never reads a value. They ship and are behaviour-tested against - # test/stubs/kv-status. The plan steps that emit them (SecretStatus / SecretLifecycle - # actions) are the MeshWeaver.Plugins half, which lands after this image is on - # hosting-operator:main, for the same reason as hosting-migrate above. - hosting-kv-status secrets-write-only-entry — ships and is behaviour-tested; the plan half is the Plugins secret-status action - hosting-kv-state secrets-write-only-entry — ships and is behaviour-tested; the plan half is the Plugins secret-lifecycle action - # - # hosting-kv-synced (MeshWeaver.Plugins#2778) is the same shape: the Reconcile's SyncVaultKeys - # remedy that emits it is the Plugins half, which merges after this image is on - # hosting-operator:main; the line moves up into the plan list once it has. - hosting-kv-synced Plugins#2778 — ships and is behaviour-tested; the plan half is the Plugins Reconcile SyncVaultKeys remedy - # - # hosting-db-reset (MeshWeaver.Plugins#2781, policy `reset-database-keeps-a-copy`) is the same - # shape: the ResetDatabase / PurgeKeptDatabase plan steps that emit it are the Plugins half, - # which runs only once an operator pin carries this command; the line moves up then. - hosting-db-reset Plugins#2781 — ships and is behaviour-tested; the plan half is the Plugins ResetDatabase / PurgeKeptDatabase actions - # - # hosting-inventory is the read-only node census (the Plugins `Inventory` InstanceAction): it - # ships and is behaviour-tested against test/stubs/inventory. Its plan step is the Plugins half, - # which merges after this image is on hosting-operator:main, for the same reason as - # hosting-migrate above; the line moves up into the plan list once it has. - hosting-inventory Inventory — ships and is behaviour-tested; the plan half is the Plugins Inventory action - # - # hosting-dump-analyze is the governed dump read (MeshWeaver#6432, the Plugins `AnalyzeDump` - # InstanceAction): it ships with its in-pod half (bin/_dump_analyze_pod.sh) and both are - # behaviour-tested against test/stubs/dump-analyze (a stub kubectl, and a stub dotnet-dump the - # real pod script runs). Its plan step is the Plugins half, which merges after this image is on - # hosting-operator:main, for the same reason as hosting-migrate above. - hosting-dump-analyze #6432 — ships and is behaviour-tested; the plan half is the Plugins AnalyzeDump action - # - # hosting-manifest-apply is the governed apply of a COMMITTED cluster manifest (the Plugins - # `ApplyClusterManifest` InstanceAction; Systemorph/Memex aks-ops.yml runs it on the lane from a - # MeshWeaver checkout — never in a Job, and it refuses to run where deploy/aks/manifests is not - # beside bin/). Behaviour-tested against test/stubs/manifest-apply. The plan half is the Plugins - # ApplyClusterManifest action; the line moves up into the plan list once that has merged. - hosting-manifest-apply ApplyClusterManifest — ships and is behaviour-tested; the plan half is the Plugins ApplyClusterManifest action - # - # hosting-kv-values-set is the governed key-level writer of the values half (helm-values-, - # policy one-values-half-per-release): behaviour-tested against test/stubs/kv-values-set. The plan - # step that emits it (a SetSecrets carrying a values-half key) is the Plugins half, which merges after - # this image is on hosting-operator:main; the line moves up into the plan list once it has. - hosting-kv-values-set values-half writer — ships and is behaviour-tested; the plan half is the Plugins SetSecrets values-half step - # - # hosting-chart-drift is the Chart Drift comparison moved off core's chart-drift.yml into the - # governed Audit (Systemorph/Memex#647): the estate's aks-ops lane runs it in the VNet as - # hosting-operator, beside hosting-audit, from the MeshWeaver checkout at the operator pin (it - # finds core's drift scripts at bin/../../scripts there, and at /opt/hosting/drift in this - # image). Behaviour-tested against stub drift scripts. The plan half is the Plugins Audit - # action's second step; the line moves up into the plan list once that has merged. - hosting-chart-drift Audit (chart drift) — ships and is behaviour-tested; the plan half is the Plugins Audit action - EOF - unclaimed=() - for path in deploy/aks/operator/bin/hosting-*; do - cmd="${path##*/}" - grep -qx "$cmd" "$plan_src" && continue - grep -qE "^${cmd}([[:space:]]|$)" "$not_wired" && continue - unclaimed+=("$cmd") - done - if [ ${#unclaimed[@]} -gt 0 ]; then - echo "::error::these operator commands ship but no plan entry names them: ${unclaimed[*]}" - echo "::error::Either add each to the plan list above (and keep MeshWeaver.Plugins' InstanceActionPlan.cs in step)," - echo "::error::or declare it in the not-yet-wired list with the reason. A shipped command nobody claims is" - echo "::error::untested and unreachable, and nothing else in CI will say so." - exit 1 - fi - echo "every shipped hosting-* command is either claimed by the plan or declared not-yet-wired" - # hosting-audit is two files: the command and the jq program beside it. A command that - # ships without its detection dies at "the detection program … is missing" inside a Job. - [ -f deploy/aks/operator/bin/_audit.jq ] || { echo "::error::bin/_audit.jq is missing — hosting-audit cannot compute anything without it"; exit 1; } - # hosting-dump-analyze is two files as well: the in-pod half it hands the analysis Job. - [ -f deploy/aks/operator/bin/_dump_analyze_pod.sh ] || { echo "::error::bin/_dump_analyze_pod.sh is missing — hosting-dump-analyze has nothing to run in the analysis Job"; exit 1; } - # Parse-check the program. jq exits 3 on a PROGRAM error and 5 on a runtime error(); - # empty inputs legitimately reach the error() refusal, so only exit 3 is a failure here — - # the behaviour tests below exercise the real detection against fixtures. - set +e - jq -n -f deploy/aks/operator/bin/_audit.jq --slurpfile manifest /dev/null --slurpfile live /dev/null \ - --arg namespace n --arg release r --arg deployment d --arg container c --argjson revision 1 \ - --arg chart "" --arg auditedAt now --arg skipped "" --argjson persistence '{}' >/dev/null 2>"$RUNNER_TEMP/audit-jq.err" - jq_rc=$? - set -e - if [ "$jq_rc" -eq 3 ]; then - echo "::error::_audit.jq does not parse:"; cat "$RUNNER_TEMP/audit-jq.err"; exit 1 - fi - echo "_audit.jq parses (exit ${jq_rc} on empty inputs is its own refusal, as designed)" - - - name: The test stubs are TRACKED and executable - # The same .gitignore trap as bin/ above, one directory over: the pull-secret behaviour - # tests run against stubs under test/stubs/pull-secret/, and a stub git refused to track - # makes those tests fail on a fresh checkout as "command not found" — or, worse, fall - # through to a REAL az/kubectl on a runner that has one. - # - # 🚨 db-release/helm is why the worse half is not hypothetical: the hosting-db-release - # happy-path case runs the command all the way to its post-install verification, so an - # untracked or non-executable stub there means a REAL `helm upgrade --install --namespace - # fabrikam` on the runner. Both db-release stubs were absent from this list until the - # MeshWeaver#4722 sweep added them with the helm one. - run: | - set -euo pipefail - for f in deploy/aks/operator/test/stubs/pull-secret/az deploy/aks/operator/test/stubs/pull-secret/kubectl deploy/aks/operator/test/stubs/pv-resize/kubectl deploy/aks/operator/test/stubs/inline-env/kubectl deploy/aks/operator/test/stubs/kv-rotate/az deploy/aks/operator/test/stubs/kv-ensure/az deploy/aks/operator/test/stubs/kv-copy/az deploy/aks/operator/test/stubs/registry-register/az deploy/aks/operator/test/stubs/registry-coupon/curl deploy/aks/operator/test/stubs/signin-app/az deploy/aks/operator/test/stubs/image-mirror/az deploy/aks/operator/test/stubs/image-mirror/crane deploy/aks/operator/test/stubs/deploy/az deploy/aks/operator/test/stubs/kv-set/az deploy/aks/operator/test/stubs/kv-set/kubectl deploy/aks/operator/test/stubs/kv-values-set/az deploy/aks/operator/test/stubs/kv-status/az deploy/aks/operator/test/stubs/db-release/kubectl deploy/aks/operator/test/stubs/db-release/helm deploy/aks/operator/test/stubs/pg/az deploy/aks/operator/test/stubs/pg/pg_dump deploy/aks/operator/test/stubs/pg/psql deploy/aks/operator/test/stubs/pg/pg_restore deploy/aks/operator/test/stubs/inventory/psql deploy/aks/operator/test/stubs/aks-upgrade/az deploy/aks/operator/test/stubs/aks-upgrade/kubectl deploy/aks/operator/test/stubs/aks-upgrade/curl deploy/aks/operator/test/stubs/db-reset/psql deploy/aks/operator/test/stubs/db-reset/az; do - git ls-files --error-unmatch "$f" >/dev/null 2>&1 || { echo "::error::${f} is not tracked"; git check-ignore -v "$f" || true; exit 1; } - [ "$(git ls-files -s "$f" | awk '{print $1}')" = "100755" ] || { echo "::error::${f} is not committed executable (mode 100755)"; exit 1; } - done - echo "operator test stubs tracked and executable" - - - name: Behaviour tests - run: deploy/aks/operator/test/run-tests.sh - - # 🚨 memex-local is the OTHER operator script in this repo — ~2000 lines that stand up every - # developer's local portal — and nothing in CI opened it until MeshWeaver#2367. What that - # cost, in one file: `memex-local help` was an unbounded FORK BOMB (a backtick in an unquoted - # heredoc re-entered main()); `helm_deploy` aborted on macOS bash 3.2 for the paths the script - # itself calls normal; and `up`/`update` reported success on a portal with no sign-in page and - # no view packs. Three live defects, zero red ticks, because nothing ever ran it. - - name: memex-local parses - run: bash -n deploy/homebrew/bin/memex-local - - - name: shellcheck (memex-local) - run: shellcheck -S warning deploy/homebrew/bin/memex-local deploy/homebrew/test/run-tests.sh - - # 🚨 This runner is bash 5 and has no colima/kubectl/helm, so what it CAN prove is asserted by - # running the CLI against stub tools (every red state of the usability verification, plus that - # a green one is reachable) and by static assertions for the two traps a Linux runner can - # never reach by execution — the heredoc backtick and macOS bash 3.2's empty-array abort. - - name: Behaviour tests (memex-local) - run: deploy/homebrew/test/run-tests.sh - - image: - name: Operator image builds - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v7 - # Build only — nothing is pushed from a pull request. The point is that the Dockerfile and its - # build context stay in step: the context is `deploy/` (not the operator directory) because - # the image carries the CHART as well as the scripts, and that is easy to break silently. - - name: Build - run: docker build -f deploy/aks/operator/Dockerfile -t hosting-operator:ci deploy/ - - - name: The image carries the scripts AND the chart - run: | - set -euo pipefail - # A chart-less image would deploy nothing and an image without run.sh cannot start; both - # are build-context mistakes that only surface during a real provision. - docker run --rm --entrypoint /bin/bash hosting-operator:ci -c ' - set -e - test -x /opt/hosting/bin/run.sh || { echo "run.sh missing"; exit 1; } - test -f /opt/hosting/chart/Chart.yaml || { echo "chart missing — build context must be deploy/"; exit 1; } - command -v hosting-deploy >/dev/null || { echo "hosting-* not on PATH"; exit 1; } - command -v helm kubectl az pg_dump jq crane >/dev/null || { echo "a required tool is missing"; exit 1; } - test -f /opt/hosting/bin/_audit.jq || { echo "_audit.jq missing — hosting-audit has no detection"; exit 1; } - test -f /opt/hosting/drift/check-chart-drift.sh || { echo "drift scripts missing — hosting-chart-drift has nothing to compare with"; exit 1; } - command -v python3 >/dev/null && python3 -c "import yaml" || { echo "python3/PyYAML missing — the drift render cannot run"; exit 1; } - echo "image carries: $(ls /opt/hosting/bin/hosting-* | wc -l) commands + chart $(grep ^version /opt/hosting/chart/Chart.yaml)" - ' - - # Stronger than running them on the runner: this is the bash, the coreutils and the PATH the - # scripts will actually execute under in a lifecycle Job. - - name: Behaviour tests, inside the image - run: | - # The manifests are NOT in the image (only bin/ and the chart are), but the suite's RBAC - # coverage case needs the ClusterRole: mount them read-only and name the file, so the - # case runs here exactly as on the runner — a missing input is exit 2 and red, not a skip. - docker run --rm -v "$PWD/deploy/aks/operator/test:/opt/hosting/test:ro" \ - -v "$PWD/deploy/aks/manifests:/opt/hosting/manifests:ro" \ - -e HOSTING_RBAC_MANIFEST=/opt/hosting/manifests/hosting-operator/operator-rbac.yaml \ - --entrypoint /bin/bash hosting-operator:ci -c 'cd /opt/hosting && bash test/run-tests.sh' - - # ═══════════════════════════════════════════════════════════════════════════════════════════ - # PUBLISH — on a push to main only. Never on a pull request (nothing from a PR reaches ACR), - # never with continue-on-error, never behind "is the secret set" (AGENTS.md → a gate never - # tests its own inputs): the preflight below asserts every external input and fails RED naming - # what to provision, and this job runs unconditionally after it. - # - # 🚨 WHY THIS JOB EXISTS. `meshweaver.azurecr.io/hosting-operator` was last built by hand on - # 2026-08-22 and NOTHING republished it: the two jobs above build the image on every push and - # then throw it away, so every fix to a hosting-* script since then — the audit, kv-rotate, the - # chart baked into the image — shipped to the repository and never to the Jobs that run them. - # The mesh names the image by tag (Hosting__Operator__Image); a tag nobody moves is a fleet - # running the operator as it was the day someone last remembered to `docker push`. - # - # Two tags per build: `` (immutable — what a deployment record pins) and `main` - # (moving — what a Provision that has not pinned yet uses). Multi-arch is deliberately NOT - # built here: the Jobs run on the cluster's amd64 node pools, and the Dockerfile's TARGETARCH - # handles a laptop build. - # ═══════════════════════════════════════════════════════════════════════════════════════════ - publish-preflight: - name: Publish preflight (every external input, or a red naming it) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Every external input, or a red build naming what to provision - env: - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - run: | - missing=() - [ -n "$AZURE_CLIENT_ID" ] || missing+=("secrets.AZURE_CLIENT_ID OIDC app registration for azure/login (the same one main-cd.yml uses)") - [ -n "$AZURE_TENANT_ID" ] || missing+=("secrets.AZURE_TENANT_ID the Systemorph AAD tenant") - [ -n "$AZURE_SUBSCRIPTION_ID" ] || missing+=("secrets.AZURE_SUBSCRIPTION_ID the subscription holding ACR meshweaver") - if [ ${#missing[@]} -gt 0 ]; then - echo "::error::the operator image cannot be published — provision the following under Settings → Secrets → Actions, then re-run:" - for m in "${missing[@]}"; do echo " • $m"; done - exit 1 - fi - echo "All external inputs are present." - - publish: - name: Publish operator image to ACR - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: [publish-preflight, scripts, image] - runs-on: ubuntu-latest - timeout-minutes: 30 - # id-token: write is required for OIDC federated login to Azure (no stored client secret) — - # the federated credential on the app registration is scoped to this repo's main branch, - # which is the only ref this job runs on. - permissions: - contents: read - id-token: write - env: - ACR: meshweaver.azurecr.io - steps: - - uses: actions/checkout@v7 - - name: Ensure Azure CLI - run: command -v az >/dev/null || curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - name: Azure login (OIDC) - uses: azure/login@v3 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - name: ACR login - # The bounded transport retry main-cd.yml uses (#2827) — the bare one-liner has died on - # `context deadline exceeded` from a single runner while sibling jobs logged in fine. - run: bash .github/scripts/acr-login.sh - - name: Build and push hosting-operator: and :main - run: | - set -euo pipefail - short="${GITHUB_SHA::7}" - docker build -f deploy/aks/operator/Dockerfile \ - -t "$ACR/hosting-operator:$short" -t "$ACR/hosting-operator:main" deploy/ - docker push "$ACR/hosting-operator:$short" - docker push "$ACR/hosting-operator:main" - - name: Verify the tags exist in the registry (never the green tick) - run: | - set -euo pipefail - short="${GITHUB_SHA::7}" - # Read the pushed manifests BACK; a push that "succeeded" against a stale login has - # produced a green step and no image before (#2827's family). - az acr manifest show-metadata --registry meshweaver --name "hosting-operator:$short" --query digest -o tsv - az acr manifest show-metadata --registry meshweaver --name "hosting-operator:main" --query digest -o tsv - echo "published $ACR/hosting-operator:$short (+ :main)" diff --git a/.github/workflows/memex-local.yml b/.github/workflows/memex-local.yml new file mode 100644 index 0000000000..2d6fcfc75e --- /dev/null +++ b/.github/workflows/memex-local.yml @@ -0,0 +1,69 @@ +name: memex-local (Linux gate) + +# memex-local — deploy/homebrew/bin/memex-local, ~2000 lines that stand up every developer's local +# portal — was gated inside hosting-operator.yml until the hosting operator moved to +# Systemorph/MeshWeaver.Plugins ("core slimming"). The CLI stays here, so its Linux gate stays here, +# unchanged: the same three steps, on every pull request, with no path filter. +# +# 🚨 Nothing in CI opened this script until MeshWeaver#2367, and what that cost, in one file: +# `memex-local help` was an unbounded FORK BOMB (a backtick in an unquoted heredoc re-entered +# main()); `helm_deploy` aborted on macOS bash 3.2 for the paths the script itself calls normal; and +# `up`/`update` reported success on a portal with no sign-in page and no view packs. Three live +# defects, zero red ticks, because nothing ever ran it. +# +# This runner is bash 5 and has no colima/kubectl/helm, so what it CAN prove is asserted by running +# the CLI against stub tools (every red state of the usability verification, plus that a green one +# is reachable) and by static assertions for the two traps a Linux runner can never reach by +# execution — the heredoc backtick and macOS bash 3.2's empty-array abort. homebrew.yml installs the +# formula and runs the same suite under bash 3.2 on macOS. +# +# 🛡️ NO SKIP-TRAPDOOR: every input is in this repository; no secret, no `continue-on-error`, no +# input-shaped `if:`, no path filter. + +on: + pull_request: + push: + branches: [main] + merge_group: + workflow_dispatch: + +permissions: + contents: read + +# One live run per ref: a newer push to the same PR branch cancels the run it supersedes; main and +# the merge queue never cancel (each main push is its own group, keyed on the commit). +concurrency: + group: memex-local-${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && github.sha || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + memex-local: + name: memex-local (parse + shellcheck + behaviour) + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + + # Asserted rather than assumed: a dependency satisfied by luck of the runner image breaks on an + # image bump as a "command not found" unrelated to the code being gated. + - name: Assert the tools this gate needs + run: | + set -euo pipefail + missing=() + command -v shellcheck >/dev/null 2>&1 || missing+=("shellcheck — apt-get install shellcheck") + command -v bash >/dev/null 2>&1 || missing+=("bash") + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::the memex-local gate cannot run — provide:" + for m in "${missing[@]}"; do echo " • $m"; done + exit 1 + fi + shellcheck --version + + - name: memex-local parses + run: bash -n deploy/homebrew/bin/memex-local + + - name: shellcheck (memex-local) + run: shellcheck -S warning deploy/homebrew/bin/memex-local deploy/homebrew/test/run-tests.sh + + - name: Behaviour tests (memex-local) + run: deploy/homebrew/test/run-tests.sh diff --git a/.github/workflows/s1-extract-image.yml b/.github/workflows/s1-extract-image.yml index 2b9c52b519..e2b739eb5f 100644 --- a/.github/workflows/s1-extract-image.yml +++ b/.github/workflows/s1-extract-image.yml @@ -17,7 +17,7 @@ name: s1-extract image # takes most of a runner's disk, and the service's own tests run INSIDE the image build # (app/test_server.py, offline). A PR that does not touch deploy/s1-extract has nothing to build. # -# Auth is the OIDC federated credential main-cd.yml and hosting-operator.yml use, scoped to this +# Auth is the OIDC federated credential main-cd.yml uses (and hosting-operator.yml used before the operator moved to MeshWeaver.Plugins), scoped to this # repository's main branch — which is the only ref the push job runs on. on: diff --git a/.gitignore b/.gitignore index 78a1fddfff..c3da088864 100644 --- a/.gitignore +++ b/.gitignore @@ -38,17 +38,6 @@ bld/ # installs it from here) — not build output. Re-include it. !/deploy/homebrew/bin/ !/deploy/homebrew/bin/** -# ...and deploy/aks/operator/bin holds the instance-lifecycle operator SCRIPTS (the image COPYs -# them at /opt/hosting/bin) — source, not build output. Re-include it. -# -# 🚨 THIS RULE'S ABSENCE IS WHY THOSE SCRIPTS DID NOT EXIST. The Dockerfile shipped with -# `COPY bin/ /opt/hosting/bin/` and an ENTRYPOINT of /opt/hosting/bin/run.sh against a directory -# git refuses to track, so `git add` on them exited 0 and did nothing — the classic silent -# failure: no error, no warning, and a file that is simply not there on the next checkout. The -# mesh's side of the seam was complete and unit-tested the whole time; the shell half was being -# written into a hole. -!/deploy/aks/operator/bin/ -!/deploy/aks/operator/bin/** # A real `.env` is deployment configuration — hosts, database names, keys. `deploy/.env.example` # is the documented surface and is tracked; the filled-in copy beside a compose file never is. # `deploy/compose/.env` and `deploy/compose-ha/.env` had been committed as Aspire emitted them diff --git a/deploy/aks/INSTANCE-LIFECYCLE.md b/deploy/aks/INSTANCE-LIFECYCLE.md index 3fa4846e94..be8ba4dcac 100644 --- a/deploy/aks/INSTANCE-LIFECYCLE.md +++ b/deploy/aks/INSTANCE-LIFECYCLE.md @@ -79,13 +79,11 @@ az role assignment create --assignee "" \ # 3. RBAC (edit the client-id annotation first — see manifests/hosting-operator/README.md). kubectl apply -f manifests/hosting-operator/ -# 4. Build and push the operator image. -# 🚨 The build context is `deploy/`, NOT the operator directory: the image carries the CHART -# (/opt/hosting/chart) as well as the scripts, so the image tag is also the chart version a -# provision deploys. Building from operator/ produces an image with no chart, which fails at -# the "Deploy release" step rather than at build time. -docker build -f deploy/aks/operator/Dockerfile -t "$ACR/hosting-operator:1" deploy/ \ - && docker push "$ACR/hosting-operator:1" +# 4. The operator image. It is PUBLISHED, not built by hand: MeshWeaver.Plugins' +# hosting-operator.yml builds it from its deploy/hosting-operator (the scripts) and this +# repository's deploy/ (the CHART, /opt/hosting/chart), and moves `hosting-operator:main`. +# The image tag is also the chart version a provision deploys — MeshWeaver.Plugins +# Hosting/HostingOperator describes how the two halves are staged into one build context. # 5. Mount the jobrunner token into the portal and set Hosting:Operator:* — README.md in # manifests/hosting-operator/ has the exact values. diff --git a/deploy/aks/manifests/hosting-operator/README.md b/deploy/aks/manifests/hosting-operator/README.md index d500fcee29..4f5b610ea7 100644 --- a/deploy/aks/manifests/hosting-operator/README.md +++ b/deploy/aks/manifests/hosting-operator/README.md @@ -84,8 +84,8 @@ Actions and blank-maintainer fixtures, and the misspelled-executor refusal. Unlike the `kubectl apply` list above (the original bootstrap), this file carries no placeholder and is applied exactly as committed, by the `ApplyClusterManifest` InstanceAction on the control instance: -Systemorph/Memex `aks-ops.yml` runs `deploy/aks/operator/bin/hosting-manifest-apply` from a checkout of -this repository. Its dry run (`check`) is a read: it prints the file's sha256 and the API server's own +Systemorph/Memex `aks-ops.yml` runs `hosting-manifest-apply` (MeshWeaver.Plugins `deploy/hosting-operator/bin`) against +this repository's `deploy/aks/manifests`. Its dry run (`check`) is a read: it prints the file's sha256 and the API server's own server-side dry-run answer, one `created|configured|unchanged` line per object, no value. The apply runs only on the lane's mutate leg, with a mesh approval that binds that sha256, and refuses other bytes. The script refuses a kind outside Namespace, ServiceAccount, an empty service-account-token Secret, RBAC and @@ -93,11 +93,14 @@ ValidatingAdmissionPolicy objects, a wildcard rule, the escalate/bind/impersonat cluster-admin/admin/edit/system:* or to anything but a ServiceAccount, and a leftover ``. That is a backstop under the approval, not a substitute for reading the file. -## The image is published on every push to `main` +## The image is published by MeshWeaver.Plugins -`meshweaver.azurecr.io/hosting-operator:` (immutable — what a deployment record pins) -and `:main` (moving) are built and pushed by `.github/workflows/hosting-operator.yml`'s `publish` -job, on `push` to `main` only, behind a preflight that fails red naming a missing OIDC secret. +`meshweaver.azurecr.io/hosting-operator:main` (moving) and an immutable `:-` +are built and pushed by MeshWeaver.Plugins' `.github/workflows/hosting-operator.yml`, which carries the +scripts (`deploy/hosting-operator`) and stages this repository's `deploy/` (the chart) into the build +context — on its own `main`, and hourly when this repository's chart or drift scripts moved +(MeshWeaver.Plugins `Hosting/HostingOperator`). The `:` tags this repository's +`hosting-operator.yml` published until the operator moved stay in the registry. Until MeshWeaver#3353 the image was built by hand (last on 2026-08-22) and nothing republished it, so every script fix shipped to the repository and never to a Job. @@ -138,7 +141,7 @@ that needs a new grant lands with the grant, and the next deploy carries both to no laptop in the loop. Measured before this existed (2026-09-09): the `storageclasses` rule had been on main since the volume-capacity step, the cluster's ClusterRole predated it, and the first record-driven Reconcile through the fixed operator stopped at step 1/6 with `Forbidden`. -`deploy/aks/operator/test/check-rbac-coverage.sh` is the other half: every `kubectl +`MeshWeaver.Plugins/deploy/hosting-operator/test/check-rbac-coverage.sh` is the other half: every `kubectl ` a `bin/` script names must be granted here, or the operator test suite is red. `kubectl apply` prints `configured` / `unchanged` per resource in the lane's log — that line is the change report. diff --git a/deploy/aks/operator/Dockerfile b/deploy/aks/operator/Dockerfile deleted file mode 100644 index 88e961a635..0000000000 --- a/deploy/aks/operator/Dockerfile +++ /dev/null @@ -1,79 +0,0 @@ -# The hosting operator image: what an instance lifecycle run executes. -# -# BUILD CONTEXT IS `deploy/`, not this directory: -# docker build -f deploy/aks/operator/Dockerfile -t $ACR/hosting-operator: deploy/ -# because the image carries the CHART as well as the scripts (see the COPY below). -# -# It carries az + kubectl + helm + the PostgreSQL client, because a lifecycle run legitimately -# needs all four and the portal image carries none of them. That is the honest reason the run -# happens in a Job rather than in the portal process — and the useful one: the Job is a separate, -# short-lived identity whose credential nothing long-running holds. -FROM mcr.microsoft.com/azure-cli:2.77.0 - -# 🚨 THE BASE IS AZURE LINUX 3, NOT ALPINE. This file previously ran `apk add` and busybox -# `adduser -D`; neither exists here, so the image had never actually been built — every `docker -# build` of it exits 127. That is worth stating rather than quietly fixing, because an operator -# image that cannot be built is indistinguishable, from the mesh's side, from one that was never -# configured: HostingOperator refuses with "no operator image configured" either way. -ARG KUBECTL_VERSION=v1.31.4 -ARG HELM_VERSION=v3.16.3 -# crane (go-containerregistry): hosting-image-mirror's registry-to-registry copy of the pinned -# platform images from ACR into the fleet registry (MeshWeaver.Plugins#1722). A manifest+blob copy -# needs a registry client; az, kubectl and helm are not one, and docker is not in this image. -ARG CRANE_VERSION=v0.20.3 -# Supplied by BuildKit. The binaries below are per-architecture, and hard-coding amd64 produces an -# image that builds fine on an arm64 laptop and then cannot exec kubectl. -ARG TARGETARCH=amd64 - -# postgresql gives pg_dump/pg_restore/psql. 🚨 The client MAJOR must be >= the server's, or pg_dump -# refuses with "server version mismatch". Azure Linux 3 ships 16.x and the fleet's flexible servers -# are major 16 (memexaks-pg, verified 2026-08-22) — bump both together when the fleet moves. -# tar: needed to unpack helm, and not in the base. shadow-utils: useradd, for the non-root user. -# jq: hosting-audit's detection is a jq program (bin/_audit.jq) — without it the audit cannot -# compare a single field, and it refuses rather than reporting a namespace it never read as clean. -# python3 + PyYAML: the values-half key check (bin/_values_half.py, policy -# `one-values-half-per-release`) reads a captured half's KEY NAMES and filters it to the record's -# declared keys without touching a kept value — YAML block scalars (a PEM key) are not something to -# rewrite in bash. The az CLI's own PyYAML lives in its private site-packages; this is the system one. -RUN tdnf install -y --refresh postgresql tar shadow-utils ca-certificates jq python3 PyYAML \ - && tdnf clean all \ - && curl -fsSLo /usr/local/bin/kubectl \ - "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" \ - && chmod +x /usr/local/bin/kubectl \ - && curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" \ - | tar -xz -C /tmp "linux-${TARGETARCH}/helm" \ - && mv "/tmp/linux-${TARGETARCH}/helm" /usr/local/bin/helm \ - && rm -rf "/tmp/linux-${TARGETARCH}" \ - && case "${TARGETARCH}" in amd64) crane_arch=x86_64 ;; arm64) crane_arch=arm64 ;; *) echo "no crane build for ${TARGETARCH}"; exit 1 ;; esac \ - && curl -fsSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \ - | tar -xz -C /usr/local/bin crane \ - && chmod +x /usr/local/bin/crane - -COPY aks/operator/bin/ /opt/hosting/bin/ -RUN chmod +x /opt/hosting/bin/* && ln -s /opt/hosting/bin/hosting-* /usr/local/bin/ - -# The Chart Drift comparison (hosting-chart-drift, the second step of an Audit — Systemorph/Memex#647): -# core's drift scripts, at the same commit as the chart they render. They stay in deploy/aks/scripts, -# where chart-gate.yml tests them on every pull request; this is where the command finds them in the -# image (the aks-ops lane finds them in its MeshWeaver checkout instead, bin/../../scripts). -COPY aks/scripts/check-chart-drift.sh aks/scripts/chart-drift-compare.py \ - aks/scripts/chart-drift-record.py aks/scripts/chart-drift-render.py /opt/hosting/drift/ - -# 🚨 THE CHART IS BAKED IN, and the build context is `deploy/` for exactly this reason. -# -# hosting-deploy runs `helm upgrade` against /opt/hosting/chart, so THE OPERATOR IMAGE VERSION IS -# THE CHART VERSION IT DEPLOYS. That pinning is the point: the scripts pass `--set` flags that only -# make sense against the templates shipped beside them, and a chart fetched at run time could drift -# from them silently — rendering a release nobody reviewed, which is the failure the whole -# repo-driven deploy exists to prevent. -COPY helm/ /opt/hosting/chart/ -# The per-instance DATABASE chart (deploy/helm-db, Doc/Architecture/InClusterDatabases), beside the -# portal chart for the same reason: hosting-db-release installs the templates this image was built with. -COPY helm-db/ /opt/hosting/chart-db/ - -# Non-root. Nothing here needs to be root, and a container that can delete namespaces should not -# also be able to do whatever a root process on a node could. -RUN useradd --uid 1654 --create-home --shell /sbin/nologin operator -USER 1654 - -ENTRYPOINT ["/bin/bash", "/opt/hosting/bin/run.sh"] diff --git a/deploy/aks/operator/bin/_audit.jq b/deploy/aks/operator/bin/_audit.jq deleted file mode 100644 index ef679da649..0000000000 --- a/deploy/aks/operator/bin/_audit.jq +++ /dev/null @@ -1,208 +0,0 @@ -# _audit.jq — the detection behind hosting-audit. Pure: manifest objects + live objects in, one -# LiveOnlyConfigReport out. Sourced by hosting-audit, never run on its own. -# -# Inputs (jq -n … -f _audit.jq): -# $manifest[0] the helm manifest + hooks as an ARRAY of Kubernetes objects (the truth) -# $live[0] every live object in the namespace, every kind, as ONE array -# $namespace $release $deployment $container $revision $chart $auditedAt $skipped -# $persistence the release's user-supplied `persistence` values — the Deployment record's -# volumes[] as HelmValues rendered them (claimName + size per volume). The one -# RECORD-vs-live comparison here: a claim's live capacity below what the record -# declares is drift the manifest cannot show, because on this fleet the portal's -# claims are not helm-managed and a bigger `size` changes nothing until -# hosting-pv-resize applies it. -# -# 🚨 NAMES, NEVER VALUES. A ConfigMap value is compared here and never emitted; an env entry's -# `value` decides whether it is a plain value and is never emitted. The report is written to a node -# and shown on a page — a value leaking through it would be the secret-in-the-pod-spec finding all -# over again, one level up. - -def oname: "\(.kind)/\(.metadata.name)"; -def owned: ((.metadata.ownerReferences // []) | length) > 0; -def csi_synced: (((.metadata.labels // {})["secrets-store.csi.k8s.io/managed"]) // "") == "true"; - -# What is NOT configuration anyone put on the cluster, and so is not a finding: -# owned objects (ReplicaSets, CronJob-spawned Jobs, CSI-synced Secrets) exist because of their -# owner; helm's own release records and ServiceAccount tokens are the machinery; kube-root-ca.crt -# and the `default` ServiceAccount are created by Kubernetes in every namespace. -def excluded: - owned - or (.kind == "Secret" and ((.type // "") | (. == "helm.sh/release.v1" or . == "kubernetes.io/service-account-token"))) - or csi_synced - or (.kind == "ConfigMap" and .metadata.name == "kube-root-ca.crt") - or (.kind == "ServiceAccount" and .metadata.name == "default"); - -# A name that looks like it carries a secret. Name-based on purpose: the VALUE is what must not -# be read, so the name is the only honest signal. A flag that merely ends in Key is a false -# positive the record can rename; a token that slipped through would be an outage. -def secretish: test("token|secret|key|password"; "i"); - -# a minus b, as sorted unique names -def only(a; b): [a[] | select(. as $x | (b | index($x)) == null)] | unique; - -# Kubernetes serialises quantities and ports as strings or numbers depending on who wrote them -# (`cpu: 4` in a template, `"4"` back from the API). Compare shapes, not spellings. -def norm: walk(if type == "number" then tostring else . end); -def compact: tojson | if length > 120 then .[:117] + "…" else . end; - -# A Kubernetes quantity as a number of bytes, or null when it is not one. Both unit families -# occur: the record writes binary units (128Gi), and a provisioner may answer in either. -def quantity_bytes: - (tostring) as $q - | if ($q | test("^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$")) | not then null - else ($q | capture("^(?[0-9]+(\\.[0-9]+)?)(?[A-Za-z]*)$")) as $m - | ($m.n | tonumber) * ({"": 1, "k": 1000, "M": 1000000, "G": 1000000000, "T": 1000000000000, - "P": 1000000000000000, "E": 1000000000000000000, - "Ki": 1024, "Mi": 1048576, "Gi": 1073741824, "Ti": 1099511627776, - "Pi": 1125899906842624, "Ei": 1152921504606846976}[$m.u]) - end; - -# 🚨 lifecycle values are REDACTED to their shape. Every other compared pod-spec field carries -# placement and sizing (replicas, selectors, quantities, probe numbers) — but a lifecycle hook -# holds COMMAND LINES, and a command line can embed anything, tokens included. The comparison uses -# the full value; the report shows the structure with every string scalar replaced, which is enough -# to see THAT a hook was patched and WHERE without quoting what it runs. -def redact($f): if ($f | endswith(".lifecycle")) then walk(if type == "string" then "…" else . end) else . end; - -def container($c): ((.spec.template.spec.containers // []) | map(select(.name == $c)) | .[0]); -def env_names: (.env // []) | map(.name); -def env_from: (.envFrom // []) | map(.secretRef.name // .configMapRef.name // empty); -def mounts: (.volumeMounts // []) | map(.mountPath); -def volumes: (.spec.template.spec.volumes // []) | map(.name); -def containers: ((.spec.template.spec.containers // []) | map(.name)) + ((.spec.template.spec.initContainers // []) | map(.name)); -def annotations: (.spec.template.metadata.annotations // {}) | keys; - -($manifest[0]) as $m -| ($live[0]) as $l -| ($m | map(oname)) as $managed -| ($m | map(select(.kind == "Deployment" and .metadata.name == $deployment)) | .[0]) as $md -| ($l | map(select(.kind == "Deployment" and .metadata.name == $deployment)) | .[0]) as $ld -| if $md == null then error("the manifest of release " + $release + " renders no Deployment/" + $deployment + " — pass --deployment naming the portal Deployment the chart renders") else . end -| if $ld == null then error("no live Deployment/" + $deployment + " in " + $namespace + " — the release is deployed but its portal is not running, which is a finding this audit cannot express as drift") else . end -| ($md | container($container)) as $mc -| ($ld | container($container)) as $lc -| if $mc == null then error("the manifest's Deployment/" + $deployment + " has no container named " + $container) else . end -| if $lc == null then error("the live Deployment/" + $deployment + " has no container named " + $container + " — a container renamed live is a finding this audit cannot express as drift") else . end - -# ── the portal Deployment, field by field ────────────────────────────────────────────────────── -| only($lc | env_names; $mc | env_names) as $envLiveOnly -| only($mc | env_names; $lc | env_names) as $envManifestOnly -| only($lc | env_from; $mc | env_from) as $envFromLiveOnly -| only($ld | volumes; $md | volumes) as $volumesLiveOnly -| only($lc | mounts; $mc | mounts) as $mountsLiveOnly -| only($ld | containers; $md | containers) as $containersLiveOnly -| only($ld | annotations; $md | annotations) as $podAnnotationsLiveOnly - -# replicas belong to the autoscaler when the manifest renders one for this Deployment — the chart -# then deliberately writes no replicas field, and comparing would blame KEDA for doing its job. -| ($m | map(select((.kind == "ScaledObject" or .kind == "HorizontalPodAutoscaler") - and (.spec.scaleTargetRef.name // "") == $deployment)) | length > 0) as $autoscaled -| ([ - (if $autoscaled then empty else - {field: ".spec.replicas", live: ($ld.spec.replicas // 1), manifest: ($md.spec.replicas // 1)} end), - {field: ".spec.template.spec.nodeSelector", live: ($ld.spec.template.spec.nodeSelector // {}), manifest: ($md.spec.template.spec.nodeSelector // {})}, - {field: ".spec.template.spec.tolerations", live: ($ld.spec.template.spec.tolerations // []), manifest: ($md.spec.template.spec.tolerations // [])}, - {field: ".spec.template.spec.affinity", live: ($ld.spec.template.spec.affinity // {}), manifest: ($md.spec.template.spec.affinity // {})}, - {field: ".spec.template.spec.serviceAccountName", live: ($ld.spec.template.spec.serviceAccountName // "default"), manifest: ($md.spec.template.spec.serviceAccountName // "default")}, - {field: ".spec.template.spec.terminationGracePeriodSeconds", live: ($ld.spec.template.spec.terminationGracePeriodSeconds // 30), manifest: ($md.spec.template.spec.terminationGracePeriodSeconds // 30)}, - {field: "containers[\($container)].resources", live: ($lc.resources // {}), manifest: ($mc.resources // {})}, - {field: "containers[\($container)].lifecycle", live: ($lc.lifecycle // {}), manifest: ($mc.lifecycle // {})}, - {field: "containers[\($container)].startupProbe.periodSeconds", live: ($lc.startupProbe.periodSeconds // 10), manifest: ($mc.startupProbe.periodSeconds // 10)}, - {field: "containers[\($container)].startupProbe.failureThreshold", live: ($lc.startupProbe.failureThreshold // 3), manifest: ($mc.startupProbe.failureThreshold // 3)} - ] - | map(select((.live | norm) != (.manifest | norm))) - | map(.field as $f - | {field: $f, live: (.live | redact($f) | compact), manifest: (.manifest | redact($f) | compact)})) as $podSpecDiffs - -# ── the chart's ConfigMaps ───────────────────────────────────────────────────────────────────── -| ($m | map(select(.kind == "ConfigMap")) | map( - .metadata.name as $n - | (.data // {}) as $mdata - | ($l | map(select(.kind == "ConfigMap" and .metadata.name == $n)) | .[0]) as $lcm - | if $lcm == null then - {name: $n, missingLive: true, liveOnlyKeys: [], manifestOnlyKeys: ($mdata | keys), differingKeys: []} - else - (($lcm.data // {}) as $ldata - | {name: $n, missingLive: false, - liveOnlyKeys: only($ldata | keys; $mdata | keys), - manifestOnlyKeys: only($mdata | keys; $ldata | keys), - differingKeys: ([($ldata | keys[]) | select(($mdata[.] != null) and ($ldata[.] != $mdata[.]))] | sort)}) - end)) as $configMaps -| ($configMaps | map((.liveOnlyKeys | length) + (.manifestOnlyKeys | length) + (.differingKeys | length) - + (if .missingLive then 1 else 0 end)) | add // 0) as $configMapCount - -# ── objects the chart does not own ───────────────────────────────────────────────────────────── -| ($l | map(select(excluded | not)) - | map(select(oname as $n | ($managed | index($n)) == null)) - | group_by(.kind) - | map({kind: .[0].kind, names: (map(.metadata.name) | sort | unique)})) as $unmanagedObjects -| ($unmanagedObjects | map(.names | length) | add // 0) as $unmanagedCount - -# ── secrets the live pod reads that nobody renders ───────────────────────────────────────────── -| ([ ($ld.spec.template.spec.containers // [])[] - | ((.envFrom // [])[] | .secretRef.name // empty), ((.env // [])[] | .valueFrom.secretKeyRef.name // empty) ] - + [ ($ld.spec.template.spec.volumes // [])[] | .secret.secretName // empty ] - | unique - | map(select(. as $s | ($managed | index("Secret/" + $s)) == null)) - | map(select(. as $s | ($l | map(select(.kind == "Secret" and .metadata.name == $s and csi_synced)) | length) == 0))) as $unmanagedSecrets - -# ── secret-shaped names carried as plain values ──────────────────────────────────────────────── -| ([ ($ld.spec.template.spec.containers // [])[] | .name as $c - | (.env // [])[] | select(.value != null) | select(.name | secretish) | "env:\($c):\(.name)" ] - + [ $l[] | select(.kind == "ConfigMap") | select(oname as $n | ($managed | index($n)) != null) - | .metadata.name as $n | ((.data // {}) | keys[]) | select(secretish) | "configmap:\($n)/\(.)" ] - | unique) as $plainSecretEntries - -# ── claims below the capacity the record declares ────────────────────────────────────────────── -# The record's volumes[] arrive as the release's `persistence` values. Every entry with a claimName -# AND a size is compared against the live claim's status.capacity (what the volume IS, not what -# was asked for). Below → a finding; absent → a finding (the pod could not have mounted it); -# unreadable on either side → a finding rather than a silent pass, because "could not compare" -# must never read as "matches". At or above → nothing: a claim larger than its record is the -# record's problem to state, and hosting-pv-resize refuses to shrink it. -| ([ ($persistence // {}) | to_entries[] - | .key as $volume | .value as $spec - | select(($spec | type) == "object" and (($spec.claimName // "") != "") and (($spec.size // "") != "")) - | ($l | map(select(.kind == "PersistentVolumeClaim" and .metadata.name == $spec.claimName)) | .[0]) as $pvc - | ($spec.size | quantity_bytes) as $declared - | (if $pvc == null then null else ($pvc.status.capacity.storage // $pvc.spec.resources.requests.storage // null) end) as $liveQ - | (if $liveQ == null then null else ($liveQ | quantity_bytes) end) as $live - | select($pvc == null or $declared == null or $live == null or $live < $declared) - | {volume: $volume, claimName: $spec.claimName, declared: ($spec.size | tostring), - live: (if $pvc == null then "absent" else ($liveQ // "unknown" | tostring) end)} ] - | sort_by(.volume)) as $volumeCapacityBelowRecord - -# ── the verdict — derived from the lists, and re-derived by the mesh ─────────────────────────── -| (($envLiveOnly | length) + ($envManifestOnly | length) + ($envFromLiveOnly | length) - + ($volumesLiveOnly | length) + ($mountsLiveOnly | length) + ($containersLiveOnly | length) - + ($podAnnotationsLiveOnly | length) + ($podSpecDiffs | length) + $configMapCount - + $unmanagedCount + ($unmanagedSecrets | length) + ($plainSecretEntries | length) - + ($volumeCapacityBelowRecord | length)) as $findingCount -| { - namespace: $namespace, - release: $release, - deploymentName: $deployment, - containerName: $container, - helmRevision: $revision, - chartVersion: (if $chart == "" then null else $chart end), - auditedAt: $auditedAt, - managedObjectCount: ($managed | length), - runningImage: ($lc.image // null), - manifestImage: ($mc.image // null), - skippedKinds: ($skipped | split(" ") | map(select(length > 0))), - envLiveOnly: $envLiveOnly, - envManifestOnly: $envManifestOnly, - envFromLiveOnly: $envFromLiveOnly, - volumesLiveOnly: $volumesLiveOnly, - mountsLiveOnly: $mountsLiveOnly, - containersLiveOnly: $containersLiveOnly, - podAnnotationsLiveOnly: $podAnnotationsLiveOnly, - podSpecDiffs: $podSpecDiffs, - configMaps: $configMaps, - unmanagedObjects: $unmanagedObjects, - unmanagedSecrets: $unmanagedSecrets, - plainSecretEntries: $plainSecretEntries, - volumeCapacityBelowRecord: $volumeCapacityBelowRecord, - findingCount: $findingCount, - verdict: (if $findingCount == 0 then "clean" else "drift" end) - } diff --git a/deploy/aks/operator/bin/_common.sh b/deploy/aks/operator/bin/_common.sh deleted file mode 100644 index 597bbdd8f9..0000000000 --- a/deploy/aks/operator/bin/_common.sh +++ /dev/null @@ -1,477 +0,0 @@ -# shellcheck shell=bash -# Shared harness for the hosting-* operator commands. Sourced, never executed. -# -# 🚨 THE ONE RULE THIS FILE EXISTS TO ENFORCE: a step that could not do its job must EXIT NON-ZERO, -# and a step that verifies something must say so only after reading the thing back. run.sh stops at -# the first failure, and the mesh reads `::hosting::` lines out of the pod log — so a script that -# swallows an error does not produce a warning, it produces a green teardown of a live instance. - -set -uo pipefail - -# The prefix the mesh's OperatorOutput parser reads. Anything else on stdout is human output. -HOSTING_MARKER="::hosting::" - -# A machine-readable fact for the mesh: hosting::say size 12345 -> "::hosting:: size=12345" -hosting::say() { printf '%s %s=%s\n' "$HOSTING_MARKER" "$1" "$2"; } - -# Announce the step being entered. run.sh emits this; a script may emit sub-steps. -hosting::step() { printf '%s step=%s\n' "$HOSTING_MARKER" "$1"; } - -# Human narration. Goes to stdout so it lands in the pod log the mesh streams back. -hosting::log() { printf ' %s\n' "$*"; } - -# Fail loudly, naming the command. Never `exit 0` on a problem. -hosting::die() { - printf '%s: ERROR: %s\n' "${HOSTING_CMD:-hosting}" "$*" >&2 - HOSTING_DIED=1 - exit 1 -} - -# ── a script that stops WITHOUT saying why still reports a cause ───────────────────────────────── -# hosting::die is how a script refuses: `: ERROR: ` on stderr, the line the control -# plane quotes as the cause of a failed action (it reads the FIRST `: ERROR:` line of the -# run's log tail, and run.sh's `step i/n '' failed` line for the step). A script that BASH -# stops — an unbound variable under `set -u`, a bad substitution — never reaches hosting::die: bash -# prints `: line N: X: unbound variable`, which matches neither shape, so the action lands as -# a bare "concluded failure". Measured on Memex run 38038325130 (a Roll of memex-cloud): step 3/3 -# died on `_common.sh: line 269: HOSTING_AZ_ERR: unbound variable` and the control log quoted nothing. -# -# So every script that sources this file reports its own unexplained exit in the refusal's shape: -# WHERE it stopped (file:line — tracked by the DEBUG trap below, because bash has already unwound -# its call stack when an EXIT trap runs) and the command it was running, as SOURCE TEXT -# (BASH_COMMAND is the command before expansion, so no value a variable holds is ever printed). -# -# Silent on purpose for: exit 0; an exit through hosting::die (it has spoken); and a deliberate -# `exit N` / `return N` (the script chose it, and said what it had to say). -# -# 🚨 A script that sets its OWN EXIT trap replaces this one, so its trap must call hosting::on_exit -# FIRST — `trap 'hosting::on_exit; rm -f "$tmp"' EXIT`. First, because the function reads `$?` and -# the line bash was on, and both are only still true for the first command of a trap. -# test/run-tests.sh refuses an EXIT trap under bin/ that does not. -HOSTING_DIED="" -HOSTING_AT="" -HOSTING_AT_PREV="" -HOSTING_RUN_STEP="" -# run.sh says which step it is in — `i/n ''`, or nothing between steps. -hosting::in_step() { HOSTING_RUN_STEP="${1:-}"; } -hosting::on_exit() { - local rc=$? cmd="${BASH_COMMAND:-}" at="${HOSTING_AT_PREV:-}" who="${HOSTING_CMD:-${0##*/}}" - [ "$rc" -ne 0 ] || return 0 - [ -z "${HOSTING_DIED:-}" ] || return 0 - case "$cmd" in exit|exit\ *|return|return\ *) return 0 ;; esac - HOSTING_DIED=1 - at="${at##*/}" - cmd="${cmd//$'\n'/ }" - [ "${#cmd}" -le 300 ] || cmd="${cmd:0:300}…" - printf '%s: ERROR: stopped with exit %s at %s without stating a cause, while running: %s — a shell error in the operator script itself (the message bash printed is the line above, unless it died under a redirect), not a refusal and not a property of the plan. Nothing after that line ran.\n' \ - "$who" "$rc" "${at:-an unknown line}" "$cmd" >&2 - # run.sh names the step it was in, in the shape of its own step-failure line. - [ -z "${HOSTING_RUN_STEP:-}" ] \ - || printf 'run.sh: ERROR: step %s failed: the operator stopped on a shell error of its own before or while running it — the cause is the line above.\n' "$HOSTING_RUN_STEP" >&2 - return 0 -} -# The DEBUG trap fires before every simple command — and once more for the EXIT trap's own call of -# hosting::on_exit, which is why the line is read from HOSTING_AT_PREV there. functrace carries it -# into functions, where the lines worth naming are. `:-$0` because BASH_SOURCE is EMPTY under -# `bash -c`, and an unbound expansion in this trap would be the very defect it exists to report. -set -o functrace -trap '[ "${FUNCNAME[0]:-}" = hosting::on_exit ] || { HOSTING_AT_PREV="$HOSTING_AT"; HOSTING_AT="${BASH_SOURCE[0]:-$0}:${LINENO}"; }' DEBUG -trap 'hosting::on_exit' EXIT - -# Require a non-empty environment variable, naming what to set when it is missing. -hosting::need_env() { - local name="$1" why="${2:-}" - local value="${!name:-}" - [ -n "$value" ] || hosting::die "environment variable ${name} is empty or unset${why:+ — ${why}}. It is supplied by Hosting:Operator:Environment on the control instance." -} - -# Require a non-empty flag value. -hosting::need_flag() { - local name="$1" value="${2:-}" - [ -n "$value" ] || hosting::die "missing required flag --${name}" -} - -# Refuse anything that is not a plain identifier. Every name this operator receives ends up -# interpolated into an az/kubectl/helm command line, so the validation is a security boundary and -# not tidiness: the mesh composes the plan, but the mesh is driven by a node an admin can edit. -# -# 🚨 THESE VALIDATE IN PLACE AND PRINT NOTHING — deliberately. The obvious shape, -# name="$(hosting::safe_name name "$name")" -# is BROKEN and silently so: a command substitution runs in a SUBSHELL, so the `exit 1` inside -# hosting::die ends the substitution and not the script. The caller carries on with an EMPTY value -# and every guard here becomes a no-op. Caught by test/run-tests.sh, which is why the injection -# cases in it are worth their length. -hosting::safe_name() { - local what="$1" value="${2:-}" - [[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \ - || hosting::die "${what} '${value}' is not a plain name (letters, digits, dot, dash, underscore) — refusing to interpolate it into a command" -} - -# A hostname, for DNS/TLS/probe steps. Validates in place; see the note above. -hosting::safe_host() { - local what="$1" value="${2:-}" - [[ "$value" =~ ^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$ ]] \ - || hosting::die "${what} '${value}' is not a hostname — refusing" -} - -# Is this a dry run? The mesh sets HOSTING_DRY_RUN=true for a rehearsal. -hosting::dry() { [ "${HOSTING_DRY_RUN:-false}" = "true" ]; } - -# Run a command, or narrate it when rehearsing. Use for every MUTATION. -hosting::do() { - # 🚨 The narration goes to STDERR, never stdout. `hosting::do` wraps MUTATIONS, and a mutation can - # sit in a pipe whose consumer reads the command's output as data — hosting-deploy's - # hosting::do kubectl create namespace … --dry-run=client -o yaml | kubectl apply -f - - # is one. Measured 2026-09-08 on memex (Deployments/memex-reconcile-20260908-ci8118): the - # ` + kubectl create namespace …` line went down the pipe as line 1 of the manifest, `apiVersion: - # v1` became line 2, and kubectl answered "yaml: line 2: mapping values are not allowed in this - # context" — the Reconcile stopped at step 1/3 before helm ran, and every Provision would have - # stopped at the same line. The job log (what the mesh's OperatorOutput parser reads) carries both - # streams, so nothing an operator reads changes; only the data channel is clean. - if hosting::dry; then - printf ' DRY-RUN would run: %s\n' "$*" >&2 - return 0 - fi - printf ' + %s\n' "$*" >&2 - "$@" -} - -# Capture a command's stdout (queries, never mutations — a dry run still needs to read). -hosting::read() { "$@"; } - -# ── DENIED, ABSENT and PRESENT are THREE answers, never two (MeshWeaver#4722) ─────────────────── -# -# Run a READ and classify what came back: -# 0 PRESENT — it succeeded (and, with `output`, printed something) -# 1 ABSENT — it failed for a reason that is NOT a refusal, or succeeded printing nothing -# 2 REFUSED — this identity was not permitted to look, so NOTHING is known either way -# Stdout lands in HOSTING_PROBE_OUT, the first line of stderr in HOSTING_PROBE_ERR. -# -# 🚨 WHY THIS IS A SHARED PRIMITIVE. The obvious shape is -# value="$(kubectl get thing 2>/dev/null)"; [ -n "$value" ] || hosting::die "there is no thing" -# and `2>/dev/null` throws away the one fact that decides which sentence is true. A Forbidden — -# this operator's ClusterRole lacking the grant — comes out as the thing being ABSENT: the operator -# announcing that a platform layer, an Ingress or a ConfigMap does not exist when it was merely not -# permitted to LOOK, and sending the reader off to re-create something that is already there. -# -# It is the denied-vs-deleted confusion that closed MeshWeaver#1391 on a `Not found` and had the -# identical defect re-filed unchanged as #3883 four weeks later; here it is in an operator's own -# diagnostics, the one output a reader is supposed to trust. -# -# This lived as a local helper inside hosting-db-release when #4436 fixed the three probes #4722 -# measured — which is exactly why the fix did not sweep: two more reads in that same file and two -# in other commands went on collapsing the two answers. A discrimination that only one script can -# reach is a discrimination the next script will not make. -# -# 🚨 Callers MUST branch on all three. `hosting::probe … || hosting::die "…absent…"` is the defect -# wearing the fix's clothes: it turns REFUSED back into ABSENT. Branch 2 first, and say that -# nothing was ruled out. test/check-stderr-discarded.sh is the static half. -HOSTING_PROBE_OUT="" HOSTING_PROBE_ERR="" -hosting::probe() { - local need="$1"; shift - local errfile rc - errfile="$(mktemp)" - HOSTING_PROBE_OUT="$("$@" 2>"$errfile")"; rc=$? - HOSTING_PROBE_ERR="$(head -1 "$errfile")"; rm -f "$errfile" - if [ "$rc" -ne 0 ]; then - # kubectl and Key Vault say Forbidden; ARM (`az identity show`, `az acr show`, …) says - # `(AuthorizationFailed) The client … does not have authorization …` with NO "Forbidden" in - # it — matching only the first would read every ARM refusal as an absence. Same set as - # hosting-aks-upgrade's classifier. - case "$HOSTING_PROBE_ERR" in - *Forbidden*|*forbidden*|*AuthorizationFailed*|*"does not have authorization"*) return 2 ;; - *) return 1 ;; - esac - fi - # `kubectl get nodes -l workload=db` exits 0 and prints NOTHING when the selector matches no - # node. That is an ABSENCE, not a failure, and only the caller knows which reads apply. - if [ "$need" = output ] && [ -z "$HOSTING_PROBE_OUT" ]; then return 1; fi - return 0 -} - -# The one sentence a REFUSED probe is allowed to produce. It says what is NOT known, names the file -# the grant lives in and the lane that carries it — never that anything is absent, because a probe -# that could not run ruled nothing out. -hosting::die_refused() { - hosting::die "REFUSED, not absent: this operator's ClusterRole does not permit reading $* — so whether it exists is UNKNOWN, and nothing was ruled out. Grant it in MeshWeaver deploy/aks/manifests/hosting-operator/operator-rbac.yaml; it reaches the cluster through Systemorph/Memex's helm-release lane, never from this Job. Refused: ${HOSTING_PROBE_ERR}" -} - -# ── the Postgres admin password is resolved by NAME, never carried as a value (Memex#132) ────── -# -# hosting::pg_password -# -# Sets and exports PGPASSWORD for the pg_dump / psql / pg_restore that follow, read from Key Vault -# object in vault with the identity this operator already holds — the SAME -# `az keyvault secret show --query value` hosting-kv-ensure makes to compose an instance's -# connection string, so no new grant is involved and the read is one this identity has already -# been trusted with. The value lands in this process only: never printed, never on an argv (az -# writes it to STDOUT, which is captured), never reported. -# -# 🚨 WHY A NAME. The only environment a plan step has is `Hosting:Operator:Environment` (the Job) or -# the bundle's `environment` map (the aks-ops lane) — both rendered from the control record into a -# ConfigMap, or a workflow_dispatch input: PLAINTEXT by construction. Every database action -# therefore failed at step 1 (`PGPASSWORD is empty or unset … supplied by -# Hosting:Operator:Environment`) rather than put the server's admin password there — and that -# sentence named a channel that cannot carry a secret. The record already carries the object's NAME -# (`operator.environment.AZ_POSTGRES_PASSWORD_SECRET`) and the vault's name (`keyVault`); the plan -# hands both to the script as `--vault` / `--password-secret`, exactly as it does to -# hosting-kv-ensure, and this resolves them on either executor. -# -# Both flags, or neither: with neither, a PGPASSWORD already in the environment is honoured (the -# by-hand shape — an operator at a shell with the password exported), and its absence is refused -# naming the flags, not the ConfigMap. The two names are validated here as plain identifiers -# because they are interpolated into an az command line — the same boundary hosting::safe_name -# guards everywhere else. ABSENT and REFUSED are two different sentences (MeshWeaver#4722): a -# vault that refused this identity has ruled nothing out about the object. An EMPTY value is refused -# as loudly as an unreadable one: psql with an empty PGPASSWORD prompts, and a Job that prompts -# hangs to its deadline saying nothing that names this step. A dry run must not call this — it reads -# a secret, and a rehearsal reads none (hosting-kv-ensure's rule). -hosting::pg_password() { - local vault="${1:-}" object="${2:-}" rc - if [ -z "$vault" ] && [ -z "$object" ]; then - [ -n "${PGPASSWORD:-}" ] && return 0 - hosting::die "no Postgres admin password: pass --vault --password-secret — the plan supplies both from the control record (keyVault, and operator.environment.AZ_POSTGRES_PASSWORD_SECRET, e.g. memex-postgres-password), and this script reads the value from the vault as the identity it already runs as. The operator's environment is a ConfigMap and cannot carry the value (Memex#132); PGPASSWORD in the environment is honoured only when set by hand." - fi - [ -n "$vault" ] && [ -n "$object" ] \ - || hosting::die "--vault and --password-secret go together: the vault to read from and the object holding the server's admin password (got --vault '${vault}' --password-secret '${object}')" - hosting::safe_name vault "$vault" - hosting::safe_name password-secret "$object" - hosting::probe output az keyvault secret show --vault-name "$vault" --name "$object" --query value -o tsv; rc=$? - case "$rc" in - 2) hosting::die "REFUSED, not absent: the identity this step runs as may not read Key Vault object ${object} in vault ${vault}, so whether it exists is UNKNOWN. Grant it secret GET on that vault (hosting-kv-ensure reads the same object with the same identity when it composes a connection string, so a Provision that did that already holds the grant). Refused: ${HOSTING_PROBE_ERR}" ;; - 1) hosting::die "could not read Key Vault object ${object} from vault ${vault} — absent, or holding an EMPTY value. It must hold the admin password of the flexible server (the record's operator.environment.AZ_POSTGRES_PASSWORD_SECRET names it; hosting-kv-ensure composes connection strings from the same object). Nothing has been dumped, restored or destroyed. az said: ${HOSTING_PROBE_ERR:-nothing}" ;; - esac - PGPASSWORD="$HOSTING_PROBE_OUT"; HOSTING_PROBE_OUT="" - export PGPASSWORD - hosting::log "Postgres admin password read from vault ${vault} object ${object} (value never shown)" -} - -# ── Azure sign-in from a FRESH federated assertion ────────────────────────────────────────────── -# az signs in with a client ASSERTION (a federated token) and, for every new resource it later -# needs a token for (Key Vault, Storage, ARM), presents that SAME assertion to Entra again. The -# assertion is short-lived — a GitHub Actions OIDC token is valid for 5 minutes — so one sign-in at -# the start of a run cannot carry a step that does long work and then calls az: measured on -# Ops/Actions/backup-memex-cloud-memex132-i (Memex run 38025769128), pg_dump took 16.6 minutes and -# the upload that followed failed with "AADSTS700024: Client assertion is not within its valid time -# range … expiry time of assertion 2026-10-10T05:02:10Z". The workload-identity Job has the same -# shape: the kubelet refreshes the projected token FILE, but az only ever read it once. -# -# So the session is re-established from a fresh assertion before any az call that follows long -# work: run.sh before every step, and a step script between its long work and its next az call. -# Where the fresh assertion comes from: -# • the GitHub Actions lane — a NEW OIDC token from ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN (the -# runner exports both to every process of a job granted `id-token: write`), audience -# api://AzureADTokenExchange; the token file is rewritten so anything else reading it is current; -# • the workload-identity Job — the projected AZURE_FEDERATED_TOKEN_FILE, re-read (the kubelet -# keeps it fresh). -# The assertion goes from that source straight into az's argument; it is never printed, logged or -# passed through hosting::do (which narrates argv). - -# Put a fresh federated assertion in HOSTING_AZ_ASSERTION. Non-zero, with HOSTING_AZ_ERR saying why, -# when none can be had. -# -# 🚨 IN PLACE, NEVER ON STDOUT — the rule at the top of this file (hosting::safe_name), met again. -# This used to PRINT the assertion for `assertion="$(hosting::az_assertion)"`. A command -# substitution is a SUBSHELL, so every HOSTING_AZ_ERR set in here died with it, and the caller's -# refusal then expanded a variable that had never been set in ITS shell. Under `set -u` that is not -# an empty cause, it is the end of the script: Memex run 38038325130 (a Roll of memex-cloud) stopped -# at step 3/3 on `_common.sh: line 269: HOSTING_AZ_ERR: unbound variable`, after the chart had been -# applied, and the reason no assertion could be had was lost. Both variables are initialised here so -# neither is ever unbound, and the two places that quote the cause carry a fallback as well. -HOSTING_AZ_ERR="" -HOSTING_AZ_ASSERTION="" -hosting::az_assertion() { - local resp tok - HOSTING_AZ_ERR="" - HOSTING_AZ_ASSERTION="" - if [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] && [ -n "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ]; then - if ! resp="$(curl -fsS --max-time 30 -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ - "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=${HOSTING_OIDC_AUDIENCE:-api://AzureADTokenExchange}" 2>&1)"; then - HOSTING_AZ_ERR="the GitHub OIDC token endpoint did not answer: ${resp}"; return 1 - fi - tok="$(printf '%s' "$resp" | sed -n 's/.*"value"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')" - [ -n "$tok" ] || { HOSTING_AZ_ERR="the GitHub OIDC token endpoint answered without a token value"; return 1; } - if [ -n "${AZURE_FEDERATED_TOKEN_FILE:-}" ]; then - ( umask 077; printf '%s' "$tok" > "$AZURE_FEDERATED_TOKEN_FILE" ) \ - || { HOSTING_AZ_ERR="could not rewrite ${AZURE_FEDERATED_TOKEN_FILE} with the fresh token"; return 1; } - fi - HOSTING_AZ_ASSERTION="$tok" - return 0 - fi - if [ -n "${AZURE_FEDERATED_TOKEN_FILE:-}" ]; then - [ -r "$AZURE_FEDERATED_TOKEN_FILE" ] \ - || { HOSTING_AZ_ERR="AZURE_FEDERATED_TOKEN_FILE=${AZURE_FEDERATED_TOKEN_FILE} is not readable"; return 1; } - tok="$(cat "$AZURE_FEDERATED_TOKEN_FILE")" - [ -n "$tok" ] || { HOSTING_AZ_ERR="AZURE_FEDERATED_TOKEN_FILE=${AZURE_FEDERATED_TOKEN_FILE} is empty"; return 1; } - HOSTING_AZ_ASSERTION="$tok" - return 0 - fi - HOSTING_AZ_ERR="no federated token source: neither ACTIONS_ID_TOKEN_REQUEST_URL/_TOKEN (the Actions lane) nor AZURE_FEDERATED_TOKEN_FILE (the workload-identity Job) is set" - return 2 -} - -# Sign in to Azure as the operator identity from a FRESH assertion. Returns non-zero with -# HOSTING_AZ_ERR set; the caller decides what that refusal says. The assertion is held for the one -# az call and cleared on every path out. -hosting::az_signin() { - local rc=0 - hosting::az_assertion || return 1 - az login --service-principal --username "${AZURE_CLIENT_ID:-}" --tenant "${AZURE_TENANT_ID:-}" \ - --federated-token "$HOSTING_AZ_ASSERTION" --allow-no-subscriptions --output none 2>/tmp/az-login.err || rc=$? - HOSTING_AZ_ASSERTION="" - if [ "$rc" -ne 0 ]; then - HOSTING_AZ_ERR="az login as ${AZURE_CLIENT_ID:-?} failed: $(tr -d '\n' < /tmp/az-login.err)" - return 1 - fi - return 0 -} - -# Re-establish the operator session before an az call that follows long work. A no-op when run.sh -# opened no federated session (HOSTING_AZ_SESSION unset — a by-hand run uses the caller's own az -# login); otherwise a refusal names WHY, because an az call on an expired assertion fails later with -# a message that names neither the session nor the step. -hosting::az_refresh() { - local why="$1" - [ "${HOSTING_AZ_SESSION:-}" = "1" ] || return 0 - hosting::az_signin \ - || hosting::die "the Azure session could not be refreshed ${why}: ${HOSTING_AZ_ERR:-no cause was recorded}. The assertion az signed in with is short-lived, so this call cannot be made on the old session." - hosting::log "azure session refreshed from a fresh federated token (${why})" -} - -# ── the plugin registry's key-lifecycle surface (MeshWeaver#2802) ─────────────────────────────── - -# A registry BASE URL: https, a hostname, an optional port — no path, no query, nothing else. It is -# interpolated into a curl command line, so it is validated like every other name. In place. -hosting::safe_url() { - local what="$1" value="${2:-}" - [[ "$value" =~ ^https://[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?(:[0-9]{1,5})?$ ]] \ - || hosting::die "${what} '${value}' is not an https base URL (https://host[:port], nothing after it) — refusing" -} - -# The decoded value of one key of a Secret, on STDOUT — for CAPTURE into a variable, never for -# printing. An absent Secret, an absent key and an empty value all print nothing and return 1; -# 🚨 a READ THIS IDENTITY WAS REFUSED returns 2, because "there is no such key" and "I was not -# allowed to look" are different sentences and only the caller can say which one it owes its -# reader (MeshWeaver#4722). A caller that only tests success is unaffected — both are non-zero. -# 🚨 Callers capture with $(...) and CHECK THE STATUS THEMSELVES: a hosting::die inside a command -# substitution ends the substitution, not the script (see hosting::safe_name). -hosting::secret_value() { - local namespace="$1" secret="$2" key="$3" value rc - hosting::probe any kubectl -n "$namespace" get secret "$secret" -o json; rc=$? - [ "$rc" -eq 0 ] || return "$rc" - value="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r --arg k "$key" '.data[$k] // empty' 2>/dev/null | base64 -d 2>/dev/null)" || return 1 - [ -n "$value" ] || return 1 - printf '%s' "$value" -} - -# The containers of in that set INLINE (an `env:` entry), joined by -# ", " on STDOUT — empty when none does. Names only; a value is never read out. Returns 1 when the -# Deployment cannot be read. An inline entry outranks every envFrom, so where one exists the pods -# present ITS value, and no Secret an operator step reads says which key that is. -# 🚨 Returns 2 when the read was REFUSED, 1 when the Deployment could not be read for any other -# reason — see hosting::secret_value above (MeshWeaver#4722). -hosting::inline_setters() { - local namespace="$1" deployment="$2" key="$3" rc - hosting::probe any kubectl -n "$namespace" get deployment "$deployment" -o json; rc=$? - [ "$rc" -eq 0 ] || return "$rc" - printf '%s' "$HOSTING_PROBE_OUT" | jq -r --arg k "$key" '[.spec.template.spec.containers[] | select(any(.env[]?; .name == $k)) | .name] | join(", ")' -} - -# Does vault hold a LIVE object named ? Answered from `secret list` (names and -# metadata), never `secret show`: a `show`, even with `--query id`, is a VALUE read that needs `get`, -# which the writer identity does not hold (policy `secrets-write-only-entry`, -# Doc/Architecture/SecretsWriteOnlyEntry). Returns 0 present, 1 absent, 2 when the listing itself -# failed. That last case is NOT "absent": the caller must refuse, because a create-if-absent step -# that read a refused listing as "absent" would overwrite a live secret. -hosting::kv_exists() { - local vault="$1" name="$2" names - names="$(az keyvault secret list --vault-name "$vault" --query "[].name" -o tsv 2>/dev/null)" || return 2 - printf '%s\n' "$names" | grep -qx -- "$name" -} - -# SHA-256 hex of STDIN — how two keys are compared without either being shown. -hosting::sha256() { sha256sum | cut -c1-64; } - -# The FINGERPRINT of the secret value in , on STDOUT: `sha256:`, or -# `withheld:low-entropy` when the value is too guessable to publish a hash of it (policy -# `secrets-write-only-entry`, Doc/Architecture/SecretsWriteOnlyEntry). It is written as the vault -# object's `mw-fp` TAG at write time. A status view can then compare the two ends of a pairing from -# metadata alone, and nobody ever reads the value back. -# -# 🚨 A bare SHA-256 prefix of a LOW-entropy value (a human password) lets anyone guess it offline, -# so the hash is published only when the value is estimated at >= 128 bits: its length × log2 of -# the alphabet its character classes span. Every generated value (32 random bytes), and every real -# API key, client secret or PEM, clears that bar by a wide margin. There is no keyed HMAC: its key -# would be one more secret that both ends of a pairing must READ, which is exactly the reader this -# design removes. Mirrors MeshWeaver.Mesh.SecretFingerprint.Of; the two must agree byte for byte. -hosting::fingerprint() { - local file="$1" len alphabet=0 bits - len="$(LC_ALL=C wc -c < "$file" | tr -d ' ')" - LC_ALL=C grep -q '[a-z]' "$file" && alphabet=$((alphabet + 26)) - LC_ALL=C grep -q '[A-Z]' "$file" && alphabet=$((alphabet + 26)) - LC_ALL=C grep -q '[0-9]' "$file" && alphabet=$((alphabet + 10)) - LC_ALL=C grep -q '[^A-Za-z0-9]' "$file" && alphabet=$((alphabet + 32)) - [ "$alphabet" -gt 0 ] || { echo "withheld:low-entropy"; return 0; } - # Pure bash integer arithmetic — the operator image (Azure Linux 3) carries no awk. log2 of every - # alphabet the four classes can sum to, × 10^9 and truncated; bits × 10^9 = len × that. - local log2e9 - case "$alphabet" in - 10) log2e9=3321928094 ;; 26) log2e9=4700439718 ;; 32) log2e9=5000000000 ;; - 36) log2e9=5169925001 ;; 42) log2e9=5392317422 ;; 52) log2e9=5700439718 ;; - 58) log2e9=5857980995 ;; 62) log2e9=5954196310 ;; 68) log2e9=6087462841 ;; - 84) log2e9=6392317422 ;; 94) log2e9=6554588851 ;; - *) echo "withheld:low-entropy"; return 0 ;; - esac - bits=$(( len * log2e9 / 1000000000 )) - if [ "$bits" -ge 128 ]; then - echo "sha256:$(hosting::sha256 < "$file" | cut -c1-12)" - else - echo "withheld:low-entropy" - fi -} - -# Who asked for a secret write, in a form that can be written as a vault TAG: an email-like -# principal or an identifier. Validates in place; see hosting::safe_name for why it prints nothing. -hosting::safe_actor() { - local what="$1" value="${2:-}" - [[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9@._+-]{0,127}$ ]] \ - || hosting::die "${what} '${value}' is not a principal name (letters, digits, @ . _ + -) — refusing to write it as a vault tag" -} - -# One call to the registry, AUTHENTICATED BY THE KEY IN THE NAMED VARIABLE: -# hosting::registry_call [json-body] -# Sets REGISTRY_STATUS (the HTTP code; "000" when nothing answered) and REGISTRY_BODY. -# -# 🚨 The key travels by variable NAME and reaches curl on STDIN as a config line (`-K -`), so it is -# never an ARGUMENT — not this function's, not curl's, never visible in `ps` while the call runs. -# printf is a shell builtin: it forks no process that could carry the key in its argv either. The -# body is only ever a HASH. Must not be called in a command substitution (it sets globals). -# shellcheck disable=SC2034 # REGISTRY_STATUS/REGISTRY_BODY are read by the scripts sourcing this file -REGISTRY_STATUS="" REGISTRY_BODY="" -hosting::registry_call() { - local keyvar="$1" method="$2" url="$3" body="${4:-}" out - out="$(mktemp)" - local args=(-sS -K - -X "$method" -o "$out" -w '%{http_code}' --max-time 20) - [ -z "$body" ] || args+=(-H 'Content-Type: application/json' --data "$body") - REGISTRY_STATUS="$(printf 'header = "Authorization: Bearer %s"\n' "${!keyvar}" | curl "${args[@]}" "$url" 2>/dev/null)" || true - REGISTRY_STATUS="${REGISTRY_STATUS:-000}" - REGISTRY_BODY="$(cat "$out" 2>/dev/null || true)" - rm -f "$out" -} - -# One field of the last registry answer (REGISTRY_BODY), or nothing. -hosting::registry_field() { - printf '%s' "$REGISTRY_BODY" | jq -r --arg f "$1" 'if type == "object" then (.[$f] // empty | tostring) else empty end' 2>/dev/null -} - -# hosting::values_half — run the values-half key check / filter -# (_values_half.py, policy `one-values-half-per-release`). It needs python3 with PyYAML, which the -# operator image installs (Dockerfile: tdnf PyYAML). Missing either is a named refusal, never a skip: -# a half whose keys cannot be read is not a half whose keys are right. -hosting::values_half() { - local helper - helper="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_values_half.py" - command -v python3 >/dev/null 2>&1 \ - || hosting::die "python3 is not on PATH — the values-half key check needs it (operator image: tdnf python3 PyYAML)" - python3 "$helper" "$@" -} diff --git a/deploy/aks/operator/bin/_dump_analyze_pod.sh b/deploy/aks/operator/bin/_dump_analyze_pod.sh deleted file mode 100644 index 816dfb7ae3..0000000000 --- a/deploy/aks/operator/bin/_dump_analyze_pod.sh +++ /dev/null @@ -1,114 +0,0 @@ -# shellcheck shell=sh -# The IN-POD half of hosting-dump-analyze — NOT run by the operator. hosting-dump-analyze reads this -# file and hands it to the analysis Job as `sh -c`, in the instance's namespace, on the analyser image -# (Debian, GNU coreutils/findutils), with the dump claim mounted READ-ONLY at $MW_ROOT. -# -# It prints ONLY sentinel lines (`@@MW-DUMP \t…`) and the text of a FIXED set of dotnet-dump -# commands; hosting-dump-analyze parses nothing else: -# -# @@MW-DUMP listing then one `\t\t` per dump -# @@MW-DUMP selected\t\t\t -# @@MW-DUMP section\t\t then that command's output (at most $MW_SECTION_BYTES) -# @@MW-DUMP end\t\t\t -# @@MW-DUMP done the analysis reached its end -# @@MW-DUMP refused\t it refused, and why — then exit non-zero -# -# 🚨 The selected path is resolved and must lie UNDER the mount: a symlink (file or directory) cannot -# walk the read out of the dump volume. The operator validated the selector's shape before the Job -# existed; this is the second half, where the filesystem is. -# -# Inputs (env): MW_DUMP | MW_POD | MW_AROUND (at most one), MW_DOTNET_DUMP_VERSION, MW_CMD_TIMEOUT, -# MW_SECTION_BYTES; MW_ROOT (default /dumps) and MW_TOOLS (default /tmp/dd) exist for the test. -set -u -root="${MW_ROOT:-/dumps}" -tools="${MW_TOOLS:-/tmp/dd}" -tab="$(printf '\t')" -emit() { printf '@@MW-DUMP %s\n' "$*"; } -refuse() { emit "refused${tab}$*"; exit 3; } - -rootreal="$(realpath -e "$root")" || refuse "the dump root is not mounted" -emit "listing" -find "$rootreal" -mindepth 2 -maxdepth 2 -type f -name 'coredump.*' -printf '%s\t%Ts\t%P\n' | sort -t "$tab" -k2,2nr -k3,3 - -# The newest dump matching extra find predicates, relative to the root. -newest() { - find "$rootreal" -mindepth 2 -maxdepth 2 -type f -name 'coredump.*' "$@" -printf '%Ts\t%P\n' \ - | sort -t "$tab" -k1,1nr | head -1 | cut -f2 -} - -rel="" -if [ -n "${MW_DUMP:-}" ]; then - rel="$MW_DUMP" -elif [ -n "${MW_POD:-}" ]; then - # A pod's live directory and its parked ones (`.`, crash-dump-retention.sh step 0). - rel="$(newest \( -path "$rootreal/$MW_POD/*" -o -path "$rootreal/$MW_POD.*/*" \))" - [ -n "$rel" ] || refuse "pod $MW_POD left no dump on the volume" -elif [ -n "${MW_AROUND:-}" ]; then - at="$(date -u -d "$MW_AROUND" +%s)" || refuse "cannot read the instant $MW_AROUND" - best="" bestd="" - # createdump names the file coredump.. (DOTNET_DbgMiniDumpName …/coredump.%p.%t). - candidates="$(find "$rootreal" -mindepth 2 -maxdepth 2 -type f -name 'coredump.*.*' -printf '%P\n' | sort)" - while IFS= read -r f; do - t="${f##*.}" - case "$t" in ''|*[!0-9]*) continue ;; esac - d=$((t - at)); [ "$d" -lt 0 ] && d=$((0 - d)) - if [ "$d" -le 1800 ] && { [ -z "$bestd" ] || [ "$d" -lt "$bestd" ]; }; then best="$f"; bestd="$d"; fi - done </tmp/mw-dd-install.log 2>&1 \ - || refuse "could not install dotnet-dump $MW_DOTNET_DUMP_VERSION from nuget.org: $(tail -3 /tmp/mw-dd-install.log | tr '\n' ' ')" - -# section . `head` keeps the first $MW_SECTION_BYTES of the output. -# `stat` is for `dumpheap -stat`, which lists its types SMALLEST first and closes the table with its -# `Total … objects, … bytes` line, so the types that hold the heap are at the table's END. It keeps -# the output THROUGH that Total line — dropping what SOS prints after it (the "Fragmented blocks" -# table, which names object addresses and whose rows would displace the statistics) — and of that, -# the last $MW_SECTION_BYTES from the first whole line. Either way the full byte count is reported on -# the `end` line, so a cut is never silent. -section() { - keep="$1" name="$2"; shift 2 - emit "section${tab}${name}${tab}$*" - timeout "$MW_CMD_TIMEOUT" "$tools/dotnet-dump" analyze "$real" -c "setsymbolserver -ms" -c "$*" -c exit /tmp/mw-dd-out.txt 2>&1 - rc=$? - size="$(stat -c %s /tmp/mw-dd-out.txt)" - if [ "$keep" = stat ]; then - sed '/^[[:space:]]*Total [0-9,]* objects/q' /tmp/mw-dd-out.txt >/tmp/mw-dd-stat.txt - if [ "$(stat -c %s /tmp/mw-dd-stat.txt)" -gt "$MW_SECTION_BYTES" ]; then - tail -c "$MW_SECTION_BYTES" /tmp/mw-dd-stat.txt | sed 1d - else - cat /tmp/mw-dd-stat.txt - fi - else - head -c "$MW_SECTION_BYTES" /tmp/mw-dd-out.txt - fi - printf '\n' - emit "end${tab}${name}${tab}${rc}${tab}${size}" -} - -# FIXED. None of these prints an object's fields, a string's value or the environment: `gcheapstat` -# prints per-heap generation sizes, `dumpheap -stat` one row per TYPE (method table, object count, -# total bytes, type name) — never an object, never an address of one. -section head eeversion eeversion -section head threadpool threadpool -section head gcheapstat gcheapstat -section stat dumpheap dumpheap -stat -section head syncblk syncblk -section head clrthreads clrthreads -section head clrstack clrstack -all -section head dumpasync dumpasync --coalesce -emit "done" diff --git a/deploy/aks/operator/bin/_storage_prune_pod.sh b/deploy/aks/operator/bin/_storage_prune_pod.sh deleted file mode 100644 index 5c5fcf55e7..0000000000 --- a/deploy/aks/operator/bin/_storage_prune_pod.sh +++ /dev/null @@ -1,246 +0,0 @@ -# shellcheck shell=sh -# The IN-POD half of hosting-storage-prune — NOT run by the operator. hosting-storage-prune reads this -# file and hands it to a short-lived Job as `sh -c`, in the instance's namespace, on a busybox image, -# with the portal's DATA claim mounted at $MW_ROOT — READ-ONLY for a report, read-write only for an -# --execute run. -# -# It prints ONLY sentinel lines (`@@MW-STORE \t…`); hosting-storage-prune parses nothing else: -# -# @@MW-STORE begin\t\t -# @@MW-STORE df\t\t\t -# @@MW-STORE top\t\t every top-level entry but prebuilt-bundles -# @@MW-STORE releases\t\t prebuilt-bundles/_releases -# @@MW-STORE identity\t\t\t\t kind: compat | legacy | other (KiB of a compat -# identity is 0 here — its generations carry it) -# @@MW-STORE generation\t\t\t\t\t\t\t -# state: current | sealed | unsealed -# @@MW-STORE flat\t\t\t\t\t\t -# @@MW-STORE tmp\t\t\t an abandoned assembly-cache temp file (.tmp-*) -# @@MW-STORE dump\t\t\t a file under the DATA share's dumps/ directory -# @@MW-STORE deleted\t\t\t --execute only -# @@MW-STORE kept\t\t\t --execute only: a candidate the re-check spared -# @@MW-STORE unreadable\t\t a read that failed — the report is then INCOMPLETE -# @@MW-STORE done the walk reached its end -# @@MW-STORE refused\t it refused, and why — then exit non-zero -# -# 🚨 WHAT THIS SCRIPT MAY DELETE, and it is deliberately almost nothing (Doc/Architecture/StoragePrune): -# • `cache-tmp` — assembly-cache/**/.tmp-* older than MW_TMP_AGE_DAYS (default 1): an interrupted -# write of the content-addressed assembly store. No reader resolves a temp name. -# • `misplaced-dump` — files under the DATA share's own dumps/ directory, older than -# MW_DUMP_AGE_DAYS (default 7), and ONLY when the operator established that the -# portal mounts a SEPARATE dump volume over /data/dumps (MW_DUMPS_SHADOWED=1). Then -# nothing in the portal can see these files at all: they are bytes written before -# the dump volume existed, hidden under its mount point. -# It NEVER deletes anything under prebuilt-bundles/. Those are release artifacts, governed by -# Doc/Architecture/ReleasedArtifactRetention: deleting one needs the complete fleet consumer inventory -# and the publication lease, and neither is available to a pod. The bundle store is REPORTED here — -# every identity and generation, with its age, size, seal and lock — so the control instance can -# classify it; it is never written. -# -# Every deletion is re-checked at the moment it happens: the path is resolved and must lie UNDER the -# mount (a symlink cannot walk out), it must still be a regular file, and its age is re-read. A file -# that got younger, moved, or became a link is KEPT and reported as kept. -# -# Inputs (env): MW_MODE (report|execute), MW_DUMPS_SHADOWED (0|1), MW_TMP_AGE_DAYS, MW_DUMP_AGE_DAYS; -# MW_ROOT (default /store) and MW_NOW (default: the clock) exist for the test. -set -u -root="${MW_ROOT:-/store}" -mode="${MW_MODE:-report}" -now="${MW_NOW:-$(date +%s)}" -tmp_age_days="${MW_TMP_AGE_DAYS:-1}" -dump_age_days="${MW_DUMP_AGE_DAYS:-7}" -dumps_shadowed="${MW_DUMPS_SHADOWED:-0}" -tab="$(printf '\t')" -emit() { printf '@@MW-STORE %s\n' "$*"; } -refuse() { emit "refused${tab}$*"; exit 3; } - -case "$mode" in report|execute) ;; *) refuse "unknown mode '$mode'" ;; esac -for n in "$now" "$tmp_age_days" "$dump_age_days"; do - case "$n" in ''|*[!0-9]*) refuse "a numeric setting ('$n') is not a whole number" ;; esac -done -# A day is the smallest age that makes anything a candidate: a zero window would delete a temp file a -# compile is writing right now. -[ "$tmp_age_days" -ge 1 ] || refuse "MW_TMP_AGE_DAYS must be at least 1" -[ "$dump_age_days" -ge 1 ] || refuse "MW_DUMP_AGE_DAYS must be at least 1" - -rootreal="$(cd "$root" 2>/dev/null && pwd -P)" || refuse "the data root $root is not mounted" -[ -d "$rootreal" ] || refuse "the data root $root is not a directory" -# An empty mount is a claim that did not attach (or the wrong claim), never a clean volume: refusing -# keeps "nothing to prune" from being reported over a share nobody looked at. -[ -n "$(ls -A "$rootreal" 2>/dev/null)" ] || refuse "the data root $root is EMPTY — the claim did not attach, or it is not the portal's data claim" - -emit "begin${tab}${now}${tab}${mode}" - -# shellcheck disable=SC2046 # df's columns, split on purpose -set -- $(df -Pk "$rootreal" | tail -1) -emit "df${tab}${2:-0}${tab}${3:-0}${tab}${4:-0}" - -# fd 3 is the log itself: kib/mtime run inside $(…), whose stdout is the VALUE, so their report of a -# failed read goes to fd 3, which still points at the pod's stdout. -exec 3>&1 -incomplete="$(mktemp)"; rm -f "$incomplete" # exists ⇔ some read failed -# A read that fails is NEVER a zero: it is reported as `unreadable` (and the size printed is what du -# managed to count), so the operator can mark the whole report incomplete — an incomplete walk licenses -# no deletion and must not read as a clean one. -kib() { - _out="$(du -sk "$1" 2>/dev/null)" || { emit "unreadable${tab}${1#"$rootreal"/}${tab}du" >&3; : > "$incomplete"; } - _k="$(printf '%s' "$_out" | cut -f1)" - echo "${_k:-0}" -} -mtime() { - stat -c %Y "$1" 2>/dev/null && return 0 - emit "unreadable${tab}${1#"$rootreal"/}${tab}stat" >&3 - : > "$incomplete" - echo 0 -} -# The newest mtime of a directory and its immediate children: a directory's own stamp on an SMB share -# moves only when an entry is added or removed, so a generation still being written could read older -# than it is from the directory alone. -newest() { - best=0 - for f in "$1" "$1"/* "$1"/.[!.]*; do - [ -e "$f" ] || continue - t="$(mtime "$f")" - [ "$t" -gt "$best" ] && best="$t" - done - echo "$best" -} - -for entry in "$rootreal"/* "$rootreal"/.[!.]*; do - [ -e "$entry" ] || continue - name="${entry##*/}" - [ "$name" = "prebuilt-bundles" ] && continue - case "$name" in linux-x64|linux-arm64|linux-musl-x64|linux-musl-arm64|osx-arm64|osx-x64|win-x64|win-arm64) - [ -d "$entry/prebuilt-bundles" ] && continue ;; esac - emit "top${tab}${name}${tab}$(kib "$entry")" -done - -# The sources of one identity. `_prefix` is unused today and kept empty; the lease file sits beside -# the source directories it guards. -walk_sources() { - _idpath="$1" _level="$2" _prefix="$3" - for srcpath in "$_level"/*; do - [ -d "$srcpath" ] || continue - src="${srcpath##*/}" - locked=0 - [ -e "$_level/_publishing.$src" ] && locked=1 - if [ -f "$srcpath/_current" ]; then - current="$(head -1 "$srcpath/_current" | tr -d '\r\n ')" - for genpath in "$srcpath"/*; do - [ -d "$genpath" ] || continue - gen="${genpath##*/}" - if [ "$gen" = "$current" ]; then state=current - elif [ -e "$genpath/_complete" ]; then state=sealed - else state=unsealed; fi - size=0 - # A legacy identity was already measured whole; its generations are reported for their - # state only, so the walk does not read the same bytes twice. - [ "$kind" = legacy ] || size="$(kib "$genpath")" - emit "generation${tab}${id}${tab}${_prefix}${src}${tab}${gen}${tab}${state}${tab}$(newest "$genpath")${tab}${size}${tab}${locked}" - done - elif [ "$kind" = compat ]; then - sealed=0; [ -e "$srcpath/_complete" ] && sealed=1 - emit "flat${tab}${id}${tab}${_prefix}${src}${tab}${sealed}${tab}$(newest "$srcpath")${tab}$(kib "$srcpath")${tab}${locked}" - fi - done -} - -# The bundle stores under the root. A per-portal data volume has ONE, `prebuilt-bundles/`. The shared -# artifact store (Doc/Architecture/SharedArtifactStore) has one PER ARCHITECTURE, `/prebuilt-bundles/` -# (linux-x64, osx-arm64, …): a compatibility key names no architecture, so each runtime gets its own -# store and a reader resolves its own. Identities of a RID store are reported as `/`. -walk_store() { - store="$1" idprefix="$2" - for idpath in "$store"/*; do - [ -e "$idpath" ] || continue - base="${idpath##*/}" - id="${idprefix}${base}" - if [ "$base" = "_releases" ]; then - emit "releases${tab}$(ls "$idpath" 2>/dev/null | wc -l | tr -d ' ')${tab}$(kib "$idpath")" - continue - fi - [ -d "$idpath" ] || { emit "identity${tab}${id}${tab}other${tab}$(mtime "$idpath")${tab}$(kib "$idpath")"; continue; } - case "$base" in - c[0-9][0-9][0-9]e[0-9][0-9][0-9]) kind=compat ;; - s[0-9a-f]*|g[0-9a-f]*) kind=legacy ;; - *) kind=other ;; - esac - if [ "$kind" = compat ]; then - emit "identity${tab}${id}${tab}${kind}${tab}$(mtime "$idpath")${tab}0" - else - emit "identity${tab}${id}${tab}${kind}${tab}$(newest "$idpath")${tab}$(kib "$idpath")" - fi - walk_sources "$idpath" "$idpath" "" - done -} -[ -d "$rootreal/prebuilt-bundles" ] && walk_store "$rootreal/prebuilt-bundles" "" -for ridpath in "$rootreal"/*; do - rid="${ridpath##*/}" - case "$rid" in - linux-x64|linux-arm64|linux-musl-x64|linux-musl-arm64|osx-arm64|osx-x64|win-x64|win-arm64) - [ -d "$ridpath/prebuilt-bundles" ] && walk_store "$ridpath/prebuilt-bundles" "$rid/" ;; - esac -done - -# ── the two classes this script may delete ───────────────────────────────────────────────────── -tmp_cut=$((now - tmp_age_days * 86400)) -dump_cut=$((now - dump_age_days * 86400)) - -# A candidate list, one `\t` per line; reported in both modes. -cands="$(mktemp)" -trap 'rm -f "$cands" "$incomplete"' EXIT -# A listing that fails partway is an incomplete walk too, never a shorter one. -listerr="$(mktemp)" -trap 'rm -f "$cands" "$incomplete" "$listerr"' EXIT -if [ -d "$rootreal/assembly-cache" ]; then - find "$rootreal/assembly-cache" -type f -name '.tmp-*' 2>"$listerr" | while IFS= read -r f; do - t="$(mtime "$f")" - rel="${f#"$rootreal"/}" - emit "tmp${tab}${rel}${tab}$(kib "$f")${tab}${t}" - [ "$t" -gt 0 ] && [ "$t" -lt "$tmp_cut" ] && printf 'cache-tmp\t%s\n' "$rel" >> "$cands" - done -fi -[ -s "$listerr" ] && { emit "unreadable${tab}assembly-cache${tab}find"; : > "$incomplete"; } -: > "$listerr" -if [ -d "$rootreal/dumps" ]; then - find "$rootreal/dumps" -type f 2>"$listerr" | while IFS= read -r f; do - t="$(mtime "$f")" - rel="${f#"$rootreal"/}" - emit "dump${tab}${rel}${tab}$(kib "$f")${tab}${t}" - [ "$dumps_shadowed" = 1 ] && [ "$t" -gt 0 ] && [ "$t" -lt "$dump_cut" ] && printf 'misplaced-dump\t%s\n' "$rel" >> "$cands" - done -fi - -[ -s "$listerr" ] && { emit "unreadable${tab}dumps${tab}find"; : > "$incomplete"; } - -if [ "$mode" = execute ]; then - while IFS="$tab" read -r class rel; do - [ -n "$rel" ] || continue - # An incomplete walk licenses no deletion — not even of the garbage classes. - if [ -e "$incomplete" ]; then emit "kept${tab}${class}${tab}${rel}${tab}the walk was incomplete (a read failed)"; continue; fi - f="$rootreal/$rel" - case "$class" in - cache-tmp) cut="$tmp_cut"; under="$rootreal/assembly-cache/" ;; - misplaced-dump) cut="$dump_cut"; under="$rootreal/dumps/" ;; - *) emit "kept${tab}${class}${tab}${rel}${tab}unknown class"; continue ;; - esac - if [ -L "$f" ] || [ ! -f "$f" ]; then emit "kept${tab}${class}${tab}${rel}${tab}no longer a regular file"; continue; fi - dir="$(cd "${f%/*}" 2>/dev/null && pwd -P)" || { emit "kept${tab}${class}${tab}${rel}${tab}its directory cannot be resolved"; continue; } - case "$dir/" in "$under"*) ;; *) emit "kept${tab}${class}${tab}${rel}${tab}resolves outside ${under#"$rootreal"/}"; continue ;; esac - case "$class" in - cache-tmp) case "${f##*/}" in .tmp-*) ;; *) emit "kept${tab}${class}${tab}${rel}${tab}not a temp file name"; continue ;; esac ;; - misplaced-dump) [ "$dumps_shadowed" = 1 ] || { emit "kept${tab}${class}${tab}${rel}${tab}the dumps directory is not shadowed by a dump volume"; continue; } ;; - esac - t="$(mtime "$f")" - if [ "$t" -le 0 ] || [ "$t" -ge "$cut" ]; then emit "kept${tab}${class}${tab}${rel}${tab}younger than its window on re-check"; continue; fi - size="$(kib "$f")" - if rm -f -- "$f"; then emit "deleted${tab}${class}${tab}${rel}${tab}${size}" - else emit "kept${tab}${class}${tab}${rel}${tab}rm failed"; fi - done < "$cands" - # Directories the misplaced dumps leave empty go too — never the dumps/ directory itself, which is - # the dump volume's mount point. - [ "$dumps_shadowed" = 1 ] && [ -d "$rootreal/dumps" ] \ - && find "$rootreal/dumps" -mindepth 1 -depth -type d -exec rmdir {} \; 2>/dev/null -fi - -emit "done" diff --git a/deploy/aks/operator/bin/_values_half.py b/deploy/aks/operator/bin/_values_half.py deleted file mode 100755 index f8c10823ea..0000000000 --- a/deploy/aks/operator/bin/_values_half.py +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env python3 -""" -The helm VALUES HALF (`helm-values-` in Key Vault) holds EXACTLY the keys its Deployment -record declares in `vaultValuesKeys` — policy `one-values-half-per-release` (MeshWeaver#4685). - - _values_half.py check --keys K1,K2,... names only; exit 0 exact, 3 mismatch - _values_half.py filter --keys K1,... value-preserving rewrite; exit 0 / 3 - _values_half.py set --keys K1,... --set K,... - change NAMED keys, keep everything else; exit 0 / 3 - _values_half.py verify --set K,... the named keys hold exactly those values; exit 0 / 3 - -WHY. The half was a CAPTURE of whatever the live release carried (`helm get values`, filtered to the -secret families), so it could feed keys nobody declared — a `config` leaf riding in the vault renders -as a `CLUSTER-ONLY` finding that is wrongly CAUSED (Systemorph/Memex#295), and a secret nobody -declared is a secret nobody reviews. The record already says which keys the chart's own Secret is fed -from the vault (`vaultValuesKeys`, the same list HelmValues attributes to `memex-portal-secrets`); the -half is now held to that list in BOTH directions: no undeclared key, no declared key missing. - -THE KEY NAMES. A declared key is a SECRET ENV KEY (`ConnectionStrings__memex`, `MEMEX_PASSWORD`, -`Anthropic__ApiKey`). The half stores it under `secrets.memex_portal.` (and the migration's -copy under `secrets.memex_migration.`), where the leaf is the env key itself except for ONE -chart alias: `memex_postgres_password` renders as `MEMEX_PASSWORD`. `ConnectionStrings__orleans` may -be absent from the half when `ConnectionStrings__memex` is present — the chart derives it by its ONE -rule (`memex.orleansConnectionString`), and hosting-kv-ensure deliberately never writes a second copy. -Anything else in the half — `parameters`, `pgbackrest`, `secrets.memex_postgres`, any other -`secrets.` — names no declared key and is UNDECLARED. - -SET (the governed key-level writer, `hosting-kv-values-set`). Before it, only a Provision could -write the half, and only when it was ABSENT — so rotating a value that also lives in the half (the -control instance's `control-db-connection` through the record page's secret dialog) left a stale copy -in it that the next Reconcile layered straight back onto the pods. `set` changes exactly the named -keys and nothing else: - • every named key must be one the record declares (`--keys`), or nothing is written (exit 3); - • every other byte of meaning in the half is KEPT — other leaves, other halves, other families - (`parameters`, `pgbackrest`), multi-line values (a PEM block scalar) — the document is re-emitted - as JSON, which helm reads as YAML, so a block scalar's value survives byte-for-byte; - • a key is written into every half that already carries its leaf; one no half carries yet goes into - `memex_portal`, and also into `memex_migration` when the migration reads it too - (`ConnectionStrings__memex`, `MEMEX_PASSWORD` — the two hosting-kv-ensure composes into both); - • `ConnectionStrings__orleans` is written only where a half already carries it: otherwise the chart - DERIVES it from `ConnectionStrings__memex` by its one rule, and writing a first copy would freeze - that derivation — rotate `ConnectionStrings__memex` instead (refused, named); - • an empty value is refused: a blank connection string fails at silo start with nothing naming this. -`verify` reads a half back and compares the named leaves with the values file — by equality, printing -names only. - -🔒 NO VALUE IS EVER PRINTED. `check` prints key names; `filter` writes values only to the output file -(the caller creates it 0600) and prints names. A value never reaches argv, stdout or stderr. -""" -import json, sys - -HALVES = ("memex_portal", "memex_migration") -ALIAS = {"memex_postgres_password": "MEMEX_PASSWORD"} -LEAF_FOR = {v: k for k, v in ALIAS.items()} -DERIVED = {"ConnectionStrings__orleans": "ConnectionStrings__memex"} - - -def env_name(leaf): - return ALIAS.get(leaf, leaf) - - -def _yaml(): - # Only a YAML half needs PyYAML (a captured half is block YAML). What hosting-kv-ensure COMPOSES is - # JSON and is read by the standard library, so a Provision never depends on PyYAML: the aks-silos - # runner lacked it and the pg3 Provision stopped at 'Create Key Vault secrets' (2026-10-17). - try: - import yaml - except ImportError: # the operator image installs PyYAML; a missing module is a named refusal - print("::error::_values_half.py needs PyYAML (python3 'yaml' module) to read a YAML values half — " - "the operator image installs it (Dockerfile: tdnf PyYAML). Refusing rather than guessing " - "the half's keys.", file=sys.stderr) - sys.exit(2) - return yaml - - -def load(path): - text = open(path).read() - try: - doc = json.loads(text) - except ValueError: - try: - doc = _yaml().safe_load(text) - except Exception as e: # noqa: BLE001 — named, fatal - print(f"::error::the values half cannot be parsed ({type(e).__name__}) — refusing.", file=sys.stderr) - sys.exit(2) - if not isinstance(doc, dict): - print("::error::the values half is not a mapping — refusing.", file=sys.stderr) - sys.exit(2) - return doc - - -def inventory(doc): - """(fed: {half: {envName}}, undeclared: [path names]) — names only.""" - fed = {h: set() for h in HALVES} - other = [] - for family, body in doc.items(): - if family != "secrets": - other.append(str(family)) - continue - if not isinstance(body, dict): - other.append("secrets") - continue - for half, leaves in body.items(): - if half not in HALVES: - other.append(f"secrets.{half}") - continue - for leaf in (leaves or {}): - fed[half].add(env_name(str(leaf))) - return fed, other - - -def compare(doc, declared): - fed, other = inventory(doc) - supplied = set() - for h in HALVES: - supplied |= fed[h] - for derived, source in DERIVED.items(): - if source in fed[h]: - supplied.add(derived) - undeclared = sorted({k for h in HALVES for k in fed[h]} - declared) + sorted(other) - missing = sorted(declared - supplied) - return undeclared, missing - - -def parse_keys(argv): - if "--keys" not in argv: - print("::error::--keys K1,K2,... (the record's vaultValuesKeys) is required", file=sys.stderr) - sys.exit(2) - raw = argv[argv.index("--keys") + 1] if argv.index("--keys") + 1 < len(argv) else "" - keys = {k.strip() for k in raw.split(",") if k.strip()} - if not keys: - print("::error::--keys is empty — a record with no vaultValuesKeys has no values half", file=sys.stderr) - sys.exit(2) - return keys - - -def report(undeclared, missing): - if undeclared: - print("undeclared: " + " ".join(undeclared)) - if missing: - print("missing: " + " ".join(missing)) - - -# The keys the MIGRATION Job reads from its own half too — hosting-kv-ensure composes exactly these two -# into both `memex_portal` and `memex_migration`, so a first write of either lands in both. -MIGRATION_KEYS = ("ConnectionStrings__memex", "MEMEX_PASSWORD") - - -def parse_set(argv): - if "--set" not in argv or argv.index("--set") + 1 >= len(argv): - print("::error::--set K,... (the keys to change) is required", file=sys.stderr) - sys.exit(2) - raw = argv[argv.index("--set") + 1] - keys = [k.strip() for k in raw.split(",") if k.strip()] - if not keys: - print("::error::--set names no key — a write that changes nothing is a bug in the caller", file=sys.stderr) - sys.exit(2) - return list(dict.fromkeys(keys)) - - -def load_values(path, keys): - """{envKey: value} from the 0600 values file; every named key a non-empty string, or exit 3.""" - try: - values = json.load(open(path)) - except Exception as e: # noqa: BLE001 — named, fatal - print(f"::error::the values file cannot be parsed ({type(e).__name__}) — refusing.", file=sys.stderr) - sys.exit(2) - if not isinstance(values, dict): - print("::error::the values file is not an object of {key: value} — refusing.", file=sys.stderr) - sys.exit(2) - empty = [k for k in keys if not isinstance(values.get(k), str) or values.get(k) == ""] - if empty: - print("novalue: " + " ".join(empty)) - sys.exit(3) - return values - - -def targets_for(doc, key): - """The (half, leaf) pairs a key is written to, or None when it may not be written (derived).""" - leaf = LEAF_FOR.get(key, key) - secrets = doc.get("secrets") if isinstance(doc.get("secrets"), dict) else {} - carrying = [h for h in HALVES if isinstance(secrets.get(h), dict) and leaf in secrets[h]] - if carrying: - return [(h, leaf) for h in carrying] - if key in DERIVED: - return None - halves = ["memex_portal"] + (["memex_migration"] if key in MIGRATION_KEYS else []) - return [(h, leaf) for h in halves] - - -def set_keys(doc, values, keys): - """Mutates doc: each key's leaf set in its target halves. Returns [(key, [paths])].""" - if "secrets" not in doc or doc["secrets"] is None: - doc["secrets"] = {} - if not isinstance(doc["secrets"], dict): - print("::error::the half's `secrets` is not a mapping — refusing to write into it.", file=sys.stderr) - sys.exit(2) - written = [] - for key in keys: - targets = targets_for(doc, key) - for half, leaf in targets: - body = doc["secrets"].get(half) - if body is None: - body = doc["secrets"][half] = {} - if not isinstance(body, dict): - print(f"::error::secrets.{half} is not a mapping — refusing to write into it.", file=sys.stderr) - sys.exit(2) - body[leaf] = values[key] - written.append((key, [f"secrets.{h}.{l}" for h, l in targets])) - return written - - -def leaf_values(doc, key): - leaf = LEAF_FOR.get(key, key) - secrets = doc.get("secrets") if isinstance(doc.get("secrets"), dict) else {} - return [secrets[h][leaf] for h in HALVES if isinstance(secrets.get(h), dict) and leaf in secrets[h]] - - -def main(argv): - if len(argv) < 2 or argv[1] not in ("check", "filter", "set", "verify"): - print(__doc__.strip().splitlines()[0], file=sys.stderr) - return 2 - if argv[1] == "verify": - keys = parse_set(argv) - doc = load(argv[2]) - values = load_values(argv[3], keys) - wrong = [k for k in keys if not leaf_values(doc, k) or any(v != values[k] for v in leaf_values(doc, k))] - if wrong: - print("differs: " + " ".join(wrong)) - return 3 - return 0 - declared = parse_keys(argv) - if argv[1] == "set": - keys = parse_set(argv) - undeclared = [k for k in keys if k not in declared] - if undeclared: - print("undeclared: " + " ".join(undeclared)) - return 3 - doc = load(argv[2]) - derived = [k for k in keys if targets_for(doc, k) is None] - if derived: - print("derived: " + " ".join(f"{k}<-{DERIVED[k]}" for k in derived)) - return 3 - values = load_values(argv[3], keys) - written = set_keys(doc, values, keys) - try: - text = json.dumps(doc) - except (TypeError, ValueError) as e: - # A scalar YAML typed as something JSON has no word for (a bare date): re-emitting it as a - # string would change what helm reads for a key this write never named — refused instead. - print(f"::error::the half holds a value JSON cannot carry unchanged ({type(e).__name__}) — " - "refusing to re-emit it. Nothing was written.", file=sys.stderr) - return 2 - with open(argv[4], "w") as out: - out.write(text) - for key, paths in written: - print(f"set: {key} -> {','.join(paths)}") - undeclared_after, missing_after = compare(doc, declared) - if undeclared_after: - print("still-undeclared: " + " ".join(undeclared_after)) - if missing_after: - print("still-missing: " + " ".join(missing_after)) - return 0 - doc = load(argv[2]) - undeclared, missing = compare(doc, declared) - if argv[1] == "check": - report(undeclared, missing) - return 0 if not undeclared and not missing else 3 - # filter: keep exactly the declared keys, values untouched; a declared key the half cannot - # supply cannot be invented, so that is a refusal and nothing is written. - if missing: - report([], missing) - return 3 - out_path = argv[3] - secrets = doc.get("secrets") or {} - kept = {} - for half in HALVES: - leaves = secrets.get(half) or {} - if not isinstance(leaves, dict): - continue - keep = {leaf: value for leaf, value in leaves.items() if env_name(str(leaf)) in declared} - if keep: - kept[half] = keep - with open(out_path, "w") as out: - json.dump({"secrets": kept}, out) - if undeclared: - print("dropped: " + " ".join(undeclared)) - return 0 - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/deploy/aks/operator/bin/hosting-aks-upgrade b/deploy/aks/operator/bin/hosting-aks-upgrade deleted file mode 100755 index 26983dd24b..0000000000 --- a/deploy/aks/operator/bin/hosting-aks-upgrade +++ /dev/null @@ -1,481 +0,0 @@ -#!/usr/bin/env bash -# hosting-aks-upgrade --resource-group --cluster [flags] -# -# The cluster half of the governed `UpgradeCluster` InstanceAction (MeshWeaver.Plugins -# Hosting/ClusterUpgrade.md). The Kubernetes upgrade of the AKS cluster itself — control plane, -# then every node pool — as ordered, idempotent, resumable steps. Never run by hand: the plan is -# composed and ORDERED by the Hosting plugin's pure ClusterUpgradePlan (one approval, control-plane -# minors chained, every pool upgraded ONCE to the final version, the pool hosting the portals LAST, -# a health gate before each pool) and arrives as a signed bundle on the aks-ops lane. -# -# 🚨 THIS SCRIPT DECIDES NOTHING ABOUT THE ORDER. Which versions, which pools, in what sequence is -# the plan's (tested in the plugin). What lives here is what only the cluster can answer, measured -# at the moment it matters: -# -# facts READ-ONLY. Everything the plan is evaluated from — cluster version, the versions -# AKS offers (get-upgrades + the region's get-versions graph), every node pool, -# every node, every portal Deployment (its TEMPLATE image against the image its -# pods actually SERVE), each portal host's /health, every PodDisruptionBudget, the -# API server's own deprecated-API usage metric and the apiVersions the scanned -# manifests name — as ONE base64 JSON fact (`aks_facts=`). -# gate The health gate between pools: no operation in flight, every node Ready and -# schedulable, every portal converged on its template image and answering -# /health 200. `--strict` refuses at once (the pre-flight); otherwise it waits -# inside the job budget, and FAILS on what Kubernetes itself has given up on -# (ProgressDeadlineExceeded, a pod not Ready past its own startup budget). -# control-plane One control-plane-only hop to --version (the plan chains one hop per minor). -# node-pool One pool straight to --version (the node skew policy allows it; the plan -# upgrades each pool exactly once). -# verify Everything at --version and healthy → `aks_upgrade=complete`, the ONLY success. -# -# 🚨 A STUCK ROLL REFUSES THE UPGRADE. A portal Deployment whose template names an image its -# serving pods do not run is a roll in flight or wedged. Draining a node recreates those pods FROM -# THE TEMPLATE — measured 2026-09-25: memex's template named 3.0.0-ci.9260 (pods of that image -# deadlock at bake) while its serving pods were 3.0.0-ci.9218, alive only because nothing evicted -# them. A node drain would have taken the control instance down. `gate --strict` names every such -# Deployment and runs nothing. -# -# RESUMABLE, because an AKS upgrade outlives a 45-minute job. Every mutating step is idempotent -# (already at the version → "already"; an operation in flight → it is waited on, never re-issued), -# and the job's budget (--budget-seconds, measured from the FIRST step of this job) bounds every -# wait. A wait that reaches the budget PAUSES the run: it says `aks_upgrade=incomplete`, exits 0, -# and every later step of the same job is skipped — the control plane dispatches a continuation of -# the same plan. A step that would drain the node its own runner stands on issues the upgrade and -# pauses the same way (`aks_step=detached`) rather than waiting for its own eviction. -# -# Output contract (the ::hosting:: lines the plugin's OperatorOutput reads): -# aks_facts= facts, the whole plan input -# aks_gate=healthy a gate passed -# aks_step=already|done|detached|skipped|would-upgrade -# aks_upgrade=incomplete the run paused at its budget — continue it -# aks_upgrade=complete verify passed — the ONLY completion -# aks_versions= what verify measured -# -# Identity: the aks-ops lane signs in as `hosting-operator` (OIDC) and mints the admin kubeconfig -# per job. The az reads/writes here need Microsoft.ContainerService managedClusters read+write, -# agentPools read+write and upgradeProfiles read on the cluster ("Azure Kubernetes Service -# Contributor Role"); a missing grant is refused BY NAME below, never read as "nothing to upgrade". - -source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_common.sh" -# shellcheck disable=SC2034 # read by hosting::die in _common.sh, which shellcheck does not follow here -HOSTING_CMD="hosting-aks-upgrade" - -sub="${1:-}" -[ $# -gt 0 ] && shift -rg="" cluster="" version="" pool="" budget="2100" strict=false -scans=() -while [ $# -gt 0 ]; do - case "$1" in - --resource-group) rg="${2:-}"; shift 2 ;; - --cluster) cluster="${2:-}"; shift 2 ;; - --version) version="${2:-}"; shift 2 ;; - --pool) pool="${2:-}"; shift 2 ;; - --budget-seconds) budget="${2:-}"; shift 2 ;; - --scan) scans+=("${2:-}"); shift 2 ;; - --strict) strict=true; shift ;; - *) hosting::die "unknown argument '$1'" ;; - esac -done - -case "$sub" in - facts|gate|control-plane|node-pool|verify) ;; - *) hosting::die "unknown subcommand '${sub}' — facts, gate, control-plane, node-pool or verify" ;; -esac -hosting::need_flag resource-group "$rg" -hosting::need_flag cluster "$cluster" -hosting::safe_name resource-group "$rg" -hosting::safe_name cluster "$cluster" -[[ "$budget" =~ ^[0-9]+$ ]] || hosting::die "--budget-seconds '${budget}' is not a whole number of seconds" -if [ "$sub" = control-plane ] || [ "$sub" = node-pool ] || [ "$sub" = verify ]; then - hosting::need_flag version "$version" - # A Kubernetes version and nothing else: it lands on an az command line. - [[ "$version" =~ ^[0-9]{1,2}\.[0-9]{1,3}\.[0-9]{1,3}$ ]] \ - || hosting::die "--version '${version}' is not a Kubernetes version (MAJOR.MINOR.PATCH) — refusing" -fi -if [ "$sub" = node-pool ]; then - hosting::need_flag pool "$pool" - [[ "$pool" =~ ^[a-z][a-z0-9]{0,11}$ ]] \ - || hosting::die "--pool '${pool}' is not an AKS node pool name (lower-case letter, then letters/digits, at most 12) — refusing" -fi -command -v jq >/dev/null 2>&1 || hosting::die "jq is not installed — every answer read here is JSON" -command -v az >/dev/null 2>&1 || hosting::die "az is not installed — the cluster's versions and pools are ARM facts" - -poll="${HOSTING_UPGRADE_POLL_SECONDS:-30}" -state_dir="${HOSTING_UPGRADE_STATE:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}}" -paused_file="${state_dir}/hosting-aks-upgrade.paused" -started_file="${state_dir}/hosting-aks-upgrade.started" -now() { date -u +%s; } -# a > b for MAJOR.MINOR.PATCH, numerically (portable — no `sort -V`). -ver_gt() { - local IFS=. i - # shellcheck disable=SC2206 # the split on '.' is the point; both are validated versions - local a=($1) b=($2) - for i in 0 1 2; do - if (( 10#${a[i]:-0} > 10#${b[i]:-0} )); then return 0; fi - if (( 10#${a[i]:-0} < 10#${b[i]:-0} )); then return 1; fi - done - return 1 -} - -# The job budget is measured from the FIRST step of THIS job — a later step does not get a fresh one. -if [ ! -f "$started_file" ]; then now > "$started_file"; fi -deadline=$(( $(cat "$started_file") + budget )) - -# An earlier step of this job paused it: everything after is the continuation's, not ours. -if [ -f "$paused_file" ] && [ "$sub" != facts ]; then - hosting::log "skipped — an earlier step of this run paused it ($(cat "$paused_file")); the control plane continues the plan in a new run" - hosting::say aks_step skipped - exit 0 -fi - -pause() { - printf '%s\n' "$1" > "$paused_file" - hosting::log "PAUSED at the job budget: $1 — nothing failed; the control plane dispatches a continuation of the same plan, and every step already done is skipped there" - hosting::say aks_upgrade incomplete - exit 0 -} - -# ── reads ───────────────────────────────────────────────────────────────────────────────────────── -# Every read keeps its stderr: a Forbidden must come out as a REFUSED read naming the grant, never -# as an empty list read as "nothing to upgrade" (the denied-vs-absent rule, bin/_common.sh). -az_read() { # what… -- az args - local what="$1"; shift - local out err - err="$(mktemp)" - if ! out="$(az "$@" -o json 2>"$err")"; then - local e; e="$(head -3 "$err" | tr '\n' ' ')"; rm -f "$err" - case "$e" in - *AuthorizationFailed*|*Forbidden*|*"does not have authorization"*) - hosting::die "REFUSED, not absent: the signed-in identity may not read ${what} — nothing about it is known. Grant 'Azure Kubernetes Service Contributor Role' on the cluster to hosting-operator (Systemorph/Memex docs/aks-ops.md §3). az said: ${e}" ;; - *) hosting::die "could not read ${what}: ${e}" ;; - esac - fi - rm -f "$err" - printf '%s' "$out" -} -kube_read() { # what… -- kubectl args - local what="$1"; shift - local out err - err="$(mktemp)" - if ! out="$(kubectl "$@" 2>"$err")"; then - local e; e="$(head -3 "$err" | tr '\n' ' ')"; rm -f "$err" - case "$e" in - *Forbidden*|*forbidden*) hosting::die "REFUSED, not absent: this identity may not read ${what} — nothing about it is known. kubectl said: ${e}" ;; - *) hosting::die "could not read ${what}: ${e}" ;; - esac - fi - rm -f "$err" - printf '%s' "$out" -} - -cluster_json() { - az_read "cluster ${cluster}" aks show -g "$rg" -n "$cluster" | jq -c '{ - name, resourceGroup, location, - kubernetesVersion: (.currentKubernetesVersion // .kubernetesVersion), - provisioningState, powerState: (.powerState.code // null), - tier: (.sku.tier // null), supportPlan: (.supportPlan // null), - autoUpgradeChannel: (.autoUpgradeProfile.upgradeChannel // "none"), - nodeOsUpgradeChannel: (.autoUpgradeProfile.nodeOsUpgradeChannel // null)}' -} -pools_json() { - az_read "the node pools of ${cluster}" aks nodepool list -g "$rg" --cluster-name "$cluster" | jq -c '[.[] | { - name, mode, count: (.count // 0), minCount, maxCount, autoscaling: (.enableAutoScaling // false), - version: (.currentOrchestratorVersion // .orchestratorVersion), - provisioningState, powerState: (.powerState.code // null), - maxSurge: (.upgradeSettings.maxSurge // null)}] | sort_by(.name)' -} -nodes_json() { - kube_read "nodes" get nodes -o json | jq -c '[.items[] | { - name: .metadata.name, - pool: (.metadata.labels["kubernetes.azure.com/agentpool"] // .metadata.labels.agentpool // null), - kubeletVersion: .status.nodeInfo.kubeletVersion, - ready: ([.status.conditions[]? | select(.type == "Ready") | .status == "True"] | any), - unschedulable: (.spec.unschedulable // false)}] | sort_by(.name)' -} -health_of() { # host → HTTP status of https://host/health (000 when nothing answered) - local code - code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 20 "https://$1/health" 2>/dev/null)" || true - [[ "$code" =~ ^[0-9]{3}$ ]] || code="000" - printf '%s' "$code" -} -# Every portal Deployment on the cluster (the chart names it memex-portal-deployment), its template -# image against what its pods SERVE, and its hosts' /health. -portals_json() { - local nodes="$1" deployments ns rows="[]" deployment selector pods ingress hosts health h code - deployments="$(kube_read "portal deployments" get deployments --all-namespaces -o json \ - | jq -c '[.items[] | select(.metadata.name == "memex-portal-deployment")]')" - for ns in $(printf '%s' "$deployments" | jq -r '.[].metadata.namespace' | sort -u); do - hosting::safe_name namespace "$ns" - deployment="$(printf '%s' "$deployments" | jq -c --arg ns "$ns" '.[] | select(.metadata.namespace == $ns)')" - selector="$(printf '%s' "$deployment" | jq -r '.spec.selector.matchLabels // {} | to_entries | map("\(.key)=\(.value)") | join(",")')" - [ -n "$selector" ] || hosting::die "deployment ${ns}/memex-portal-deployment has no matchLabels selector — its pods cannot be told from anything else's" - pods="$(kube_read "the portal pods in ${ns}" -n "$ns" get pods -l "$selector" -o json)" - ingress="$(kube_read "the ingresses in ${ns}" -n "$ns" get ingress -o json)" - hosts="$(printf '%s' "$ingress" | jq -r '[.items[].spec.rules[]?.host // empty] | unique | .[]')" - health="[]" - for h in $hosts; do - hosting::safe_host host "$h" - code="$(health_of "$h")" - health="$(jq -c --arg h "$h" --arg c "$code" '. + [{host: $h, status: ($c | tonumber)}]' <<<"$health")" - done - rows="$(jq -c --argjson d "$deployment" --argjson p "$pods" --argjson n "$nodes" --argjson hl "$health" ' - def portal(c): ([c[]? | select(.name == "memex-portal")][0] // c[0]); - ($n | map({(.name): .pool}) | add // {}) as $poolOf - | ($d.spec.template.spec.containers | portal(.)) as $tc - | . + [{ - namespace: $d.metadata.namespace, deployment: $d.metadata.name, - templateImage: $tc.image, - replicas: ($d.spec.replicas // 1), updated: ($d.status.updatedReplicas // 0), - ready: ($d.status.readyReplicas // 0), available: ($d.status.availableReplicas // 0), - generation: ($d.metadata.generation // 0), observedGeneration: ($d.status.observedGeneration // 0), - progressDeadlineExceeded: ([$d.status.conditions[]? | select(.type == "Progressing" and .reason == "ProgressDeadlineExceeded")] | length > 0), - startupBudgetSeconds: (if $tc.startupProbe then (($tc.startupProbe.periodSeconds // 10) * ($tc.startupProbe.failureThreshold // 3)) else null end), - health: $hl, - pods: [$p.items[] | { - name: .metadata.name, - image: (.spec.containers | portal(.)).image, - ready: ([.status.conditions[]? | select(.type == "Ready") | .status == "True"] | any), - terminating: (.metadata.deletionTimestamp != null), - node: (.spec.nodeName // null), pool: $poolOf[.spec.nodeName // ""], - ageSeconds: (if .status.startTime then ([0, ((now - (.status.startTime | fromdateiso8601)) | floor)] | max) else 0 end)}] - }]' <<<"$rows")" - done - printf '%s' "$rows" -} -pdbs_json() { - kube_read "pod disruption budgets" get pdb --all-namespaces -o json | jq -c '[.items[] | { - namespace: .metadata.namespace, name: .metadata.name, - disruptionsAllowed: (.status.disruptionsAllowed // 0), expectedPods: (.status.expectedPods // 0)}]' -} -# The API server's own count of deprecated APIs requested since it started, with the release each -# is removed in — the authority for what the cluster's workloads actually call. An unreadable -# metric is carried as UNREAD, never as "none in use". -deprecated_json() { - local metrics err e - err="$(mktemp)" - if ! metrics="$(kubectl get --raw /metrics 2>"$err")"; then - e="$(head -2 "$err" | tr '\n' ' ')"; rm -f "$err" - jq -cn --arg e "$e" '{read: false, error: $e, items: []}' - return 0 - fi - rm -f "$err" - printf '%s\n' "$metrics" | grep '^apiserver_requested_deprecated_apis{' | jq -R -s -c ' - [split("\n")[] | select(length > 0) - | ([match("([a-z_]+)=\"([^\"]*)\""; "g") | {(.captures[0].string): .captures[1].string}] | add) as $l - | {group: ($l.group // ""), version: ($l.version // ""), resource: ($l.resource // ""), - removedRelease: ($l.removed_release // "")}] | unique | {read: true, error: null, items: .}' -} -# apiVersion+kind of every manifest under the scanned directories — the charts, the operator -# manifests, the estate's overlays. A templated apiVersion ({{ … }}) is skipped: it is decided at -# render time, and the live metric above covers what the cluster actually receives. -manifests_json() { - local dir file lines="" line - for dir in "${scans[@]+"${scans[@]}"}"; do - [ -d "$dir" ] || hosting::die "--scan '${dir}' is not a directory — a scan of nothing is not a clean scan" - while IFS= read -r file; do - while IFS= read -r line; do lines+="${file}"$'\t'"${line}"$'\n'; done \ - < <(grep -E '^(apiVersion|kind):' "$file" || true) - done < <(find "$dir" -type f \( -name '*.yaml' -o -name '*.yml' \) | sort) - done - printf '%s' "$lines" | jq -R -s -c ' - [split("\n")[] | select(length > 0) | split("\t") | {file: .[0], line: .[1]}] - | reduce .[] as $r ({out: [], pending: null}; - if ($r.line | startswith("apiVersion:")) then - .pending = {file: $r.file, apiVersion: ($r.line | sub("^apiVersion:\\s*"; "") | gsub("[\"'"'"' ]"; ""))} - elif (.pending != null and .pending.file == $r.file) then - .out += [.pending + {kind: ($r.line | sub("^kind:\\s*"; "") | gsub("[\"'"'"' ]"; ""))}] | .pending = null - else . end) - | .out | map(select((.apiVersion | test("\\{\\{")) | not)) | group_by(.apiVersion + " " + .kind) | map(.[0])' -} -runner_json() { # the pool this job's own runner pod stands on, when it runs inside the cluster - local me="${HOSTNAME:-}" found - [ -n "$me" ] || { printf 'null'; return 0; } - hosting::safe_name pod "$me" - found="$(kube_read "this runner's own pod" get pods --all-namespaces --field-selector "metadata.name=${me}" -o json \ - | jq -c '.items[0] // null | if . == null then null else {pod: .metadata.name, node: (.spec.nodeName // null)} end')" - printf '%s' "$found" -} -runner_pool() { # nodes runner → pool name or "" - jq -r --argjson r "$2" 'if $r == null then "" else ([.[] | select(.name == $r.node) | .pool][0] // "") end' <<<"$1" -} - -# ── the gate ────────────────────────────────────────────────────────────────────────────────────── -# Prints one line per problem, each prefixed WAIT (may settle inside the budget) or FAIL (Kubernetes -# itself has given up on it). Empty = healthy. -gate_problems() { # cluster pools nodes portals - jq -rn --argjson c "$1" --argjson pl "$2" --argjson n "$3" --argjson p "$4" ' - ( if $c.provisioningState != "Succeeded" then ["WAIT cluster \($c.name) is \($c.provisioningState) — an operation is in flight"] else [] end ) - + [ $pl[] | select(.provisioningState != "Succeeded") | "WAIT node pool \(.name) is \(.provisioningState)" ] - + [ $n[] | select(.ready | not) | "WAIT node \(.name) (pool \(.pool)) is NOT Ready" ] - + [ $n[] | select(.unschedulable) | "WAIT node \(.name) (pool \(.pool)) is cordoned — a drain is in progress" ] - + [ $p[] | . as $d - | ([.pods[] | select(.terminating | not)]) as $live - | ([$live[] | .image] | unique) as $serving - | ( if .progressDeadlineExceeded then - ["FAIL \(.namespace)/\(.deployment): Kubernetes reports ProgressDeadlineExceeded — the roll to \(.templateImage) has stopped progressing"] - else [] end ) - + ( [$live[] | select((.ready | not) and ($d.startupBudgetSeconds != null) and (.ageSeconds > $d.startupBudgetSeconds)) - | "FAIL \($d.namespace)/\($d.deployment): pod \(.name) (\(.image)) is not Ready after \(.ageSeconds)s, past its own startup budget of \($d.startupBudgetSeconds)s"] ) - + ( if ($serving != [$d.templateImage]) or ($d.updated < $d.replicas) or ($d.observedGeneration < $d.generation) then - ["WAIT STUCK ROLL \($d.namespace)/\($d.deployment): the template names \($d.templateImage) but the pods serve \($serving | join(", ")) (\($d.updated)/\($d.replicas) updated) — a drain would recreate them from the template"] - elif $d.ready < $d.replicas then - ["WAIT \($d.namespace)/\($d.deployment): \($d.ready)/\($d.replicas) pods Ready"] - else [] end ) - + ( [$d.health[] | select(.status != 200) as $h - | if ([$live[] | select(.ready)] | length) == $d.replicas and ([$live[] | .ageSeconds] | min // 0) > 600 then - "FAIL \($d.namespace): https://\($h.host)/health answers \($h.status) although every pod has been Ready for over 10 minutes" - else "WAIT \($d.namespace): https://\($h.host)/health answers \($h.status)" end] ) - | .[] ] - | .[]' -} - -run_gate() { - local c pl n p problems fatal - while :; do - c="$(cluster_json)"; pl="$(pools_json)"; n="$(nodes_json)"; p="$(portals_json "$n")" - problems="$(gate_problems "$c" "$pl" "$n" "$p")" - if [ -z "$problems" ]; then - hosting::log "healthy: $(jq -r 'length' <<<"$p") portal(s) converged and answering /health 200; $(jq -r 'length' <<<"$n") node(s) Ready and schedulable; no operation in flight" - hosting::say aks_gate healthy - return 0 - fi - printf '%s\n' "$problems" | sed 's/^/ /' - fatal="$(printf '%s\n' "$problems" | grep '^FAIL' || true)" - [ -z "$fatal" ] || hosting::die "the health gate FAILED — Kubernetes itself has given up on: $(printf '%s' "$fatal" | sed 's/^FAIL //' | tr '\n' ';')" - if [ "$strict" = true ]; then - hosting::die "refused before anything changed — the cluster is not in a state to upgrade: $(printf '%s' "$problems" | sed 's/^WAIT //' | tr '\n' ';')" - fi - [ "$(now)" -lt "$deadline" ] || pause "the health gate is still waiting: $(printf '%s' "$problems" | head -1 | sed 's/^WAIT //')" - sleep "$poll" - done -} - -case "$sub" in - facts) - c="$(cluster_json)" - location="$(jq -r '.location' <<<"$c")"; hosting::safe_name location "$location" - current="$(jq -r '.kubernetesVersion' <<<"$c")" - minor="$(printf '%s' "$current" | cut -d. -f2)" - upgrades="$(az_read "the upgrades offered to ${cluster}" aks get-upgrades -g "$rg" -n "$cluster" \ - | jq -c '[.controlPlaneProfile.upgrades[]? | select((.isPreview // false) | not) | .kubernetesVersion] | unique | sort_by(split(".") | map(tonumber))')" - # The REGION's version graph is what chains a second minor and names the LTS offer. It is a - # subscription-scope read (locations/kubernetesVersions), which a cluster-scoped grant does not - # carry — so an unreadable graph is carried as UNREAD with its error (the plan then chains no - # further than THIS cluster's own get-upgrades, and says so), never as "nothing offered". - verr="$(mktemp)" versions_read=true versions_error="" - if ! vraw="$(az aks get-versions -l "$location" -o json 2>"$verr")"; then - versions_read=false; versions_error="$(head -2 "$verr" | tr '\n' ' ')"; vraw='{"values":[]}' - fi - rm -f "$verr" - versions="$(jq -c --argjson m "${minor:-0}" ' - [.values[]? | . as $v | ($v.patchVersions // {}) | to_entries[] - | {version: .key, upgrades: (.value.upgrades // []), supportPlans: ($v.capabilities.supportPlan // []), preview: ($v.isPreview // false)} - | select((.version | split(".")[1] | tonumber) >= $m)] | sort_by(.version | split(".") | map(tonumber))' <<<"$vraw")" - pl="$(pools_json)"; n="$(nodes_json)"; p="$(portals_json "$n")"; pdbs="$(pdbs_json)" - dep="$(deprecated_json)"; man="$(manifests_json)"; runner="$(runner_json)" - facts="$(jq -cn --argjson c "$c" --argjson u "$upgrades" --argjson v "$versions" --argjson pl "$pl" \ - --argjson n "$n" --argjson p "$p" --argjson pdb "$pdbs" --argjson d "$dep" --argjson m "$man" --argjson r "$runner" \ - --argjson vr "$versions_read" --arg ve "$versions_error" \ - '{schema: 1, readAt: (now | todate), cluster: $c, controlPlaneUpgrades: $u, versions: $v, - versionsRead: $vr, versionsError: (if $ve == "" then null else $ve end), pools: $pl, - nodes: $n, portals: $p, pdbs: $pdb, deprecatedApis: $d, manifestApis: $m, runner: $r}')" - hosting::log "cluster ${cluster}: Kubernetes ${current} ($(jq -r '.provisioningState' <<<"$c")), tier $(jq -r '.tier' <<<"$c"), support plan $(jq -r '.supportPlan' <<<"$c")" - hosting::log "control-plane upgrades offered: $(jq -r 'join(", ")' <<<"$upgrades")" - jq -r '.[] | " pool \(.name): \(.mode), \(.count) node(s), \(.version), \(.provisioningState), maxSurge \(.maxSurge // "not set")"' <<<"$pl" - jq -r '.[] | " portal \(.namespace): template \(.templateImage); serving \([.pods[] | select(.terminating | not) | .image] | unique | join(", ")); \(.ready)/\(.replicas) Ready"' <<<"$p" - hosting::log "deprecated-API metric read: $(jq -r '.read' <<<"$dep"), entries: $(jq -r '.items | length' <<<"$dep"); manifests scanned: $(jq -r 'length' <<<"$man") apiVersion/kind pair(s)" - hosting::say aks_facts "$(printf '%s' "$facts" | base64 | tr -d '\n')" - ;; - - gate) - run_gate - ;; - - control-plane) - issued=false - while :; do - c="$(cluster_json)" - v="$(jq -r '.kubernetesVersion' <<<"$c")"; st="$(jq -r '.provisioningState' <<<"$c")" - # A continuation re-runs the plan from the top: a hop an earlier run already went PAST is done, - # and issuing it would ask AKS for a downgrade. - if [ "$st" = Succeeded ] && [ "$issued" = false ] && ver_gt "$v" "$version"; then - hosting::log "control plane of ${cluster} is at ${v}, already past this hop to ${version} — nothing to do" - hosting::say aks_step already; exit 0 - fi - if [ "$v" = "$version" ] && [ "$st" = Succeeded ]; then - if [ "$issued" = true ]; then - hosting::log "control plane of ${cluster} is at ${version}"; hosting::say aks_step "done" - else - hosting::log "control plane of ${cluster} is already at ${version} — nothing to do"; hosting::say aks_step already - fi - exit 0 - fi - case "$st" in - Failed|Canceled) hosting::die "the control plane of ${cluster} is ${st} at ${v} — AKS ended the operation; read 'az aks show' / the activity log before anything else runs" ;; - Succeeded) - [ "$issued" = false ] || hosting::die "the control plane reports Succeeded at ${v}, not ${version}, after the upgrade was issued — AKS did not do what was asked" - hosting::do az aks upgrade -g "$rg" -n "$cluster" --kubernetes-version "$version" --control-plane-only --yes --no-wait \ - || hosting::die "az aks upgrade --control-plane-only to ${version} was refused (see above)" - issued=true - if hosting::dry; then hosting::say aks_step would-upgrade; exit 0; fi - ;; - *) hosting::log "control plane of ${cluster}: ${v}, ${st} — waiting" ;; - esac - [ "$(now)" -lt "$deadline" ] || pause "control plane of ${cluster} still ${st} on its way to ${version}" - sleep "$poll" - done - ;; - - node-pool) - issued=false - rp="" - while :; do - pj="$(az_read "node pool ${pool}" aks nodepool show -g "$rg" --cluster-name "$cluster" -n "$pool")" - v="$(jq -r '.currentOrchestratorVersion // .orchestratorVersion' <<<"$pj")"; st="$(jq -r '.provisioningState' <<<"$pj")" - if [ "$st" = Succeeded ] && [ "$issued" = false ] && ver_gt "$v" "$version"; then - hosting::log "node pool ${pool} is at ${v}, already past ${version} — nothing to do" - hosting::say aks_step already; exit 0 - fi - if [ "$v" = "$version" ] && [ "$st" = Succeeded ]; then - if [ "$issued" = true ]; then - hosting::log "node pool ${pool} is at ${version}"; hosting::say aks_step "done" - else - hosting::log "node pool ${pool} is already at ${version} — nothing to do"; hosting::say aks_step already - fi - exit 0 - fi - case "$st" in - Failed|Canceled) hosting::die "node pool ${pool} is ${st} at ${v} — AKS ended the operation; read 'az aks nodepool show' / the activity log before anything else runs" ;; - Succeeded) - [ "$issued" = false ] || hosting::die "node pool ${pool} reports Succeeded at ${v}, not ${version}, after the upgrade was issued — AKS did not do what was asked" - if [ -z "$rp" ]; then rp="$(runner_pool "$(nodes_json)" "$(runner_json)")"; fi - hosting::do az aks nodepool upgrade -g "$rg" --cluster-name "$cluster" -n "$pool" --kubernetes-version "$version" --yes --no-wait \ - || hosting::die "az aks nodepool upgrade of ${pool} to ${version} was refused (see above)" - issued=true - if hosting::dry; then hosting::say aks_step would-upgrade; exit 0; fi - if [ "$rp" = "$pool" ]; then - hosting::say aks_step detached - pause "node pool ${pool} hosts this runner — the upgrade was issued and is not waited on from a node it will drain" - fi - ;; - *) hosting::log "node pool ${pool}: ${v}, ${st} — waiting" ;; - esac - [ "$(now)" -lt "$deadline" ] || pause "node pool ${pool} still ${st} on its way to ${version}" - sleep "$poll" - done - ;; - - verify) - run_gate - c="$(cluster_json)"; pl="$(pools_json)"; n="$(nodes_json)" - off="$(jq -rn --arg v "$version" --argjson c "$c" --argjson pl "$pl" --argjson n "$n" ' - ( if $c.kubernetesVersion != $v then ["control plane at \($c.kubernetesVersion)"] else [] end ) - + [ $pl[] | select(.version != $v) | "node pool \(.name) at \(.version)" ] - + [ $n[] | select(.kubeletVersion != ("v" + $v)) | "node \(.name) runs kubelet \(.kubeletVersion)" ] - | .[]')" - [ -z "$off" ] || hosting::die "NOT at ${version}: $(printf '%s' "$off" | tr '\n' ';') — the upgrade is not complete" - hosting::say aks_versions "$(jq -rn --argjson c "$c" --argjson pl "$pl" '"cp=\($c.kubernetesVersion)" + ([$pl[] | ";\(.name)=\(.version)"] | join(""))')" - hosting::log "cluster ${cluster}: control plane, every pool and every node at ${version}, and healthy" - hosting::say aks_upgrade complete - ;; -esac diff --git a/deploy/aks/operator/bin/hosting-artifact-store-bind b/deploy/aks/operator/bin/hosting-artifact-store-bind deleted file mode 100755 index 6e17989278..0000000000 --- a/deploy/aks/operator/bin/hosting-artifact-store-bind +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env bash -# hosting-artifact-store-bind --namespace --claim --source-namespace --source-claim -# -# Bind the environment's SHARED ARTIFACT STORE into one consumer namespace, READ-ONLY -# (Doc/Architecture/SharedArtifactStore). The store is ONE Azure Files share the control instance's -# record declares and its chart creates (the owner's claim, /). A PVC is -# namespaced, so every portal namespace that reads the store needs its own claim on that same share: -# a STATIC PersistentVolume pointing at the owner's share, and a claim in bound to it — the shape -# the CI platform volume has run since 2026-09-12 (Systemorph/Memex -# deployments/aks/ci-runners/ci-platform-runner-volumes.yaml). -# -# Output contract (the ::hosting:: lines the mesh reads): -# artifact_store_share=/ the share the binding reads, read off the owner's PV -# artifact_store_bound=/ the consumer claim, Bound, read back -# artifact_store_created=0|1 whether this run created the PV/PVC (0 = already bound, verified) -# -# 🚨 THE SHARE IS READ OFF THE OWNER'S BOUND VOLUME, NEVER TYPED INTO A RECORD. A record that named an -# account and share could point a portal at any share in the subscription; this command derives both -# from the volume the owner's claim is bound to, and refuses an owner claim that is not Bound, not on -# the Azure Files CSI driver, or whose handle it cannot read. -# -# 🚨 READ-ONLY END TO END: accessMode ReadOnlyMany, csi.readOnly, the claim ReadOnlyMany (and the record -# mounts the volume readOnly: true — DeploymentPortalConfig.ArtifactStoreProblems refuses a writable -# consumer). The store has one owner and one producer; a portal that could write it is a second writer -# nobody sweeps. -# -# 🚨 IT NEVER DELETES AND NEVER REBINDS. The PV's reclaim policy is Retain, so removing a consumer's -# binding can never delete the share. A PV or claim of the binding's name that already exists is -# VERIFIED — same share, read-only — and a mismatch is a refusal naming both, never an overwrite: a -# claim pointing elsewhere is a human's question. -# -# Every volumeHandle is distinct on purpose (its 5th segment is the PV name): the kubelet stages a CSI -# volume under a hash of the handle, and two PVs with one handle on one node collide. - -source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_common.sh" -# shellcheck disable=SC2034 # read by hosting::die in _common.sh, which shellcheck does not follow here -HOSTING_CMD="hosting-artifact-store-bind" - -namespace="" claim="" source_namespace="" source_claim="" -attempts="${HOSTING_BIND_ATTEMPTS:-30}" interval="${HOSTING_BIND_INTERVAL:-5}" -# A value flag with no value would make `shift 2` fail without shifting — an endless loop. -needs_value() { [ $# -ge 2 ] || hosting::die "$1 needs a value"; } -while [ $# -gt 0 ]; do - case "$1" in - --namespace) needs_value "$@"; namespace="$2"; shift 2 ;; - --claim) needs_value "$@"; claim="$2"; shift 2 ;; - --source-namespace) needs_value "$@"; source_namespace="$2"; shift 2 ;; - --source-claim) needs_value "$@"; source_claim="$2"; shift 2 ;; - *) hosting::die "unknown argument '$1'" ;; - esac -done - -hosting::need_flag namespace "$namespace" -hosting::need_flag claim "$claim" -hosting::need_flag source-namespace "$source_namespace" -hosting::need_flag source-claim "$source_claim" -hosting::safe_name namespace "$namespace" -hosting::safe_name claim "$claim" -hosting::safe_name source-namespace "$source_namespace" -hosting::safe_name source-claim "$source_claim" -[[ "$attempts" =~ ^[0-9]+$ && "$interval" =~ ^[0-9]+$ ]] || hosting::die "a numeric setting is not a whole number — refusing" -[ "$namespace" != "$source_namespace" ] \ - || hosting::die "the consumer namespace IS the owner's namespace (${namespace}) — the owner mounts its own claim ${source_claim} writable; it needs no read-only binding" -command -v jq >/dev/null 2>&1 || hosting::die "jq is not installed in this image — the volumes are JSON and cannot be read without it" - -pv_name="artifact-store-${namespace}" - -hosting::step "Bind the shared artifact store" - -# ── the owner's claim and the share behind it ─────────────────────────────────────────────────── -hosting::probe output kubectl -n "$source_namespace" get pvc "$source_claim" -o json -case $? in - 2) hosting::die_refused "PersistentVolumeClaim ${source_claim} in ${source_namespace}, the owner's artifact store" ;; - 1) hosting::die "the owner's artifact store ${source_namespace}/${source_claim} does not exist (${HOSTING_PROBE_ERR:-nothing listed}). It is created by the OWNER's record (the control instance's volumes[], create: true) on its next roll or Reconcile — bind consumers only after it is Bound." ;; -esac -src_phase="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.status.phase // ""')" -src_volume="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.spec.volumeName // ""')" -[ "$src_phase" = "Bound" ] && [ -n "$src_volume" ] \ - || hosting::die "the owner's artifact store ${source_namespace}/${source_claim} is '${src_phase:-}', not Bound — there is no share behind it to bind yet" -hosting::safe_name source-volume "$src_volume" - -hosting::probe output kubectl get pv "$src_volume" -o json -case $? in - 2) hosting::die_refused "PersistentVolume ${src_volume}, behind the owner's artifact store" ;; - 1) hosting::die "cannot read PersistentVolume ${src_volume} behind ${source_namespace}/${source_claim} (${HOSTING_PROBE_ERR:-nothing listed})" ;; -esac -src_pv="$HOSTING_PROBE_OUT" -driver="$(printf '%s' "$src_pv" | jq -r '.spec.csi.driver // ""')" -handle="$(printf '%s' "$src_pv" | jq -r '.spec.csi.volumeHandle // ""')" -capacity="$(printf '%s' "$src_pv" | jq -r '.spec.capacity.storage // ""')" -[ "$driver" = "file.csi.azure.com" ] \ - || hosting::die "the owner's store ${src_volume} is on driver '${driver:-}', not the Azure Files CSI driver — a read-only static binding of it is not something this command knows how to make" -# rg#account#share#…#…#secretnamespace — what the azurefile provisioner writes. The PV's explicit -# attributes win over the handle where it states them (a statically provisioned owner may carry a -# handle that is not in the provisioner's shape), exactly as hosting-pv-purge reads them. -IFS='#' read -r h_rg h_account h_share _ _ h_secretns <<< "$handle" -attr_account="$(printf '%s' "$src_pv" | jq -r '.spec.csi.volumeAttributes.storageAccount // ""')" -attr_share="$(printf '%s' "$src_pv" | jq -r '.spec.csi.volumeAttributes.shareName // ""')" -attr_rg="$(printf '%s' "$src_pv" | jq -r '.spec.csi.volumeAttributes.resourceGroup // ""')" -h_account="${attr_account:-$h_account}"; h_share="${attr_share:-$h_share}"; h_rg="${attr_rg:-$h_rg}" -[[ "$h_rg" =~ ^[A-Za-z0-9._()-]+$ && "$h_account" =~ ^[a-z0-9]{3,24}$ && "$h_share" =~ ^[a-z0-9][a-z0-9-]{1,62}$ ]] \ - || hosting::die "cannot read the owner's share from volumeHandle '${handle}' (expected ###…) — refusing to bind a share it could not name" -attr_secretns="$(printf '%s' "$src_pv" | jq -r '.spec.csi.volumeAttributes.secretnamespace // ""')" -secretns="${h_secretns:-${attr_secretns:-$source_namespace}}" -hosting::safe_name secret-namespace "$secretns" -[ -n "$capacity" ] || hosting::die "the owner's store ${src_volume} states no capacity — refusing to bind a volume the API did not size" -share="${h_account}/${h_share}" -hosting::log "owner ${source_namespace}/${source_claim} → ${src_volume} (share ${share}, ${capacity})" -hosting::log "consumer ${namespace}/${claim} via PersistentVolume ${pv_name}, READ-ONLY" - -# The key Secret the node stages the share with: the OWNER's PV's own nodeStageSecretRef when it names -# one (a share provisioned with a custom Secret), else the provisioner's convention in its namespace. -src_secret_name="$(printf '%s' "$src_pv" | jq -r '.spec.csi.nodeStageSecretRef.name // ""')" -src_secret_ns="$(printf '%s' "$src_pv" | jq -r '.spec.csi.nodeStageSecretRef.namespace // ""')" -secret_name="${src_secret_name:-azure-storage-account-${h_account}-secret}" -secretns="${src_secret_ns:-$secretns}" -hosting::safe_name secret "$secret_name" -hosting::safe_name secret-namespace "$secretns" -new_handle="${h_rg}#${h_account}#${h_share}##${pv_name}#${secretns}" - -# ── an existing binding is VERIFIED, never overwritten ────────────────────────────────────────── -created=0 -hosting::probe output kubectl get pv "$pv_name" -o json -case $? in - 2) hosting::die_refused "PersistentVolume ${pv_name}" ;; - 0) - have_share="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '"\(.spec.csi.volumeAttributes.storageAccount // "")/\(.spec.csi.volumeAttributes.shareName // "")"')" - have_ro="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.spec.csi.readOnly // false')" - have_ref="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '"\(.spec.claimRef.namespace // "")/\(.spec.claimRef.name // "")"')" - have_policy="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.spec.persistentVolumeReclaimPolicy // ""')" - have_label="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.metadata.labels["meshweaver.io/purpose"] // ""')" - have_handle="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.spec.csi.volumeHandle // ""')" - have_secret="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '"\(.spec.csi.nodeStageSecretRef.name // "")/\(.spec.csi.nodeStageSecretRef.namespace // "")"')" - [ "$have_share" = "$share" ] \ - || hosting::die "PersistentVolume ${pv_name} already exists and points at ${have_share}, not the owner's ${share} — refusing to rebind it. Whether the old binding is still read is a human's question; remove it deliberately, then re-run." - [ "$have_ro" = "true" ] \ - || hosting::die "PersistentVolume ${pv_name} already exists and is NOT read-only — a consumer binding that can write the store is a second writer; remove it deliberately, then re-run" - [ "$have_ref" = "${namespace}/${claim}" ] \ - || hosting::die "PersistentVolume ${pv_name} already exists for claim ${have_ref}, not ${namespace}/${claim} — refusing to rebind it" - # 🚨 A binding whose reclaim policy is not Retain would hand the SHARED share to the provisioner's - # delete the moment its claim goes, and one without the binding label is invisible to the teardown - # guard in hosting-pv-purge. Neither is accepted as "already bound". - [ "$have_policy" = "Retain" ] \ - || hosting::die "PersistentVolume ${pv_name} already exists with reclaim policy '${have_policy:-unset}', not Retain — removing its claim could delete the shared store; remove it deliberately, then re-run" - # The handle must be THIS binding's own (its 5th segment the PV name): two PVs sharing a handle - # collide in the kubelet's staging on one node. - [ "$have_handle" = "$new_handle" ] \ - || hosting::die "PersistentVolume ${pv_name} already exists with volumeHandle '${have_handle}', not this binding's own '${new_handle}' — refusing to accept a binding whose handle may collide with another; remove it deliberately, then re-run" - # The key it stages the share with must be the owner's CURRENT one: a binding left with a stale or - # foreign Secret mounts nothing (or the wrong account) on the next node that stages it. - [ "$have_secret" = "${secret_name}/${secretns}" ] \ - || hosting::die "PersistentVolume ${pv_name} already exists staging with Secret '${have_secret}', not the owner's '${secret_name}/${secretns}' — refusing to accept stale staging credentials; remove it deliberately, then re-run" - [ "$have_label" = "artifact-store" ] \ - || hosting::die "PersistentVolume ${pv_name} already exists without the label meshweaver.io/purpose=artifact-store — the teardown guard would not recognise it as a binding; remove it deliberately, then re-run" - hosting::log "PersistentVolume ${pv_name} exists — same share, read-only, for ${namespace}/${claim}" - ;; - 1) - pv_manifest="$(jq -n --arg name "$pv_name" --arg capacity "$capacity" --arg ns "$namespace" --arg claim "$claim" \ - --arg handle "$new_handle" --arg rg "$h_rg" --arg account "$h_account" --arg share "$h_share" \ - --arg secretns "$secretns" --arg secret "$secret_name" --arg ons "$source_namespace" --arg oclaim "$source_claim" ' - { apiVersion: "v1", kind: "PersistentVolume", - metadata: { name: $name, - labels: { "app.kubernetes.io/managed-by": "hosting-operator", "meshweaver.io/purpose": "artifact-store" }, - annotations: { "meshweaver.io/artifact-store-owner": "\($ons)/\($oclaim)" } }, - spec: { capacity: { storage: $capacity }, accessModes: ["ReadOnlyMany"], - persistentVolumeReclaimPolicy: "Retain", storageClassName: "", - claimRef: { namespace: $ns, name: $claim }, - mountOptions: ["dir_mode=0755", "file_mode=0644", "uid=1654", "gid=1654", "mfsymlinks", "cache=strict", "actimeo=30", "nobrl", "nosharesock"], - csi: { driver: "file.csi.azure.com", readOnly: true, volumeHandle: $handle, - volumeAttributes: { resourceGroup: $rg, storageAccount: $account, shareName: $share, secretnamespace: $secretns }, - nodeStageSecretRef: { name: $secret, namespace: $secretns } } } }')" - # `create`, never `apply`: a PV that appeared since the read above must make this FAIL, not be - # patched into shape without the verification an existing binding gets. - hosting::do kubectl create -f - <<< "$pv_manifest" >&2 \ - || hosting::die "could not create PersistentVolume ${pv_name}" - created=1 - ;; -esac - -hosting::probe output kubectl -n "$namespace" get pvc "$claim" -o json -case $? in - 2) hosting::die_refused "PersistentVolumeClaim ${claim} in ${namespace}" ;; - 0) - have_volume="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.spec.volumeName // ""')" - [ "$have_volume" = "$pv_name" ] \ - || hosting::die "PersistentVolumeClaim ${namespace}/${claim} already exists and is bound to '${have_volume:-nothing}', not ${pv_name} — refusing to replace a claim a portal may be mounting. Choose another claim name on the record, or remove it deliberately." - ;; - 1) - pvc_manifest="$(jq -n --arg ns "$namespace" --arg claim "$claim" --arg pv "$pv_name" --arg capacity "$capacity" ' - { apiVersion: "v1", kind: "PersistentVolumeClaim", - metadata: { name: $claim, namespace: $ns, - labels: { "app.kubernetes.io/managed-by": "hosting-operator", "meshweaver.io/purpose": "artifact-store" } }, - spec: { accessModes: ["ReadOnlyMany"], storageClassName: "", volumeName: $pv, - resources: { requests: { storage: $capacity } } } }')" - hosting::do kubectl create -f - <<< "$pvc_manifest" >&2 \ - || hosting::die "could not create PersistentVolumeClaim ${namespace}/${claim}" - created=1 - ;; -esac - -if hosting::dry; then - hosting::say artifact_store_share "$share" - hosting::say artifact_store_created 0 - hosting::say artifact_store_bound dry-run - exit 0 -fi - -# ── read back: the claim must be Bound to THIS volume ────────────────────────────────────────── -n=0 -while :; do - hosting::probe output kubectl -n "$namespace" get pvc "$claim" -o json - case $? in - 2) hosting::die_refused "PersistentVolumeClaim ${claim} in ${namespace}" ;; - 0) - phase="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.status.phase // ""')" - bound_to="$(printf '%s' "$HOSTING_PROBE_OUT" | jq -r '.spec.volumeName // ""')" - if [ "$phase" = "Bound" ] && [ "$bound_to" = "$pv_name" ]; then break; fi - ;; - esac - n=$((n + 1)) - [ "$n" -lt "$attempts" ] \ - || hosting::die "PersistentVolumeClaim ${namespace}/${claim} is '${phase:-unread}' after ${n} reads — it did not bind to ${pv_name}" - sleep "$interval" -done - -hosting::log "bound ${namespace}/${claim} → ${pv_name} → ${share} (read-only)" -hosting::say artifact_store_share "$share" -hosting::say artifact_store_created "$created" -hosting::say artifact_store_bound "${namespace}/${claim}" diff --git a/deploy/aks/operator/bin/hosting-audit b/deploy/aks/operator/bin/hosting-audit deleted file mode 100755 index 9428b1b9df..0000000000 --- a/deploy/aks/operator/bin/hosting-audit +++ /dev/null @@ -1,201 +0,0 @@ -#!/usr/bin/env bash -# hosting-audit --namespace --release [--deployment memex-portal-deployment] [--container memex-portal] -# -# What lives ONLY on the cluster — configuration no repository renders? -# -# 🚨 WHY THIS EXISTS. On 2026-08-30 the memex portal crashed at boot because its Email section -# existed TWICE: the chart's ConfigMap defaults, and a cluster-only secret somebody had patched -# onto the Deployment as env with `kubectl set env`. .NET picks the winner at random per pod start. -# The audit run by hand afterwards found 25 env patches, 10 live-edited ConfigMap keys, sidecar -# containers and a volume on memex; 31 env patches and replicas/nodeSelector on memex-cloud; a -# CronJob, Deployments, Ingresses and PVCs the chart has never heard of in both namespaces; and a -# plain-text registry token in the pod spec (Systemorph/Memex#148). None of it was visible from any -# repository, because `helm upgrade` PRESERVES a live-edited ConfigMap key and never touches an -# object it does not own — drift of this class does not show up as a failed deploy, it shows up as -# an outage months later. The rule this enforces: NOTHING LIVES ONLY ON THE CLUSTER — every value is -# a typed field on the Hosting/Deployment record, rendered by the chart; secrets are Key Vault names. -# -# THE TRUTH is the helm manifest: `helm get manifest` + `helm get hooks`, turned into objects by -# `kubectl apply --dry-run=client -o json`. Everything live that is not in it, and every field of -# the portal Deployment / the chart's ConfigMaps that differs from it, is a finding. -# -# WHAT IS DETECTED (names only — a VALUE is never printed, because the point of the exercise is -# that some of them are secrets): -# * live objects in the namespace not in the manifest — owned objects (ReplicaSets, CronJob-spawned -# Jobs), helm release secrets, ServiceAccount tokens, CSI-synced secrets, kube-root-ca.crt and -# the `default` ServiceAccount excluded, since none of those is configuration anyone put there; -# * the portal Deployment's live-only env names (and manifest-only ones — a value the chart -# renders that the pod lacks is the same edit from the other side), envFrom, volumes, mounts, -# containers, pod annotations, and the pod-spec fields a `kubectl patch` reaches: replicas, -# nodeSelector, tolerations, affinity, serviceAccountName, terminationGracePeriodSeconds, the -# portal container's resources, lifecycle and startup-probe budget. Pod-spec DIFFS carry the -# two values (placement and sizing, not secrets) — except lifecycle hooks, whose string -# scalars are REDACTED to their shape: a hook holds command lines, and a command line can -# embed anything; -# * the chart's ConfigMaps: keys that exist only live, only in the manifest, or whose VALUE differs; -# * Secrets the live pod reads (envFrom, secretKeyRef, secret volumes) that are neither -# chart-managed nor CSI-synced; -# * entries whose NAME looks like a secret (token|secret|key|password) carried as a PLAIN value — -# inline env on the pod, or a key in a chart ConfigMap; -# * a PersistentVolumeClaim whose live capacity is BELOW the size the Deployment record declares -# for it (the record's volumes[] reach helm as the release's `persistence` values). The one -# record-vs-live comparison here, because the manifest cannot show it: the portal's claims on -# this fleet are not helm-managed, so a bigger `size` on the record changes nothing until -# hosting-pv-resize applies it — capacity is a record property, and this is what says when the -# cluster has not caught up (measured 2026-09-08: memex-data declared 16Gi, full at 3 MiB free). -# -# THE OUTPUT is the human summary plus the machine lines the mesh's Hosting plugin parses: -# ::hosting:: audit_verdict=clean|drift ::hosting:: audit_findings=N ::hosting:: audit= -# The JSON is the LiveOnlyConfigReport the Hosting/InstanceAction `Audit` verb records on the -# instance's Hosting/ConfigAudit node. The mesh re-derives the verdict from the lists — the lists -# are the evidence, the word is a convenience. -# -# 🚨 READ-ONLY, and it exits 0 on drift. Drift is a FINDING the report carries, not a failure of -# the run; the run fails only when it cannot measure — no release, no manifest, no live Deployment, -# or a kind it is not allowed to list. An audit that cannot read a kind must not report that kind as -# clean, so a Forbidden is fatal and names the ClusterRole to widen. A dry run audits for real: -# nothing here mutates, so there is nothing to rehearse. - -source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_common.sh" -# shellcheck disable=SC2034 # read by hosting::die in _common.sh, which shellcheck does not follow here -HOSTING_CMD="hosting-audit" - -AUDIT_JQ="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_audit.jq" - -namespace="" release="" deployment="memex-portal-deployment" container="memex-portal" -while [ $# -gt 0 ]; do - case "$1" in - --namespace) namespace="${2:-}"; shift 2 ;; - --release) release="${2:-}"; shift 2 ;; - --deployment) deployment="${2:-}"; shift 2 ;; - --container) container="${2:-}"; shift 2 ;; - *) hosting::die "unknown argument '$1'" ;; - esac -done - -hosting::need_flag namespace "$namespace" -hosting::need_flag release "$release" -hosting::safe_name namespace "$namespace" -hosting::safe_name release "$release" -hosting::safe_name deployment "$deployment" -hosting::safe_name container "$container" - -command -v jq >/dev/null 2>&1 || hosting::die "jq is not installed in this image — the audit's detection is a jq program, and without it nothing can be compared" -[ -f "$AUDIT_JQ" ] || hosting::die "the detection program ${AUDIT_JQ} is missing beside this script — the image was built without it" - -hosting::dry && hosting::log "dry run requested — the audit is read-only, so it runs for real" - -work="$(mktemp -d)" -trap 'hosting::on_exit; rm -rf "$work"' EXIT - -# ── 1. the release: which revision is being audited ───────────────────────────────────────────── -# `helm status -o json` is the whole release object — including `config`, the user-supplied values, -# which carry the Key Vault half. Only two fields leave this variable, and neither is a value. -status="$(hosting::read helm status "$release" -n "$namespace" -o json 2>"$work/err")" \ - || hosting::die "no helm release '${release}' in namespace '${namespace}' ($(tr -d '\n' < "$work/err")). An audit compares the cluster against a release's manifest; with no release there is nothing to compare against, and reporting the namespace as 'unmanaged' would be a confident wrong answer." -revision="$(printf '%s' "$status" | jq -r '.version // empty')" -chart="$(printf '%s' "$status" | jq -r '.chart.metadata.version // empty')" -# The THIRD thing that leaves the release object, and the only part of `config` that does: the -# record's volumes as HelmValues rendered them (claimName + size per volume — names and sizes, no -# value of anything). It is what a claim's live capacity is measured against below. -persistence="$(printf '%s' "$status" | jq -c '.config.persistence // {}')" -[ -n "$revision" ] || hosting::die "helm status answered for '${release}' but named no revision — refusing to audit against a release the output cannot identify" -hosting::log "release ${release} in ${namespace} — revision ${revision}${chart:+, chart ${chart}}" - -# ── 2. the manifest — the truth ───────────────────────────────────────────────────────────────── -# Hooks are included: the migration Job is a helm hook, and `helm get manifest` alone would list it -# as an object the chart has never heard of. -hosting::read helm get manifest "$release" -n "$namespace" > "$work/manifest.yaml" 2>"$work/err" \ - || hosting::die "could not read the manifest of release '${release}' ($(tr -d '\n' < "$work/err"))" -hosting::read helm get hooks "$release" -n "$namespace" > "$work/hooks.yaml" 2>"$work/err" \ - || hosting::die "could not read the hooks of release '${release}' ($(tr -d '\n' < "$work/err"))" -if ! cat "$work/manifest.yaml" "$work/hooks.yaml" \ - | kubectl apply --dry-run=client -o json -f - 2>"$work/err" \ - | jq -s '[.[] | if .kind == "List" then (.items // [])[] else . end]' > "$work/manifest.json"; then - hosting::die "could not turn the manifest into objects ($(tr -d '\n' < "$work/err")). kubectl needs the cluster's discovery to resolve every kind the chart renders — a CRD the chart uses that the cluster does not serve fails here, and that is a finding about the chart, not noise." -fi -managed="$(jq 'length' "$work/manifest.json")" -[ "$managed" -gt 0 ] \ - || hosting::die "the manifest of '${release}' rendered no objects — an audit against an empty manifest would report every live object as unmanaged, which is not a measurement" -hosting::log "manifest ${managed} chart-managed object(s)" - -# ── 3. the live namespace ─────────────────────────────────────────────────────────────────────── -# One `kubectl get ` per kind. A kind this cluster does not serve is SKIPPED and recorded in -# the report (nothing of that kind can exist); anything else — Forbidden above all — is fatal, -# because a kind the audit could not read is a kind it must not report as clean. -KINDS="deployment statefulset daemonset cronjob job configmap secret secretproviderclass ingress service serviceaccount role rolebinding poddisruptionbudget scaledobject horizontalpodautoscaler persistentvolumeclaim networkpolicy" -skipped="" -: > "$work/live.jsonl" -for kind in $KINDS; do - if hosting::read kubectl get "$kind" -n "$namespace" -o json > "$work/live-${kind}.json" 2>"$work/err"; then - jq -c '.items // []' "$work/live-${kind}.json" >> "$work/live.jsonl" \ - || hosting::die "kubectl's listing of ${kind} in ${namespace} is not JSON" - else - err="$(tr -d '\n' < "$work/err")" - case "$err" in - *"doesn't have a resource type"*|*"the server could not find the requested resource"*|*"no matches for kind"*) - skipped="${skipped} ${kind}" - hosting::log "kind ${kind} is not served by this cluster — skipped" ;; - *) - hosting::die "could not list ${kind} in ${namespace}: ${err}. An audit that cannot read a kind must not report it as clean — grant the hosting-operator ClusterRole get/list on it (deploy/aks/manifests/hosting-operator/operator-rbac.yaml) and re-run." ;; - esac - fi -done -jq -s 'add // []' "$work/live.jsonl" > "$work/live.json" \ - || hosting::die "could not combine the live listings" -skipped="${skipped# }" - -# ── 4. the detection ──────────────────────────────────────────────────────────────────────────── -if ! jq -n \ - --slurpfile manifest "$work/manifest.json" \ - --slurpfile live "$work/live.json" \ - --arg namespace "$namespace" --arg release "$release" \ - --arg deployment "$deployment" --arg container "$container" \ - --argjson revision "$revision" --arg chart "$chart" \ - --arg auditedAt "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" \ - --arg skipped "$skipped" \ - --argjson persistence "$persistence" \ - -f "$AUDIT_JQ" > "$work/report.json" 2>"$work/err"; then - hosting::die "the audit could not be computed: $(sed -e 's/^jq: error (at ): //' "$work/err" | tr -d '\n')" -fi - -# ── 5. the report — names, never values ───────────────────────────────────────────────────────── -verdict="$(jq -r '.verdict' "$work/report.json")" -count="$(jq -r '.findingCount' "$work/report.json")" - -say_list() { # say_list