diff --git a/.claude/skills/new-deployment/SKILL.md b/.claude/skills/new-deployment/SKILL.md index fda4fc23f5..6cf8e8117b 100644 --- a/.claude/skills/new-deployment/SKILL.md +++ b/.claude/skills/new-deployment/SKILL.md @@ -63,9 +63,12 @@ Two orderings are not negotiable, and both fail *silently* when you get them wro ## The steps -Full runbook with commands: **`docs/new-deployment.md` in `Systemorph/Memex`** (private). Generic -mechanism: [OnboardingNewEnvironment.md](../../../src/MeshWeaver.Documentation/Data/Architecture/OnboardingNewEnvironment.md) -and [DEPLOY-RUNBOOK.md](../../../deploy/aks/DEPLOY-RUNBOOK.md). +🚨 **The maintained runbook is `docs/new-deployment.md` in `Systemorph/Memex`** (private) β€” it is +authoritative wherever it and anything here disagree, and the table below is a map of the steps, +NOT a copy of them. Do not restate its commands here: two copies of a procedure drift, and the one +in the public repo is the one nobody updates. Generic mechanism (public): +[OnboardingNewEnvironment.md](../../../src/MeshWeaver.Documentation/Data/Architecture/OnboardingNewEnvironment.md). +Shared-platform bring-up (public sample): [deploy/aks/README.md](../../../deploy/aks/README.md). | # | Step | Gets it wrong how | |---|---|---| @@ -73,7 +76,7 @@ and [DEPLOY-RUNBOOK.md](../../../deploy/aks/DEPLOY-RUNBOOK.md). | 2 | `az postgres flexible-server db create -d ` on `memexaks-pg` | migrating? **reuse the source master key** or every `enc:` provider key is undecryptable | | 3 | DNS A-record β†’ the **shared** ingress IP | pointing at a per-env IP that doesn't exist | | 4 | Entra app (multi-tenant) + Key Vault secrets `-*` | invitation-only is the gate, not the audience | -| 5 | `deploy/aks/envs//` scaffold + `deploy.sh` | see traps below | +| 5 | Env-folder scaffold + `deploy.sh` β€” **in the PRIVATE ops repo**, not `deploy/aks/envs//` (folders moved out 2026-08-08/09, `a69959165`); `deploy/aks/envs/example/` here is only the shape to copy | see traps below | | 6 | **Plugins** β†’ `/plugins` skill | new deployment starts with none | | 7 | `tls.sh` | needs step 3 resolving publicly | | 8 | Register the GitHub Environment on `Systemorph/Memex` | otherwise it's untracked | diff --git a/.github/workflows/main-cd.yml b/.github/workflows/main-cd.yml index 69894023ce..598bc05d1b 100644 --- a/.github/workflows/main-cd.yml +++ b/.github/workflows/main-cd.yml @@ -1657,5 +1657,5 @@ jobs: # credentials β€” which is also why the old `az aks command invoke` deploy job was deleted: it # required a central service principal with per-cluster Azure RBAC (the `commandResults/read` # authorization failure on 2026-06-29), a model that cannot scale to installs prod doesn't manage. - # See SelfUpdate.md / DEPLOY-RUNBOOK.md for enabling self-update on an environment (the SA + RBAC are + # See deploy/aks/SELF-UPDATE.md / deploy/aks/README.md for enabling self-update on an environment (the SA + RBAC are # in the Helm chart; set selfUpdate.azureClientId and grant the UAMI AcrPull). diff --git a/AGENTS.md b/AGENTS.md index dc81b44eed..e9d2514ac7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -472,17 +472,18 @@ dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.cs dotnet publish memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj -c Release \ -t:PublishContainer -p:ContainerRegistry=meshweaver.azurecr.io \ -p:ContainerRepository=memex-migration -p:ContainerImageTag= -# Roll out (NS = memex): +# Roll out (NS = memex). 🚨 The MIGRATION is a Job, not a Deployment β€” see below. az aks command invoke -g -n --command "\ kubectl -n set image deployment/memex-portal-deployment memex-portal=meshweaver.azurecr.io/memex-portal-ai:; \ - kubectl -n set image deployment/memex-migration-deployment memex-migration=meshweaver.azurecr.io/memex-migration:; \ - kubectl -n rollout restart deployment/memex-migration-deployment deployment/memex-portal-deployment; \ + kubectl -n rollout restart deployment/memex-portal-deployment; \ kubectl -n rollout status deployment/memex-portal-deployment --timeout=300s" ``` -- **`deploy/aks/envs//deploy.sh` is first-time ENV SETUP only** (helm install + PVCs + KV SecretProviderClass + ingress + connection-string patch). Do NOT use it for a code update β€” it re-runs the whole chart and can reset live config. +- **An env's `deploy.sh` is first-time ENV SETUP only** (helm install + PVCs + KV SecretProviderClass + ingress + connection-string patch). Do NOT use it for a code update β€” it re-runs the whole chart and can reset live config. 🚨 **Env folders live in the PRIVATE `Systemorph/Memex` repo**, not `deploy/aks/envs//` β€” they moved out 2026-08-08/09 (`a69959165`) because their directory names are tenant identities; `deploy/aks/envs/example/` here is the reference template only. - **Don't run `tools/deploy.sh` or `aspire deploy` against the AKS cluster** β€” those are the *Container Apps* route (a different target), not a code-update path for AKS. -- `memex-migration` runs the migration then exits 0 and the Deployment restarts it (benign `CrashLoopBackOff`). Before declaring success, confirm its log shows `Database migration completed. Version: N` AND the portal serves (HTTP 200). +- 🚨 **The migration is a run-once `Job`, NOT a Deployment** (`deploy/helm/templates/memex-migration/job.yaml`: `kind: Job`, `restartPolicy: Never`, name embeds `.Release.Revision`, `ttlSecondsAfterFinished`). It runs on **`helm upgrade`**, not on an image-tag roll β€” so `kubectl set image` / `rollout restart` against a `memex-migration-deployment` is a command with no object: the chart does not define one, and any that exists in a live namespace is a cluster-only orphan of a chart revision that predates the Job (#1788). Do not resurrect it. A `MigrationWorkloadModelGuard` (test/MeshWeaver.Documentation.Test) fails the build if a command like that returns to this file, a doc or a deploy script. +- **A crash-looping migration pod is a FAILURE, not noise.** It used to be modelled as a Deployment, which restarted it forever after each clean exit β€” three prod namespaces sat at 50/53/38 restarts, each rebuilding `public.top_level_index` across every partition schema. Documenting that as "benign" is what made a real migration failure unreadable. With the Job form the pod runs once and stops, so `CrashLoopBackOff` on it now means exactly what it says. Before declaring a deploy successful, confirm the Job's log shows `Database migration completed. Version: N` AND the portal serves (HTTP 200) β€” `kubectl -n get jobs -l app.kubernetes.io/component=memex-migration` and `kubectl -n logs job/memex-migration-`. +- **The schema still has a hard gate.** `DbVersionGate` (a hosted service in `Memex.Portal.Distributed`) reads `admin.mesh_nodes.db_version` at startup and `StopApplication()`s with a `LogCritical` when it is below `DbVersion.Latest`. So a portal rolled ahead of its schema refuses to serve rather than serving a half-migrated database β€” loud, and the reason the ordering between the two workloads cannot silently invert. Routes + full reference: [Deployment.md](src/MeshWeaver.Documentation/Data/Architecture/Deployment.md) (index) Β· [DeploymentAKS.md](src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md) Β· [DeploymentContainerApps.md](src/MeshWeaver.Documentation/Data/Architecture/DeploymentContainerApps.md) diff --git a/Dockerfile.logwatcher b/Dockerfile.logwatcher index fb9e458ba8..a67cc2f5b3 100644 --- a/Dockerfile.logwatcher +++ b/Dockerfile.logwatcher @@ -6,7 +6,7 @@ # --file Dockerfile.logwatcher . # # NOT the sanctioned build path β€” that stays `dotnet publish -t:PublishContainer` -# (deploy/aks/DEPLOY-RUNBOOK.md Β§ 6b), which is what CI and the runbook use. This is tracked only +# (deploy/aks/README.md β†’ "Red-log ticketing"), which is what CI and the deployment sample use. This is tracked only # so the escape hatch exists when the local link to ACR cannot carry image layers: on 2026-08-08 the # SDK push wedged on repeated "Broken pipe" retries, and the deploy completed only by building # ACR-side from a source-only context. Build the context deliberately β€” a bare `.` from the repo diff --git a/content/ai/Skill/group.md b/content/ai/Skill/group.md index 077cf8208c..ea02cdaac7 100644 --- a/content/ai/Skill/group.md +++ b/content/ai/Skill/group.md @@ -121,9 +121,16 @@ existing user β†’ membership now; unknown β†’ invitation + `AddToGroup` subscrip # 🚨 Email caveat β€” verify it actually sends The invitation email only goes out when the portal has **`Email:Enabled = true`** (Microsoft Graph -`Mail.Send`). It defaults to **false** β†’ `NoOpEmailSender` just logs. If email is off, the invitation -still authorises onboarding and the durable grant still fires β€” but **no email is sent**; tell the user -rather than reporting a phantom send. Check the deployment's `Email__Enabled`. +`Mail.Send`) **AND** the `MeshWeaver.Mail.MicrosoftGraph` module landed. `Email:Enabled` defaults to +**false** β†’ `NoOpEmailSender` just logs. If email is off, the invitation still authorises onboarding +and the durable grant still fires β€” but **no email is sent**; tell the user rather than reporting a +phantom send. Check the deployment's `Email__Enabled`. + +`Enabled=true` with the module ABSENT is a **refused** configuration, not a working one: the +invitation emailer does not start and logs an error naming the missing module, so the invitation +stays un-emailed (`EmailSentAt` null) and goes out by itself once the module lands. It used to be +stamped as sent while nothing was delivered (#2023) β€” if you see that on an old record, the +invitation was never actually emailed. # Verify β€” never declare done without this diff --git a/deploy/aks/DEPLOY-RUNBOOK.md b/deploy/aks/DEPLOY-RUNBOOK.md deleted file mode 100644 index 484596e496..0000000000 --- a/deploy/aks/DEPLOY-RUNBOOK.md +++ /dev/null @@ -1,465 +0,0 @@ -# Deploy Memex to AKS β€” the `portal.example.com` runbook - -This is the **exact, verified** sequence used to bring up `https://portal.example.com` on a -private AKS cluster in **swedencentral**. It is the reproducible template behind the sample in -this folder (`infra/` Bicep, `values.aks.yaml`, `manifests/`) and the image-based Aspire -AppHost at [`../aspire/Memex.Deploy.AppHost`](../aspire/Memex.Deploy.AppHost). - -> **Model:** one Aspire AppHost (`Memex.Deploy.AppHost`) models the workload from published -> images; the **Kubernetes publisher** generates the Helm chart (`../helm`); this folder adds the -> AKS *platform* (Bicep) + overlay. AKS is the deploy target. All config flows from deploy -> parameters β†’ env. See "Why a runbook and not pure `aspire up`" at the bottom. - -Architecture decisions baked in (see the AGENTS memory + `../helm`): -- **Private** AKS API server + **private** Postgres Flexible Server; **only** the portal is public (`:443`). -- **One shared ACR** `meshweaver.azurecr.io` (RG `meshweaver-shared`) across all solutions. -- **Filesystem backend** with content on RWX **Azure Files** (`/mnt/content`); mesh data in Postgres. -- **Blazor sticky sessions** (cookie affinity = ACA's "bind tab to server"); **1 replica** today - (multi-replica needs Orleans `AzureTables` clustering β€” a follow-up). -- TLS via **cert-manager + Let's Encrypt** (HTTP-01). - ---- - -## 0. Prerequisites -- `az` β‰₯ 2.84 (logged in to the target subscription/tenant), `az bicep`, `docker`, .NET 10 SDK. -- A globally-unique shared ACR (here `meshweaver`). Create once: - `az group create -n meshweaver-shared -l swedencentral` ; - `az acr create -g meshweaver-shared -n meshweaver --sku Premium`. -- DNS zone for your domain in Azure DNS (here `systemorph.com`, RG `dns`). - -## 1. Build + push images to the shared ACR - -Images are **multi-arch** (linux/amd64 + linux/arm64) β€” one tag serves x86 cloud nodes AND -Apple-silicon (arm64) local k3s, so every install can pull-and-self-update natively from ACR. - -```bash -# Base image (node + Claude Code + Copilot CLIs) β€” MULTI-ARCH manifest list. -# @anthropic-ai/claude-code is pure JS (arch-independent); @github/copilot resolves its -# per-platform binary via npm optional deps, so each arch's build bakes in the matching CLI. -az acr build --registry meshweaver --image memex-portal-ai-base:latest \ - --platform linux/amd64 --platform linux/arm64 deploy/base-images/portal-ai -# (equivalent local build: `docker buildx build --platform linux/amd64,linux/arm64 \ -# -t meshweaver.azurecr.io/memex-portal-ai-base:latest --push deploy/base-images/portal-ai`) - -# App images β€” drop `-r linux-x64`; set RuntimeIdentifiers + ContainerRuntimeIdentifiers to both -# RIDs. With no single RID, the SDK (>= 8.0.405; we're on .NET 10) publishes per-RID and combines -# them into an OCI Image Index (manifest list). ContainerRuntimeIdentifiers MUST be a subset of -# RuntimeIdentifiers (set them equal). The arm64 leg layers on the multi-arch base above. -az acr login --name meshweaver -dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj \ - -c Release --no-self-contained -t:PublishContainer -p:PublishProfile= \ - -p:RuntimeIdentifiers="linux-x64;linux-arm64" -p:ContainerRuntimeIdentifiers="linux-x64;linux-arm64" \ - -p:ContainerRegistry=meshweaver.azurecr.io -p:ContainerRepository=memex-portal-ai \ - -p:ContainerImageTag=latest -p:ContainerBaseImage=meshweaver.azurecr.io/memex-portal-ai-base:latest -dotnet publish memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj \ - -c Release --no-self-contained -t:PublishContainer -p:PublishProfile= \ - -p:RuntimeIdentifiers="linux-x64;linux-arm64" -p:ContainerRuntimeIdentifiers="linux-x64;linux-arm64" \ - -p:ContainerRegistry=meshweaver.azurecr.io -p:ContainerRepository=memex-migration -p:ContainerImageTag=latest -``` - -> **First multi-arch roll:** the multi-arch base (`memex-portal-ai-base:latest`) must exist before -> the first multi-arch app build β€” the arm64 leg has no base layer otherwise. Rebuild the base -> multi-arch once (the `az acr build … --platform …` above), then the app builds (and the -> continuous self-update path) work for both architectures. - -## 2. Provision the AKS platform (Bicep) -Edit `infra/main.parameters.json` (region, node size/count within your vCPU quota β€” swedencentral -defaulted to 2Γ— `Standard_D4s_v3` under a 10-vCPU cap). Then: -```bash -PG_PW="$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 28)Aa1" # or your own -az deployment sub create --name memex-aks-infra-sc --location swedencentral \ - --template-file deploy/aks/infra/main.bicep \ - --parameters @deploy/aks/infra/main.parameters.json \ - --parameters postgresAdminPassword="$PG_PW" -``` -Outputs: cluster name, the Postgres FQDN, the shared-ACR login server. Grant the cluster kubelet -**AcrPull** on the shared ACR (cross-RG, so done out-of-band): -```bash -KUBELET=$(az aks show -g -n --query identityProfile.kubeletidentity.objectId -o tsv) -az role assignment create --assignee-object-id $KUBELET --assignee-principal-type ServicePrincipal \ - --role AcrPull --scope $(az acr show -n meshweaver --query id -o tsv) -``` -Same cross-RG grant for the **portal Workload Identity** (the shared UAMI the in-pod self-updater uses -to list ACR tags β€” provisioned by `infra/modules/portal-identity.bicep`, federated to -`system:serviceaccount::memex-portal-sa` for every portal namespace): -```bash -PORTAL_MI=$(az identity show -g -n --query principalId -o tsv) -az role assignment create --assignee-object-id $PORTAL_MI --assignee-principal-type ServicePrincipal \ - --role AcrPull --scope $(az acr show -n meshweaver --query id -o tsv) -# Then wire its clientId into selfUpdate.azureClientId for each env (same value everywhere): -az deployment sub show --name memex-aks-infra-sc --query properties.outputs.portalIdentityClientId.value -o tsv -``` -(Pure-IaC alternative to both out-of-band grants: deploy with `grantSharedAcrPull=true` for the portal -UAMI β€” needs User Access Administrator on `meshweaver-shared`. See [DeploymentAKS β†’ Portal self-update](../../src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md).) -> Postgres connection uses the **private IP + password + SSL** (the FQDN would trip the portal's -> `database.azure.com` β†’ Entra-token branch, which doesn't match a password server). Get it with: -> `az network private-dns record-set a list -g -z -o table`. - -## 3. External sign-in (OAuth) apps -- **Microsoft/Entra** (single-tenant home): - ```bash - az ad app create --display-name "Memex Portal (portal.example.com)" --sign-in-audience AzureADMyOrg \ - --web-redirect-uris "https://portal.example.com/signin-microsoft" - az ad app credential reset --id --display-name aks --years 1 # => client secret - ``` -- **Google** (Cloud Console) + **LinkedIn** (Developer portal): create web OAuth clients with - redirect URIs `https://portal.example.com/signin-google` and `/signin-linkedin`. - -## 4. Deploy the workload (private cluster β†’ `az aks command invoke`) -Copy `scripts/values.deploy.example.yaml` β†’ `scripts/values.deploy.yaml`, fill in the **real** -connection string, master key, and OAuth secrets (keep it OUT of git β€” `artifacts/`/Key Vault), then: -```bash -az aks approuting enable -g -n # managed nginx (public LB) -cd deploy/aks/scripts -export MEMEX_PG_CONN='Host=;Port=5432;Username=memexadmin;Password=;Database=memex;SslMode=Require;Trust Server Certificate=true' -az aks command invoke -g -n --command "bash deploy.sh" --file . -``` -`deploy.sh` does: namespace + RWX PVCs β†’ `helm upgrade --install` (chart + `values.aks.yaml` + -`values.deploy.yaml`) β†’ scale the chart's in-cluster pg to 0 (we use the Flexible Server) β†’ -`kubectl set image` to the shared ACR β†’ patch the portal to 1 replica + the Azure Files mounts β†’ -**patch the connection-string secret** (the generated chart hardcodes the in-cluster pg β€” known -chart-gen gap). **Observability is folded in:** export `GRAFANA_PW=...` alongside `MEMEX_PG_CONN` -and `deploy.sh` also brings up Grafana + Loki + Prometheus (see Β§6); omit it to skip monitoring. -At the model level, `AddMemex`'s `OtlpEndpoint` option wires `OTEL_EXPORTER_OTLP_ENDPOINT` for -OTLP traces/metrics (not needed for log shipping β€” Promtail scrapes stdout). - -## 5. Public ingress + TLS + DNS -```bash -IP=$(az aks command invoke -g -n \ - --command "kubectl get svc -n app-routing-system nginx -o jsonpath='{.status.loadBalancer.ingress[0].ip}'") -az network dns record-set a add-record -g dns -z systemorph.com -n memex --ipv4-address $IP --ttl 300 -cd deploy/aks/scripts -az aks command invoke -g -n --command "bash tls.sh" --file tls.sh # cert-manager + Let's Encrypt + ingress -``` -HTTPβ†’HTTPS redirect is automatic once the ingress has TLS. Verify (bypassing DNS cache): -```bash -curl -sS -o /dev/null -w "%{http_code} verify=%{ssl_verify_result}\n" \ - --resolve portal.example.com:443:$IP https://portal.example.com/ -``` - -**Default SSL certificate (cluster-wide, one-time).** Without it, any client that connects -without SNI gets the self-signed "Kubernetes Ingress Controller Fake Certificate" β€” and -corporate TLS-inspection / URL-categorization appliances probe exactly that way, then flag the -whole domain as insecure and block it for their users (seen 2026-08: a client's IT blocked -`memex.meshweaver.cloud` in Firefox *and* Edge over this). Point the app-routing controller's -default cert at the flagship host's cert-manager secret β€” patch the `NginxIngressController` -CR, **not** the nginx deployment (the addon operator reverts direct deployment edits): -```bash -az aks command invoke -g -n --command \ - "kubectl patch nginxingresscontroller default --type merge -p '{\"spec\":{\"defaultSSLCertificate\":{\"secret\":{\"name\":\"memexcloud-tls\",\"namespace\":\"memex-cloud\"}}}}'" -# verify from outside β€” must show the real cert, not "Acme Co": -echo | openssl s_client -connect :443 -noservername 2>/dev/null | openssl x509 -noout -subject -``` -The setting survives addon updates but is NOT re-created on a cluster rebuild β€” re-apply it -whenever the cluster (or the `NginxIngressController` CR) is recreated. - ---- - -## 6. Observability (Grafana + Loki + Prometheus) + admin access via VPN -Everything except the portal stays private, so admin tools (Grafana, kubectl) go through the -**P2S VPN**, not a public endpoint. - -**Install the stack** (`scripts/install-observability.sh` β€” grafana/loki-stack: Loki + Promtail + -Grafana + Prometheus, datasources auto-wired, Promtail ships every pod's logs to Loki): -```bash -export GRAFANA_PW='' -cd deploy/aks/scripts -az aks command invoke -g -n \ - --command "GRAFANA_PW=$GRAFANA_PW bash install-observability.sh" --file install-observability.sh -``` - -**Set up the P2S VPN client** (the gateway + a root cert are provisioned by the Bicep + step 2): -```bash -# 1. Generate a P2S root+client cert (Windows) and upload the ROOT public cert to the gateway: -# $root = New-SelfSignedCertificate -Type Custom -KeySpec Signature -Subject "CN=MemexP2SRootCert" -KeyUsage CertSign -KeyExportPolicy Exportable -CertStoreLocation Cert:\CurrentUser\My -HashAlgorithm sha256 -KeyLength 2048 -# $client = New-SelfSignedCertificate -Type Custom -DnsName MemexP2SChild -KeySpec Signature -Subject "CN=MemexP2SChildCert" -Signer $root -KeyExportPolicy Exportable -CertStoreLocation Cert:\CurrentUser\My -HashAlgorithm sha256 -KeyLength 2048 -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") -# [IO.File]::WriteAllText("root.txt",[Convert]::ToBase64String($root.RawData)) -# NOTE: this az version reads --public-cert-data as a FILE PATH, so pass the path (NOT the inline string, NOT @file): -# az network vnet-gateway root-cert create -g --gateway-name --name MemexP2SRootCert --public-cert-data root.txt -# 2. Download + install the VPN client, then connect: -az network vnet-gateway vpn-client generate -g -n -o tsv # -> download URL (zip) -# 3. With the VPN connected: -az aks get-credentials -g -n -kubectl -n monitoring port-forward svc/loki-grafana 3000:80 # http://localhost:3000 (admin / $GRAFANA_PW) -``` -In Grafana β†’ Explore β†’ Loki, the portal logs are `{namespace="memex"}` (e.g. add -`|= "error"` or `|~ "signin-microsoft"`). - -### 6b. Red-log ticketing (optional) β€” every `fail:`/`crit:` opens exactly one issue - -Once Loki is up, `mw-log-watcher` can turn production errors into GitHub issues: it reads red lines -from a persisted cursor, groups them by fault, and reports each distinct fingerprint to the portal, -which triages it with an agent and opens one ticket. A burst of ten thousand identical errors is one -issue; recurrences comment on it. **It is off until all four steps below are done** β€” the ingest -endpoint is not even mapped without a token. - -It runs in `monitoring`, NOT in the portal's namespace, on purpose: the thing that notices the portal -is throwing errors must not be hosted by the portal. - -```bash -# 1. ONE shared secret, both sides. The watcher presents it; the portal requires it. -TOKEN=$(openssl rand -hex 32) -az aks command invoke -g -n --command \ - "kubectl -n monitoring create secret generic mw-log-watcher --from-literal=ingest-token=$TOKEN" -# …then set LogWatch__IngestToken to the SAME value on the portal (KeyVault β†’ its secret store). - -# 2. Where tickets go. Without at least one route the control plane idles by design β€” -# it will not spend agent rounds on incidents it could never file. -# LogWatch__DefaultRepository = Systemorph/MeshWeaver -# LogWatch__Routes__0__Prefix = MeshWeaver. -# LogWatch__Routes__0__Repository = Systemorph/MeshWeaver -# LogWatch__Routes__1__Prefix = Memex. -# LogWatch__Routes__1__Repository = Systemorph/Memex -# Every key carries the LogWatch__ prefix β€” a bare Routes__0__Repository does not bind, and an -# unbound route reads exactly like "no route configured": incidents pile up at New, silently. -# Issues are opened as the GitHub App (GitHub:App), never a user's OAuth token. - -# 3. Build + push the watcher image. -dotnet publish tools/MeshWeaver.LogWatcher/MeshWeaver.LogWatcher.csproj -c Release \ - -t:PublishContainer -p:ContainerRegistry=meshweaver.azurecr.io \ - -p:ContainerRepository=memex-log-watcher -p:ContainerImageTag= - -# 4. Apply the Deployment + PVC (edit the image tag + watched namespaces in the manifest first). -az aks command invoke -g -n \ - --command "kubectl apply -f log-watcher.yaml" \ - --file manifests/observability/log-watcher.yaml -``` - -**Verify β€” and know what each failure looks like:** - -```bash -az aks command invoke -g -n --command \ - "kubectl -n monitoring logs deploy/mw-log-watcher --tail=50" -``` - -| What the log says | Meaning | -|---|---| -| `Loki: N red line(s) …` then `Reported … β€” 200` | Working end to end. | -| `Log watcher is not configured` | Step 1 missed β€” `PortalUrl`/`IngestToken` unset. | -| `Portal REJECTED report … with 401` | The two tokens differ β€” the watcher's secret and the portal's `LogWatch__IngestToken` must be the same string. | -| `Portal REJECTED report … with 404` | The portal's `LogWatch__IngestToken` is unset, so `/api/log-incidents` is not mapped at all β€” step 1 was done on the watcher side only. | -| `Loki: 0 line(s)` forever | The namespace label is wrong, or Promtail is not shipping that namespace. | -| `Could not persist watcher state … Access to the path … is denied` | The state PVC mounted root-owned. The pod needs `securityContext.fsGroup: 1654` (the shipped manifest sets it). The watcher still reports, but the cursor stops persisting, so a restart replays the lookback window. | -| Incidents appear but stay `New` | Step 2 missed β€” no repository routed, so the control plane idles. | - -Then browse `Admin/_LogIncident` in the portal: every incident links to the ticket it opened and to -the triage thread that wrote it. - -**🚨 The state directory must be a real volume.** On an `emptyDir` a pod restart replays the lookback -window and re-reports. The manifest ships a PVC for this reason. - -**To turn it off**: `kubectl -n monitoring scale deploy/mw-log-watcher --replicas=0`. Clearing the -portal's `LogWatch__IngestToken` also closes the ingest endpoint. Neither deletes existing incidents. - -Full reference: [LogWatchTriage.md](../../src/MeshWeaver.Documentation/Data/Architecture/LogWatchTriage.md) - -## 7. Rolling a new image β€” and the traps - -```bash -# a) The manifest MUST carry every arch leg. A partial manifest list = ImagePullBackOff on the -# missing arch, which presents as "the deploy hung" (memex-cloud V46 outage, 2026-07-19). -az acr manifest show -r meshweaver -n memex-portal-ai: \ - | jq -r '.manifests[]?.platform | "\(.os)/\(.architecture)"' # expect linux/amd64 AND linux/arm64 - -# b) Roll. -az aks command invoke -g -n --command \ - "kubectl set image deploy/memex-portal-deployment memex-portal=meshweaver.azurecr.io/memex-portal-ai: -n \ - && kubectl rollout status deploy/memex-portal-deployment -n --timeout=600s" - -# c) Verify with REAL signals, looped. HTTP 200 is the Blazor shell and proves nothing. -for i in $(seq 1 10); do curl -s -o /dev/null -w "%{http_code} " https:///api/content//content/; done -``` - -### 🧱 NodeType bake β€” the deploy MUST compile every dynamic NodeType - -Fresh pods invalidate every dynamic NodeType's cached assembly (the framework MVID changed), so -they all need a recompile. Left lazy, that compile happens on user requests after the pod is -already serving β€” and a type nothing happens to touch stays **"no definition"** until its pages -hang with *"No response received … `SubscribeRequest` β†’ target X"* (SocialMedia/Post on -memex-cloud, 2026-07-30: every post page burned the 60 s timeout all morning while the portal -looked healthy). **Managed envs therefore run the bake ON β€” these three knobs travel together** -(`values.aks.yaml` carries them; keep them in every env overlay so a `helm upgrade` never -reverts them): - -| Knob | What it does | -|---|---| -| `config.memex_portal.PreWarm__DynamicTypes: "true"` | every new pod sweeps + compiles ALL dynamic NodeTypes at start (resumes from the shared `/data` cache β€” warm restarts are cheap) | -| `config.memex_portal.PreWarm__GateReadiness: "true"` | `/health` stays red until the sweep is green; with `maxSurge 1 / maxUnavailable 0` a regressed type STALLS the rollout with the old image serving. **⚠️ Depends on the sweep, and `values.aks.yaml` currently sets BOTH to `"false"` β€” read the namespace, not this table (#1981).** The two keys are ONE setting: the gate reads state only the sweep writes, so gate-without-sweep is disarmed at startup (it used to register and stay permanently green) and logged at Critical. Turning the gate on means turning `PreWarm__DynamicTypes` on in the same change. It was tried 2026-08-02 and reverted the same day on 7 FALSE regressions β€” all cross-silo `SubscribeRequest` timeouts, not compile errors (#694 residue). The gate no longer reads "no answer" as "it broke": a `TimedOut` outcome is filed as *unevaluated* and can never gate, and that leniency now survives the cascade (a dependent of an unevaluated upstream is `UpstreamUnevaluated`, also non-gating). Only a `CompileError` β€” or an `UpstreamFailed` cascading from one β€” on a **previously-healthy** type stalls a roll | -| `config.memex_portal.PreWarm__AllowUnprovenBake: "false"` | **βœ… OFF (strict).** The gate also refuses readiness when the sweep *errored* β€” enumeration threw or timed out β€” because such a pod verified **nothing** and a gate that certifies "I verified nothing" is worse than no gate. That guard used to live in the pre-run bake Job (*"FINDING NOTHING IS NOT PASSING"*, exit 3, `Bake__AllowEmpty`) and was lost when #1357 retired the Job; it is now enforced on the surviving path as `BakePhase.Faulted`. ⚠️ **"Empty" is not "unproven"** β€” a mesh that genuinely has no dynamic NodeTypes enumerates fine, completes and serves; only the *inability to get an answer* gates. Set `"true"` only to roll forward past an environment that cannot answer the enumeration, accepting lazy compilation. It can never waive a real regression, and `/health` keeps reporting the bake as unproven | -| `probes.startup: {periodSeconds: 10, failureThreshold: 180}` (= 30 min) | ⚠️ REQUIRED with the gate: a cold bake is **~2.4 s/type**, sequential *(measured 2026-08-10, prod Loki, three portals)* β€” ~10 min on memex-cloud, the largest mesh β€” and the default 5 min budget kills the pod mid-bake forever. 30 min is that worst case plus a plain cold boot, x2. `progressDeadlineSeconds` is DERIVED from these two in the chart, so raising them can't leave it behind. **Was `1080` (3 h)** until 2026-08-10, sized from a "~90 s/type" estimate that was 37x too high β€” a window that long detected nothing | - -**🚨 Before you trust the gate, verify the namespace actually reads it.** The gate protects a -portal through exactly two deployment facts, and on 2026-08-10 two of the three portals had drifted -away from both. A gated config in a namespace missing either is *false confidence* β€” it reports and -the outage happens anyway. - -| Must be true | Why | Check | -|---|---|---| -| a `startupProbe` exists **on `/health`** | that probe is the ONLY reader of the gate; no startupProbe (or one on `/alive` / `/healthz`) ignores it entirely | `kubectl -n get deploy memex-portal-deployment -o jsonpath='{.spec.template.spec.containers[0].startupProbe}'` | -| `strategy.maxUnavailable: 0` | surge-first keeps the old pod serving until the new one passes; at `maxUnavailable:1` with `replicas:1` the only serving pod can be deleted BEFORE the replacement is ready | `kubectl -n get deploy memex-portal-deployment -o jsonpath='{.spec.strategy.rollingUpdate}'` | - -Both are rendered correctly by the chart β€” a `helm upgrade` restores them. A per-env `portal-patch.json` -or a hand `kubectl patch` can still override them, which is how the drift happened. - -πŸͺ¦ **There is no pre-run bake Job any more (#1347) β€” the CI bake replaced it (#1660 WS3).** The -separate `memex-bake` image was removed after two weeks of running in zero namespaces. On its only -AKS run (memex-cloud, 2026-07-30) `memex-bake:3.0.0-ci.1565` computed a **different framework -fingerprint** than the running `portal-ai:3.0.0-ci.1565` β€” same version, same commit, separately -published β€” so its framework-stale kickoff started flipping CURRENT NodeType records to `Pending` -and rebuilding them for a framework nothing serves. The cause was the per-build -`+build.` stamp `InformationalVersion` then carried under `CIRun`: every -`dotnet publish` minted a fresh `MeshWeaver.Graph` MVID. #1660 WS3 removed that stamp and made CI -builds deterministic β€” the framework identity is now the **API-surface hash** (`s`, -`FrameworkBuildIdentity`; identical across CI builds AND across internal-only merges, moved only -by breaking surface changes or Graph changes) β€” so the Build-and-Test run's bake artifact IS -adoptable, and usually already published: `main-cd`'s -`publish-bake` job copies it onto the portals' `/data` share -(`prebuilt-bundles///`), and each booting pod seeds its own identity's bundles -(`PreWarm__PrebuiltBundleRoot`) before its sweep. - -**The pod-side sweep remains the enforcement.** It runs in the serving process, adopts whatever CI -published, and compiles the remainder β€” 76 s for 280 types cold (memex-cloud, batch -direct-compile), seconds when the CI bake covered the shipped content. The "fail before prod" -contract is given by `PreWarm__GateReadiness` + `maxSurge:1` / `maxUnavailable:0`: the new pod -refuses readiness until its OWN bake is green while the old image keeps serving. - -**Verify after every roll** β€” the bake completing is the deploy signal, not HTTP 200: - -```bash -# The sweep's verdict (compiled=N alreadyBaked=M compileErrors=0 …) β€” read the NEWEST pod -# explicitly: `kubectl logs deploy/…` picks an arbitrary pod and mid-rollout that is often the OLD one. -NEW=$(kubectl -n get pods -l app.kubernetes.io/component=memex-portal \ - --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') -kubectl -n logs "$NEW" | grep "DynamicTypePreWarmer: warm-up complete" -# The gate's verdict (Healthy "baked in …" / Unhealthy "regressed" β€” rollout stalled on purpose): -kubectl -n get pods # new pod 0/1 while baking is CORRECT; investigate only a Regressed log line -``` - -Live-env equivalent without a helm apply (what enabled memex-cloud on 2026-07-30): - -```bash -kubectl -n patch configmap memex-portal-config --type merge \ - -p '{"data":{"PreWarm__DynamicTypes":"true","PreWarm__GateReadiness":"false"}}' -kubectl -n patch deployment memex-portal-deployment --type json -p \ - '[{"op":"replace","path":"/spec/template/spec/containers/0/startupProbe/periodSeconds","value":10}, - {"op":"replace","path":"/spec/template/spec/containers/0/startupProbe/failureThreshold","value":180}]' -# the probe patch rolls the deployment; the new pod bakes behind the gate while the old serves -``` - -- **A pod sitting 0/1 for a long time is the gate doing its job** while a cold bake runs; only a - `REFUSING READINESS … Regressions:` log line means a type broke on this image β€” fix the type, - never widen the timeout. -- **Retries are cheap β€” the sweep resumes from the shared cache.** Every type a sweep DID build is - on `/data` for good; deleting a gate-red pod re-runs only what is missing. Progress across - attempts is monotonic. -- **⚠️ Known flake while core #694 is open (cross-silo reply routing):** during a roll the mesh - briefly runs TWO silos (old serving + new baking), and in that window the pod-side sweep's - shared-source resolution can fail nondeterministically β€” the symptom is a `CompileError` full of - unresolved names that live in a SIBLING type's `Source/` (`shared=@…`), on a type that compiles - clean when triggered individually. First verify with a targeted compile (MCP `compile - @` β€” recycle the type's hub first if the subscribe times out; the targeted subscribe rides - the SAME cross-silo routing, so it completes reliably only in a single-silo window β€” delete the - baking pod and use the ~2 minutes before its replacement joins), then let the replacement's - re-sweep find the fixed types `alreadyBaked`. (Observed live on the first gated roll, memex-cloud - 2026-07-30: the gate caught 2 types with stale-green records whose newest assemblies were 6 days - old β€” real, invisible breakage β€” plus sweep failures on shared-source types that compiled clean - when triggered individually in a single-silo window.) - -### Scaling: KEDA wins -`kubectl scale --replicas=0` (the documented heal for a wedged mesh) is **silently reverted** by a -`ScaledObject` with `minReplicaCount: 2` β€” you conclude the heal failed when it never ran. -`kubectl get scaledobject -n ` first, and check `PAUSED`. - -#### 🚧 A PAUSED scaler is a FENCE β€” find out why before you remove it - -`autoscaling.keda.sh/paused-replicas: ""` pins the deployment at `n`, **deletes the HPA**, and -makes `minReplicaCount` inert. Nothing in any chart sets it, `kubectl get deploy` does not show it, -and it survives every roll β€” so it reads as "this deployment is just single-replica" long after -whoever applied it has moved on. - -**Nobody applies it for fun.** It is the most direct way to say "stop making new silos", so assume -it is suppressing a multi-silo defect until you have evidence otherwise. `memex-cloud` carried one -for **16 days** (2026-07-29 β†’ 2026-08-14): it was applied 87 minutes after the second fix for -**#694 β€” *cross-silo posts lose AccessContext: static content 500s on ~50% of requests with 2 -replicas*** β€” and 29 seconds after KEDA scaled the namespace back up. Removing it without reading -that history is re-running the incident. - -Establish provenance before unpausing β€” the same way you would for any hand-applied field: - -```bash -kubectl -n get scaledobject -o jsonpath='{range .metadata.managedFields[*]}{.manager} {.operation} {.time}{"\n"}{end}' -kubectl -n get scaledobject -o jsonpath='{.status.lastActiveTime}{"\n"}' # when it last scaled -``` - -`managedFields` gives you the writer (`kubectl-annotate`) and the timestamp; `lastActiveTime` -immediately before it is the tell that someone watched it scale and stopped it. Then find what -shipped or broke that day (`git log --since= --until=`, closed issues), and only -unpause once you can name the defect and point at its fix. Treat the unpause as a **monitored -experiment** with a rollback trigger, not a config tidy-up β€” a fix merged while the namespace was -pinned has only ever been exercised in the transient two-silo window of a rollout, never in steady -state. `check-chart-drift.sh` reports the annotation as CLUSTER-ONLY drift so it stops being -invisible. - -### Crash dumps: verify the MOUNT -`DOTNET_DbgEnableMiniDump` + `DOTNET_DbgMiniDumpName=/data/dumps/…` do nothing without a volume at -that path β€” `createdump` does not create directories, so the crash destroys its own evidence. The -chart mounts a dedicated `memex-dumps` emptyDir; verify the LIVE pod actually has it: - -```bash -kubectl get deploy memex-portal-deployment -n \ - -o jsonpath='{range .spec.template.spec.containers[0].volumeMounts[*]}{.name} -> {.mountPath}{"\n"}{end}' \ - | grep dumps || echo "NO DUMP MOUNT β€” every crash will produce nothing" -``` - -⚠️ `envs//portal-patch.json` replaces volumes **by index**, so adding or reordering a chart -volume silently misaligns an environment and a cluster can run an older volume set while the chart -in git looks right. Diff live mounts against the chart after every deploy. - -### Degraded-but-Ready replica β€” intermittent hangs, `kubectl top` divergence - -Symptom: some requests hang or serve garbled state while most succeed, typically after a burst of -content syncs or a scheduled bake. Cause class (2026-08-25, MeshWeaver#2194): each NodeType -publication mints a new AssemblyLoadContext and serving instances stay on the OLD build behind the -Recycle banner, so every superseded ALC stays rooted β€” the type-hosting silos climb to tens of GB -and go GC-bound while still answering `/alive` (both post-startup probes), so Kubernetes never -pulls them from rotation. - -```bash -az aks command invoke -g -n --command "kubectl top pods -n --no-headers" -``` - -One or two pods far above their siblings in BOTH memory and CPU = this. Remedy: `kubectl delete -pod` the outliers (grace-drain; the Deployment replaces them). Prevention: enable -`Modules:AutoRecycleOnStaleBuild` (MeshWeaver#2192) so instances converge onto each newly -published build and old ALCs collect β€” mechanism in -`src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md`. - -## Known gaps / follow-ups -- **Dump mount not applied to the live clusters** (2026-07-28): all three namespaces carry the dump - env vars while no pod mounts `/data/dumps`, so every production `exit=139` so far produced no - dump. Applying the current chart fixes it; until then a `mkdir /data/dumps` on the `/data` PVC is - a stopgap that puts heap-sized dumps on a shared 16Gi share instead of the size-bounded emptyDir. -- ~~**Multi-replica HA**~~ (chart side done 2026-08-14): Orleans **AdoNet** clustering is the HA - provider (`Features:Orleans:Clustering`, legacy key `Deployment:Orleans:Clustering`), backed by a - dedicated `orleans` database the migration's `OrleansClusteringSetup` creates. The chart now - templates the replica count and omits `spec.replicas` entirely under KEDA so the HPA owns it; - `deploy/aks/scripts/check-chart-invariants.sh` asserts the prerequisites at render time. What - remains is per-environment and per-cluster, not chart work: each env's overlay must set - `keda.enabled`, and a paused `ScaledObject` still pins the count regardless (see "Scaling: KEDA - wins" above β€” check `PAUSED` before believing a replica number). -- ~~**Chart connection string**~~ (fixed 2026-08-14): `../helm/templates/memex-portal/secrets.yaml` - and its migration counterpart now take `secrets.memex_{portal,migration}.ConnectionStrings__memex` - / `__orleans` from values, falling back to the in-cluster host only when unset. An env that - supplies them no longer needs the post-`helm` secret patch in `deploy.sh`. 🚨 Until an env's - values file DOES supply them, `helm upgrade` still re-renders the in-cluster host β€” and for - `__orleans` nothing patches it back, so an AdoNet namespace loses cluster membership. The chart - `fail`s the render if an external `ConnectionStrings__memex` is supplied without a matching - `__orleans`, so the half-configured case cannot ship silently. -- **Secrets β†’ Key Vault**: move the PG password, master key, and OAuth secrets into - Key Vault via the CSI Secrets Store add-on (enabled in `infra/modules/aks.bicep`). - -## Why a runbook and not pure `aspire up` -Aspire's Kubernetes publisher generates the **workload** chart, but it does not provision an AKS -**cluster**, a private Postgres Flexible Server, a VPN, or Let's Encrypt. Those platform pieces are -the Bicep + these steps. The AppHost (`../aspire/Memex.Deploy.AppHost`) owns the app model + the -deploy parameters (including the OAuth providers); this runbook stitches the platform around it. diff --git a/deploy/aks/README.md b/deploy/aks/README.md index f0fd928b43..5cb841dd54 100644 --- a/deploy/aks/README.md +++ b/deploy/aks/README.md @@ -1,4 +1,4 @@ -# MeshWeaver Memex on AKS β€” production-grade deployment sample +# MeshWeaver Memex on AKS β€” a public, self-contained deployment sample A reference, operator-facing deployment of the **Memex portal** on a **private Azure Kubernetes Service** cluster. It layers AKS-specific Azure infrastructure @@ -6,9 +6,21 @@ Azure Kubernetes Service** cluster. It layers AKS-specific Azure infrastructure already lives at [`../helm`](../helm). Everything here is **infra / YAML / markdown only** β€” no application code changes. -> This is a **sample**. Read it end-to-end, tune the parameters for your -> environment (regions, SKUs, CIDRs, DNS names, secrets), and treat the security -> defaults as a starting point, not a finished hardening. +> This is a **sample**, and it is **self-contained**: this one file is the whole +> AKS story β€” bring-up, ingress/TLS, the roll-out procedure, and the traps that +> have actually bitten production. Read it end-to-end, tune the parameters for +> your environment (regions, SKUs, CIDRs, DNS names, secrets), and treat the +> security defaults as a starting point, not a finished hardening. + +Architecture decisions baked in: + +- **Private** AKS API server + **private** Postgres Flexible Server; **only** the + portal is public (`:443`). +- One shared **Azure Container Registry**, reachable from every environment. +- Content on RWX **Azure Files** (`/mnt/content`); mesh data in Postgres. +- **Blazor sticky sessions** (cookie affinity), HA across replicas with Orleans + **AdoNet** clustering. +- TLS via **cert-manager + Let's Encrypt** (HTTP-01). --- @@ -17,12 +29,12 @@ markdown only** β€” no application code changes. | Concern | This sample's choice | |---|---| | Cluster | **Private AKS** (`enablePrivateCluster=true`) β€” API server has a private IP only | -| kubectl reach | **Azure VPN Gateway (Point-to-Site)** + linked **private DNS zone** `privatelink..azmk8s.io` | +| kubectl reach | **`az aks command invoke`** β€” runs kubectl/helm inside the cluster, no network line-of-sight. An **optional** Azure VPN Gateway (Point-to-Site) + linked private DNS zone `privatelink..azmk8s.io` gives interactive tooling a route into the VNet ([Appendix A](#appendix-a--point-to-site-vpn-optional)) | | Registry | **Azure Container Registry** (Premium) with **AcrPull** granted to the cluster's kubelet identity | | Networking | VNet with `aks-nodes`, `GatewaySubnet`, `AzureBastionSubnet` subnets; Azure CNI overlay + Cilium | -| Portal | Blazor Server, **HA (3 replicas across 3 zones)** behind ingress with **cookie session affinity** | +| Portal | Blazor Server, **HA** behind ingress with **cookie session affinity**; replicas owned by a KEDA `ScaledObject` (2 β†’ 8 in this overlay), never by a `kubectl patch` | | Shared storage | **Azure Files (RWX)** drives mounted at explicit paths: `/data` (caches), `/mnt/content` (content collection), `/mnt/attachments`, `/mnt/users` β€” via a custom `azurefile-memex` StorageClass tuned for the non-root portal (uid 1654) | -| Database | Self-managed **Postgres (pgvector) StatefulSet** on a Premium-SSD PVC | +| Database | **Azure Database for PostgreSQL Flexible Server**, VNet-injected + private (default). A self-managed **Postgres (pgvector) StatefulSet** on a Premium-SSD PVC is the in-cluster alternative | | Backup / PITR | **pgBackRest** β†’ **Azure Blob** (full + diff CronJobs + WAL archiving) β†’ restore `--type=time` | | Observability | **OpenTelemetry Collector DaemonSet** captures cluster-wide pod logs + portal OTLP β†’ **Azure Files** log archive (`/mnt/otel-logs`); **Grafana + Loki + Promtail + Prometheus** in `monitoring` for search and dashboards | | Error ticketing | **`mw-log-watcher`** (optional, `monitoring`) reads red logs from Loki and opens one triaged GitHub issue per distinct fault β€” off until configured | @@ -33,9 +45,11 @@ markdown only** β€” no application code changes. ``` β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ operator laptop ───────────────┐ - β”‚ azure-vpn / OpenVPN client (cert auth) β”‚ + β”‚ az aks command invoke (primary β€” no tunnel) β”‚ + β”‚ azure-vpn / OpenVPN client (OPTIONAL, Appx A) β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ P2S tunnel (172.16.201.0/24) + β”‚ β€” optional; Grafana/psql only β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ VNet 10.0.0.0/16 β”‚ β”‚ β”Œβ”€β”€ GatewaySubnet ──┐ β”Œβ”€β”€ aks-nodes 10.0.0.0/20 ──┐ β”‚ @@ -56,8 +70,19 @@ markdown only** β€” no application code changes. ``` deploy/aks/ -β”œβ”€β”€ README.md ← you are here +β”œβ”€β”€ README.md ← you are here β€” the whole AKS story +β”œβ”€β”€ SELF-UPDATE.md ← how a running portal pulls its own new image +β”œβ”€β”€ INSTANCE-LIFECYCLE.md ← provision β†’ suspend β†’ tear down, driven from the mesh β”œβ”€β”€ values.aks.yaml ← Helm overlay for ../helm (AKS overrides) +β”œβ”€β”€ scripts/ +β”‚ β”œβ”€β”€ deploy.sh ← namespace + PVCs + helm upgrade, run via `command invoke` +β”‚ β”œβ”€β”€ tls.sh ← cert-manager + Let's Encrypt issuer + ingress TLS +β”‚ β”œβ”€β”€ install-observability.sh ← Grafana + Loki + Promtail + Prometheus (ns monitoring) +β”‚ β”œβ”€β”€ import-dashboards.sh ← generic Grafana dashboard importer +β”‚ β”œβ”€β”€ check-chart-invariants.sh ← PR gate: does the CHART describe a workable shape? +β”‚ β”œβ”€β”€ check-chart-drift.sh ← does the CLUSTER run what the chart describes? +β”‚ β”œβ”€β”€ aks-extras.yaml ← StorageClass + RWX PVCs, applied by deploy.sh +β”‚ └── values.deploy.example.yaml ← template for your secrets file (never commit the real one) β”œβ”€β”€ infra/ β”‚ β”œβ”€β”€ main.bicep ← subscription-scoped orchestrator (creates RG) β”‚ β”œβ”€β”€ main.parameters.json ← edit these @@ -71,13 +96,15 @@ deploy/aks/ └── manifests/ ← applied alongside the Helm release β”œβ”€β”€ storageclass-azurefile.yaml ← custom azurefile-memex SC (uid 1654, nobrl) β”œβ”€β”€ portal-pvcs.yaml ← RWX drives: data/content/attachments/users + pg PVC - β”œβ”€β”€ portal-ha-patch.yaml ← replicas=3, zone spread, probes, RWX volume mounts - β”œβ”€β”€ postgres-pvc-patch.yaml← bind pg StatefulSet to its PVC + β”œβ”€β”€ portal-ha-patch.yaml ← LEGACY zone spread + probes; the chart now renders the + β”‚ RWX volumes and owns the replica count β€” see Step 4 + β”œβ”€β”€ postgres-pvc-patch.yaml← LEGACY; the chart's StatefulSet has its own volumeClaimTemplate β”œβ”€β”€ portal-ingress.yaml ← ingress + cookie session affinity (Blazor) β”œβ”€β”€ observability/ β”‚ β”œβ”€β”€ otel-collector-config.yaml ← collector pipeline (filelog+otlp β†’ file+debug) β”‚ β”œβ”€β”€ otel-collector.yaml ← collector DaemonSet + SA/RBAC + Service - β”‚ └── otel-pvc.yaml ← RWX Azure Files PVC for the log archive + β”‚ β”œβ”€β”€ otel-pvc.yaml ← RWX Azure Files PVC for the log archive + β”‚ └── log-watcher.yaml ← mw-log-watcher Deployment + state PVC (optional) └── pgbackrest/ β”œβ”€β”€ serviceaccount.yaml← Workload-Identity SA for keyless Blob β”œβ”€β”€ configmap.yaml ← pgbackrest.conf (Azure repo) + WAL archive conf @@ -92,28 +119,46 @@ The chart at `../helm` is generated from the Aspire model and is intentionally **not** fork or regenerate it. Instead: - **`values.aks.yaml`** sets the keys the chart already consumes (`config.*`, - `secrets.*`) with AKS-correct values (e.g. Orleans `AdoNet` clustering for HA). + `secrets.*`, `persistence.*`, `keda.*`, `portal.image`) with AKS-correct values + (e.g. Orleans `AdoNet` clustering for HA). - **`manifests/`** supplies the things the generic chart does not template β€” - RWX PVCs, ingress with sticky sessions, the HA replica/zone patch, and the - pgBackRest sidecar/CronJobs β€” as `kubectl apply` / `kubectl patch` steps you - run right after `helm install`. - -The annotated `ingress:`, `persistence:`, `replicas:`, and `pgbackrest:` blocks -at the bottom of `values.aks.yaml` double as a forward-looking contract: if the -`../helm` chart is later extended to template these, the overlay already carries -the right values. + the custom StorageClass, the RWX PVCs themselves, ingress with sticky sessions, + and the pgBackRest sidecar/CronJobs β€” as `kubectl apply` steps around + `helm install`. + +🚨 **The overlay has been catching up to the chart, and the direction is one-way: +work keeps MOVING from `manifests/` into the chart.** `persistence:` and +`replicas:`/`keda:` used to be bolt-on `kubectl patch` steps and are now rendered +by `../helm` itself, which is why `manifests/portal-ha-patch.yaml` and +`manifests/postgres-pvc-patch.yaml` are marked LEGACY above and are **not** part +of the install (Step 4 says why re-applying them is actively harmful). When you +find a patch file here, check whether the chart already does it before running it. --- ## Prerequisites -- `az` CLI β‰₯ 2.84 (`az version`) +- `az` CLI β‰₯ 2.84 (`az version`), logged in to the target subscription/tenant - `az bicep` β‰₯ 0.41 (`az bicep version`) β€” `az bicep upgrade` if older - `kubectl` and `helm` β‰₯ 3.12 - A subscription where you can create resource groups + role assignments (Owner or User Access Administrator β€” the deployment grants AcrPull, DNS, and Blob roles) -- `openssl` (to mint the P2S VPN certificates), or Windows `New-SelfSignedCertificate` +- `docker` and the **.NET 10 SDK** β€” only if you build the images yourself + (Step 3, Option C) rather than pulling published ones +- A **DNS zone** for your domain in Azure DNS (Step 4b creates the A record in it) +- `openssl` β€” only for the **optional** P2S VPN certificates + ([Appendix A](#appendix-a--point-to-site-vpn-optional)); Windows + `New-SelfSignedCertificate` works too + +A **globally-unique registry** shared across environments, created once (skip if +you already have one, or if `infra/modules/acr.bicep` provisions a +per-deployment ACR for you): + +```bash +az group create -n meshweaver-shared -l swedencentral +az acr create -g meshweaver-shared -n --sku Premium +``` Validate the Bicep before deploying: @@ -152,8 +197,12 @@ az deployment sub show --name memex-aks-infra \ ``` > The VPN Gateway takes **20–45 minutes** to provision β€” this dominates the -> deploy time. Set `deployVpnGateway: false` if you'll use -> `az aks command invoke` or a Bastion jumpbox instead (see Step 2 alternatives). +> deploy time, and **you do not need it to deploy**. Step 2's +> `az aks command invoke` is the primary path and needs no gateway, so set +> `deployVpnGateway: false` unless you specifically want the interactive VNet +> route described in [Appendix A](#appendix-a--point-to-site-vpn-optional) +> (Grafana port-forward, `psql`). It defaults to `true` +> (`infra/main.bicep`, `param deployVpnGateway bool = true`). ### Key parameters @@ -178,73 +227,96 @@ az deployment sub show --name memex-aks-infra \ | `postgresSkuName` | `Standard_D2ds_v5` | 2 vCPU / 8 GiB GeneralPurpose; bump for more DB headroom | | `postgresHighAvailability` | `true` | zone-redundant hot standby in a 2nd AZ | ---- - -## Step 2 β€” Reach the private API server (P2S VPN) +### Grant AcrPull on a SHARED registry (cross-RG, out-of-band) -Because the cluster is private, `kubectl` only works from inside the VNet. The -P2S VPN attaches your laptop to the VNet; the linked private DNS zone then -resolves the API server FQDN to its private IP. - -### 2a. Create the P2S certificates (cert-based auth) +A per-deployment ACR (`infra/modules/acr.bicep`) is granted in-bicep and needs +nothing here. A **shared** registry in another resource group cannot be granted +from a deployment scoped to this one, so two role assignments are made +out-of-band β€” and **both** are needed, for different readers: ```bash -# Root CA -openssl genrsa -out p2sRoot.key 2048 -openssl req -x509 -new -nodes -key p2sRoot.key -subj "/CN=Memex-P2S-Root" -days 3650 -out p2sRoot.crt - -# Client cert signed by the root -openssl genrsa -out p2sClient.key 2048 -openssl req -new -key p2sClient.key -subj "/CN=Memex-P2S-Client" -out p2sClient.csr -openssl x509 -req -in p2sClient.csr -CA p2sRoot.crt -CAkey p2sRoot.key -CAcreateserial -days 365 -out p2sClient.crt - -# Base64 of the ROOT public cert (single line, no PEM headers) β€” feed to Bicep -openssl x509 -in p2sRoot.crt -outform der | base64 -w0 ; echo +# 1. The cluster KUBELET β€” this is what pulls the container images. +KUBELET=$(az aks show -g -n \ + --query identityProfile.kubeletidentity.objectId -o tsv) +az role assignment create --assignee-object-id $KUBELET --assignee-principal-type ServicePrincipal \ + --role AcrPull --scope $(az acr show -n --query id -o tsv) + +# 2. The portal WORKLOAD IDENTITY β€” this is what the in-pod self-updater uses to +# LIST tags on the registry. Provisioned by infra/modules/portal-identity.bicep and +# federated to system:serviceaccount::memex-portal-sa for every portalNamespaces entry. +PORTAL_MI=$(az identity show -g -n --query principalId -o tsv) +az role assignment create --assignee-object-id $PORTAL_MI --assignee-principal-type ServicePrincipal \ + --role AcrPull --scope $(az acr show -n --query id -o tsv) + +# Wire its clientId into selfUpdate.azureClientId for each env (the same value everywhere): +az deployment sub show --name memex-aks-infra \ + --query properties.outputs.portalIdentityClientId.value -o tsv ``` -You can either: +> Pure-IaC alternative to both grants: deploy with `grantSharedAcrPull=true` for +> the portal UAMI β€” it needs *User Access Administrator* on the shared registry's +> resource group. See +> [DeploymentAKS β†’ Portal self-update](../../src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md) +> and [SELF-UPDATE.md](SELF-UPDATE.md). -- paste that base64 string into `vpnClientRootCertData` and redeploy infra, **or** -- upload it after the fact without redeploying: +### The Postgres connection string β€” use the FQDN, keep the password + +Take the server's **fully-qualified name** from the `postgresFqdn` output, not a +private IP: ```bash -az network vnet-gateway root-cert create \ - --resource-group --gateway-name -vpngw \ - --name P2SRootCert --public-cert-data "" +MEMEX_PG_CONN='Host=.postgres.database.azure.com;Port=5432;Username=memexadmin;Password=;Database=memex;SslMode=Require;Trust Server Certificate=true' ``` -### 2b. Download + connect the VPN client +🚨 A private IP used to be recommended here to dodge the portal's Entra-token +branch. That was a workaround for a bug that no longer exists, and it is now +simply wrong β€” a VNet-injected Flexible Server resolves its FQDN to the private +IP through the linked private DNS zone anyway. The rule lives in code +(`src/MeshWeaver.Hosting.PostgreSql/AzurePostgres.cs`): the token path is taken +only when the **host** ends in `.postgres.database.azure.com` **AND the string +carries no password**. FQDN + password therefore takes the plain-Npgsql path, by +design and pinned by +`test/MeshWeaver.Hosting.PostgreSql.Test/AzurePostgresAuthSelectionTests.cs` +(`AzureHostWithPassword_DoesNotUseManagedIdentity`). Leave the password out and +you get managed-identity auth; that is the switch, not the host spelling. -```bash -az network vnet-gateway vpn-client generate \ - --resource-group --name -vpngw --output tsv -``` +--- -Download the returned zip, install the profile (the **Azure VPN Client** or -OpenVPN; the bundle ships an OpenVPN `.ovpn` you augment with `p2sClient.crt` + -`p2sClient.key`), and connect. +## Step 2 β€” Reach the private API server (`az aks command invoke`) -### 2c. Get credentials and run kubectl +The API server has a private IP only, so `kubectl` cannot reach it from outside +the VNet. **`az aks command invoke` is the primary path and the only one any +automation here uses**: it runs a command inside a transient pod on the cluster, +so no tunnel, no jumpbox, and no network line-of-sight are required. ```bash -az aks get-credentials --resource-group --name -kubectl get nodes # resolves the PRIVATE API server over the tunnel +az aks command invoke -g -n --command "kubectl get pods -A" + +# --file ships local files into that pod β€” this is how every script here runs: +az aks command invoke -g -n --command "bash deploy.sh" --file . ``` -If `kubectl` times out: confirm the VPN is connected, that the private DNS zone -`privatelink..azmk8s.io` is linked to the VNet (it is, via -`network.bicep`), and that your client gets a `172.16.201.x` address. +It needs the `Azure Kubernetes Service Cluster User` role plus +`Microsoft.ContainerService/managedClusters/runcommand/action`, and nothing on +the network side. + +**When to use which** -### Alternatives to the P2S VPN (not implemented here, but supported) +| You want to… | Use | +|---|---| +| Apply manifests, run helm, read logs, patch a deployment | **`az aks command invoke`** β€” works from anywhere, and is what CI does | +| Run the scripts in `scripts/` | **`az aks command invoke … --file`** (they are written for it) | +| `kubectl port-forward` to Grafana, or `psql` straight at the private Postgres | a **route into the VNet**: the optional [P2S VPN](#appendix-a--point-to-site-vpn-optional), or a Bastion jumpbox | +| A native local `kubectl` experience with tab-completion and long-running watches | the optional [P2S VPN](#appendix-a--point-to-site-vpn-optional) | -- **`az aks command invoke`** β€” runs a command/`kubectl`/`helm` inside a - transient pod on the cluster; no network line-of-sight needed. Great for CI: - `az aks command invoke -g -n --command "kubectl get pods -A"`. -- **Jumpbox + Azure Bastion** β€” the `AzureBastionSubnet` is already carved out; - deploy Bastion + a small VM in the VNet and run kubectl from there. +Both are supported; the VPN is for *interactive* tooling that needs to open a +socket into the VNet, and nothing automated depends on it. `check-chart-drift.sh` +makes the split explicit β€” it takes `--via kubectl|aks-invoke`, and its only CI +caller (`.github/workflows/chart-drift.yml`) passes `aks-invoke`. -The P2S VPN is implemented because it gives operators a native local `kubectl` -experience without a standing VM. +> **Jumpbox + Azure Bastion** is the third option: the `AzureBastionSubnet` is +> already carved out by `network.bicep`; deploy Bastion + a small VM in the VNet +> and run kubectl from there. Not implemented in this sample. --- @@ -256,11 +328,34 @@ linux/arm64): the release path builds the base with `docker buildx --platform linux/amd64,linux/arm64 --push` and the app images with `-p:RuntimeIdentifiers="linux-x64;linux-arm64" -p:ContainerRuntimeIdentifiers="linux-x64;linux-arm64"` (no `-r linux-x64`), so one tag is an OCI image index that serves both architectures. -Two options: + +### The keys that actually set the image + +🚨 **`values.aks.yaml` carries an `image:` block (`registry` / `portal` / +`migration` / `tag` / `pullPolicy`) that NOTHING READS.** No template under +`../helm/templates/**` references `.Values.image` β€” `grep -rn '\.Values\.image' +deploy/helm/` returns nothing. Setting it changes exactly zero bytes of rendered +YAML, which is why `scripts/deploy.sh` still repoints the running Deployments +with `kubectl set image` after `helm upgrade`. The keys the chart *does* consume: + +| Key | Consumed by | Chart default (`../helm/values.yaml`) | +|---|---|---| +| `portal.image` | `../helm/templates/memex-portal/deployment.yaml` | `ghcr.io/systemorph/memex-portal-ai:latest` | +| `migration.image` | `../helm/templates/memex-migration/job.yaml` | `ghcr.io/systemorph/memex-migration:latest` | +| `portalNext.image` | `../helm/templates/memex-portal/portal-next.yaml` | *(only when `portalNext.enabled`)* | + +Each is a **full reference including the tag** β€” registry, repository and tag in +one string, not a registry prefix that gets composed with something else. + +⚠️ Prefer setting these over `kubectl set image`: a later `helm upgrade` +re-renders the Deployment from `portal.image` and silently reverts anything +`kubectl set image` had pointed it at. + +Three options: **Option A β€” pull from GHCR directly** (simplest; needs node egress to ghcr.io, -which the default `outboundType: loadBalancer` provides). Keep -`image.registry: ghcr.io/systemorph` in `values.aks.yaml`. +which the default `outboundType: loadBalancer` provides). The chart defaults +already point there β€” nothing to set. **Option B β€” import into the private ACR** (recommended for a locked-down cluster; AcrPull is already granted to the kubelet identity): @@ -280,15 +375,61 @@ az acr import --name $ACR --source ghcr.io/systemorph/memex-portal-ai-base:lates > architecture. (This is the same server-side copy `release-images.yml` uses to > mirror released images from GHCR into ACR.) -Then set `image.registry: .azurecr.io` in `values.aks.yaml` (and, since -the generic chart hardcodes the GHCR path in its templates, repoint the running -Deployments with `kubectl set image` or extend the chart to read -`.Values.image.registry`). +Then point the **real** keys at the ACR β€” on the `helm` command line, or (better, +so it survives every upgrade) in your values overlay: + +```bash +helm upgrade --install memex ../helm -f ../helm/values.yaml -f values.aks.yaml -n $NS \ + --set portal.image=.azurecr.io/memex-portal-ai:latest \ + --set migration.image=.azurecr.io/memex-migration:latest +``` + +**Option C β€” build the images yourself** (what the release path does; needs +`docker` + the .NET 10 SDK). Images are **multi-arch** so one tag serves x86 +cloud nodes AND Apple-silicon local k3s, and every install can pull-and-self-update +natively. ⚠️ Unlike the `kubectl`/`helm` commands elsewhere in this file, these +build paths are **repository-root-relative** β€” run them from the repo root, not +from `deploy/aks`: + +```bash +# Base image (node + Claude Code + Copilot CLIs) β€” MULTI-ARCH manifest list. +# @anthropic-ai/claude-code is pure JS (arch-independent); @github/copilot resolves its +# per-platform binary via npm optional deps, so each arch's build bakes in the matching CLI. +az acr build --registry --image memex-portal-ai-base:latest \ + --platform linux/amd64 --platform linux/arm64 deploy/base-images/portal-ai +# (equivalent local build: `docker buildx build --platform linux/amd64,linux/arm64 \ +# -t .azurecr.io/memex-portal-ai-base:latest --push deploy/base-images/portal-ai`) + +# App images β€” drop `-r linux-x64`; set RuntimeIdentifiers + ContainerRuntimeIdentifiers to both +# RIDs. With no single RID, the SDK (>= 8.0.405; we're on .NET 10) publishes per-RID and combines +# them into an OCI Image Index (manifest list). ContainerRuntimeIdentifiers MUST be a subset of +# RuntimeIdentifiers (set them equal). The arm64 leg layers on the multi-arch base above. +az acr login --name +dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj \ + -c Release --no-self-contained -t:PublishContainer -p:PublishProfile= \ + -p:RuntimeIdentifiers="linux-x64;linux-arm64" -p:ContainerRuntimeIdentifiers="linux-x64;linux-arm64" \ + -p:ContainerRegistry=.azurecr.io -p:ContainerRepository=memex-portal-ai \ + -p:ContainerImageTag=latest -p:ContainerBaseImage=.azurecr.io/memex-portal-ai-base:latest +dotnet publish memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj \ + -c Release --no-self-contained -t:PublishContainer -p:PublishProfile= \ + -p:RuntimeIdentifiers="linux-x64;linux-arm64" -p:ContainerRuntimeIdentifiers="linux-x64;linux-arm64" \ + -p:ContainerRegistry=.azurecr.io -p:ContainerRepository=memex-migration -p:ContainerImageTag=latest +``` + +> **First multi-arch roll β€” order matters:** the multi-arch base +> (`memex-portal-ai-base:latest`) must exist **before** the first multi-arch app +> build, or the arm64 leg has no base layer. Rebuild the base multi-arch once +> (the `az acr build … --platform …` above), then the app builds β€” and the +> continuous self-update path β€” work for both architectures. --- ## Step 4 β€” Install the portal (Helm + manifests) +Everything below runs **inside the cluster** via `az aks command invoke` (Step 2). +`scripts/deploy.sh` is the packaged form of exactly these steps β€” see +["Or run `scripts/deploy.sh`"](#or-run-scriptsdeploysh) at the end of this step. + ```bash NS=memex kubectl create namespace $NS @@ -309,17 +450,51 @@ helm install memex ../helm \ --set secrets.memex_migration.memex_postgres_password='' \ --set secrets.memex_portal.memex_postgres_password='' -# 3) Bind Postgres to its PVC, then scale the portal out + go RWX -kubectl patch statefulset memex-postgres-statefulset -n $NS \ - --type strategic --patch-file manifests/postgres-pvc-patch.yaml -kubectl patch deployment memex-portal-deployment -n $NS \ - --type strategic --patch-file manifests/portal-ha-patch.yaml - -# 4) Ingress with cookie session affinity (enable a controller first) +# 3) Ingress with cookie session affinity (enable a controller first) az aks approuting enable -g -n # managed nginx kubectl apply -n $NS -f manifests/portal-ingress.yaml ``` +### 🚨 Do NOT apply `portal-ha-patch.yaml` β€” the chart owns both halves now + +Older revisions of this document had a step 3 that ran + +```bash +kubectl patch deployment memex-portal-deployment -n $NS \ + --type strategic --patch-file manifests/portal-ha-patch.yaml # ← DO NOT +kubectl patch statefulset memex-postgres-statefulset -n $NS \ + --type strategic --patch-file manifests/postgres-pvc-patch.yaml # ← DO NOT +``` + +Both are now the **chart's** job, and re-applying them is not merely redundant: + +- **The RWX volumes are rendered by the chart.** `persistence:` in + `values.aks.yaml` supplies a `claimName` per drive and + `../helm/templates/memex-portal/deployment.yaml` mounts each one directly, so a + plain `helm upgrade` gets `/data`, `/mnt/content`, `/mnt/users` right. That + replaced the bolt-on patch precisely because **any unpatched apply silently + reverted `/data` to `emptyDir`**, wiping the assembly / NuGet / DataProtection + caches on every restart in the gap (`values.aks.yaml`, the `persistence:` + comment; the same warning heads `scripts/deploy.sh`). +- **`replicas` belongs to the autoscaler.** `portal-ha-patch.yaml` writes + `spec.replicas: 2`. KEDA is enabled in this overlay (`keda.enabled: true`, + `minReplicas: 2`), and the chart therefore **omits `spec.replicas` entirely** + under KEDA so the HPA is the single writer. Patching it back in means helm and + the HPA fight over one field, and every `helm upgrade` yanks a scaled-out + deployment back down until the HPA pushes it out again. This is invariant #1 of + `scripts/check-chart-invariants.sh` β€” *"spec.replicas is ABSENT when a + ScaledObject exists"* β€” a gate that exists because the chart described exactly + this contradiction in git for a month and produced a **production 503 on + 2026-08-14**. Writing the patch re-creates the bug the gate was built to catch. +- **Postgres has its own `volumeClaimTemplate`.** `postgres-pvc-patch.yaml` was + written when the chart used `emptyDir` for the data dir; the StatefulSet now + declares `volumeClaimTemplates: memex-pgdata` (RWO, 10Gi). And on the default + path here the in-cluster Postgres is not used at all β€” see the Database section. + +What the patch file still holds that the chart does not is the zone-spread +`topologySpreadConstraints`. If you want that, put it in your values overlay or a +chart change β€” do not hand-patch the Deployment. + > **Secrets**: never commit a real password. The CSI Secrets Store add-on is > enabled in `aks.bicep` β€” wire `secrets.memex_*` to Key Vault via a > `SecretProviderClass` for production rather than `--set`. @@ -329,6 +504,81 @@ kubectl apply -n $NS -f manifests/portal-ingress.yaml > "Attempting to reconnect…" loops. The annotations are in > `manifests/portal-ingress.yaml` (nginx today, AGIC commented). +### Or run `scripts/deploy.sh` + +The same sequence, packaged for the private cluster. Run it from a staging +directory holding the script, `aks-extras.yaml` (StorageClass + PVCs), your +`values.deploy.yaml` secrets (from `values.deploy.example.yaml` β€” keep it OUT of +git), a copy of the chart as `./helm`, and a copy of `values.aks.yaml`: + +```bash +az aks approuting enable -g -n # managed nginx (public LB) +cd deploy/aks/scripts +export MEMEX_PG_CONN='Host=.postgres.database.azure.com;Port=5432;Username=memexadmin;Password=;Database=memex;SslMode=Require;Trust Server Certificate=true' +az aks command invoke -g -n \ + --command "MEMEX_PG_CONN='$MEMEX_PG_CONN' bash deploy.sh" --file . +``` + +It creates the namespace, applies the StorageClass + RWX PVCs, runs +`helm upgrade --install` with the chart + `values.aks.yaml` + `values.deploy.yaml`, +scales the chart's in-cluster Postgres to 0 (this deployment uses the Flexible +Server), repoints both images at the shared ACR, and **patches the +connection-string secret** β€” a chart-gen gap: the generated secret template +hardcodes the in-cluster Postgres host. It does **not** patch volumes any more, +for the reason above. + +**Observability is folded in**: export `GRAFANA_PW=…` alongside `MEMEX_PG_CONN` +and `deploy.sh` also brings up Grafana + Loki + Prometheus (see Observability); +omit it to skip monitoring. At the model level, `AddMemex`'s `OtlpEndpoint` +option wires `OTEL_EXPORTER_OTLP_ENDPOINT` for OTLP traces/metrics β€” not needed +for log shipping, since Promtail scrapes stdout. + +--- + +## Step 4b β€” Public ingress + TLS + DNS + +Point DNS at the ingress controller's public IP, then let cert-manager issue the +certificate over HTTP-01: + +```bash +IP=$(az aks command invoke -g -n \ + --command "kubectl get svc -n app-routing-system nginx -o jsonpath='{.status.loadBalancer.ingress[0].ip}'") +az network dns record-set a add-record -g -z -n --ipv4-address $IP --ttl 300 +cd deploy/aks/scripts +az aks command invoke -g -n --command "bash tls.sh" --file tls.sh # cert-manager + Let's Encrypt + ingress +``` + +HTTPβ†’HTTPS redirect is automatic once the ingress has TLS. Verify while bypassing +the DNS cache: + +```bash +curl -sS -o /dev/null -w "%{http_code} verify=%{ssl_verify_result}\n" \ + --resolve :443:$IP https:/// +``` + +### Default SSL certificate (cluster-wide, one-time) + +Without it, any client that connects **without SNI** gets the self-signed +"Kubernetes Ingress Controller Fake Certificate" β€” and corporate TLS-inspection / +URL-categorization appliances probe exactly that way, then flag the whole domain +as insecure and block it for their users (seen 2026-08: a client's IT blocked a +portal host in Firefox *and* Edge over this). + +Point the app-routing controller's default cert at the flagship host's +cert-manager secret β€” patch the `NginxIngressController` **CR**, *not* the nginx +deployment (the addon operator reverts direct deployment edits): + +```bash +az aks command invoke -g -n --command \ + "kubectl patch nginxingresscontroller default --type merge -p '{\"spec\":{\"defaultSSLCertificate\":{\"secret\":{\"name\":\"\",\"namespace\":\"\"}}}}'" +# verify from outside β€” must show the real cert, not "Acme Co": +echo | openssl s_client -connect :443 -noservername 2>/dev/null | openssl x509 -noout -subject +``` + +The setting survives addon updates but is **NOT** re-created on a cluster +rebuild β€” re-apply it whenever the cluster (or the `NginxIngressController` CR) +is recreated. + --- ## Storage drives β€” mountable Azure Files at explicit `/mnt` paths @@ -524,10 +774,70 @@ Three properties worth knowing before you operate it: absence means off, never open). No repository routed β‡’ the control plane idles rather than running agent rounds that could never end in a ticket. -Deploy steps, verification table, and how to turn it off: **[DEPLOY-RUNBOOK.md Β§ 6b](DEPLOY-RUNBOOK.md)**. Design and configuration reference: **[LogWatchTriage.md](../../src/MeshWeaver.Documentation/Data/Architecture/LogWatchTriage.md)**. +#### Deploy it β€” four steps, and it stays off until all four are done + +The ingest endpoint is not even mapped without a token, so a half-configured +watcher is inert rather than half-working. + +```bash +# 1. ONE shared secret, both sides. The watcher presents it; the portal requires it. +TOKEN=$(openssl rand -hex 32) +az aks command invoke -g -n --command \ + "kubectl -n monitoring create secret generic mw-log-watcher --from-literal=ingest-token=$TOKEN" +# …then set LogWatch__IngestToken to the SAME value on the portal (KeyVault β†’ its secret store). + +# 2. Where tickets go. Without at least one route the control plane idles by design β€” +# it will not spend agent rounds on incidents it could never file. +# LogWatch__DefaultRepository = / +# LogWatch__Routes__0__Prefix = MeshWeaver. +# LogWatch__Routes__0__Repository = / +# LogWatch__Routes__1__Prefix = Memex. +# LogWatch__Routes__1__Repository = / +# Every key carries the LogWatch__ prefix β€” a bare Routes__0__Repository does not bind, and an +# unbound route reads exactly like "no route configured": incidents pile up at New, silently. +# Issues are opened as the GitHub App (GitHub:App), never a user's OAuth token. + +# 3. Build + push the watcher image. +dotnet publish tools/MeshWeaver.LogWatcher/MeshWeaver.LogWatcher.csproj -c Release \ + -t:PublishContainer -p:ContainerRegistry=.azurecr.io \ + -p:ContainerRepository=memex-log-watcher -p:ContainerImageTag= + +# 4. Apply the Deployment + PVC (edit the image tag + watched namespaces in the manifest first). +az aks command invoke -g -n \ + --command "kubectl apply -f log-watcher.yaml" \ + --file manifests/observability/log-watcher.yaml +``` + +**Verify β€” and know what each failure looks like:** + +```bash +az aks command invoke -g -n --command \ + "kubectl -n monitoring logs deploy/mw-log-watcher --tail=50" +``` + +| What the log says | Meaning | +|---|---| +| `Loki: N red line(s) …` then `Reported … β€” 200` | Working end to end. | +| `Log watcher is not configured` | Deploy step 1 missed β€” `PortalUrl`/`IngestToken` unset. | +| `Portal REJECTED report … with 401` | The two tokens differ β€” the watcher's secret and the portal's `LogWatch__IngestToken` must be the same string. | +| `Portal REJECTED report … with 404` | The portal's `LogWatch__IngestToken` is unset, so `/api/log-incidents` is not mapped at all β€” deploy step 1 was done on the watcher side only. | +| `Loki: 0 line(s)` forever | The namespace label is wrong, or Promtail is not shipping that namespace. | +| `Could not persist watcher state … Access to the path … is denied` | The state PVC mounted root-owned. The pod needs `securityContext.fsGroup: 1654` (the shipped manifest sets it). The watcher still reports, but the cursor stops persisting, so a restart replays the lookback window. | +| Incidents appear but stay `New` | Deploy step 2 missed β€” no repository routed, so the control plane idles. | + +Then browse `Admin/_LogIncident` in the portal: every incident links to the ticket +it opened and to the triage thread that wrote it. + +**🚨 The state directory must be a real volume.** On an `emptyDir` a pod restart +replays the lookback window and re-reports. The manifest ships a PVC for this reason. + +**To turn it off**: `kubectl -n monitoring scale deploy/mw-log-watcher --replicas=0`. +Clearing the portal's `LogWatch__IngestToken` also closes the ingest endpoint. +Neither deletes existing incidents. + A provisioned Grafana alert rule covers the human-facing half β€” it shows red-log volume in Grafana but deliberately does **not** open tickets. Ticketing hangs off the watcher's cursor instead, because an alert notification that fires while its receiver is down is simply lost: acceptable for a nudge, @@ -551,6 +861,33 @@ kubectl apply -n $NS -f manifests/observability/otel-collector.yaml kubectl rollout restart deployment memex-portal-deployment -n $NS ``` +### Grafana + Loki + Prometheus (log search and dashboards) + +The file archive above is the durable, cheap sink. For *searching* logs there is a +`grafana/loki-stack` install (Loki + Promtail + Grafana + Prometheus, datasources +auto-wired, Promtail shipping every pod's stdout into Loki) in the `monitoring` +namespace: + +```bash +export GRAFANA_PW='' +cd deploy/aks/scripts +az aks command invoke -g -n \ + --command "GRAFANA_PW=$GRAFANA_PW bash install-observability.sh" --file install-observability.sh +``` + +Everything except the portal stays private, so Grafana has **no public endpoint**. +Reach it over a route into the VNet β€” the optional +[P2S VPN](#appendix-a--point-to-site-vpn-optional) or a Bastion jumpbox β€” and +port-forward: + +```bash +az aks get-credentials -g -n # VPN connected +kubectl -n monitoring port-forward svc/loki-grafana 3000:80 # http://localhost:3000 (admin / $GRAFANA_PW) +``` + +In Grafana β†’ Explore β†’ Loki, the portal logs are `{namespace="memex"}` (add +`|= "error"` or `|~ "signin-microsoft"` to narrow). + ### Read / download the archived logs The archive lives on the `otel-logs` Azure Files share. Inspect from a pod: @@ -727,9 +1064,23 @@ replicas** (the `portal-ha-patch.yaml` already sets 3). - **You still must create the app registrations / OAuth clients** and fill the `CHANGE_ME_*` `ClientId`s (config) + `ClientSecret`s (secrets) β€” those are real credentials this repo does not contain. Register each redirect URI above. -- Host is `portal.example.com` (ingress + TLS). Point a DNS A/CNAME at the - ingress controller's IP and issue the `memex-tls` cert (cert-manager or a - pre-created secret). +- Host is `portal.example.com` (ingress + TLS) β€” Step 4b points DNS at the + ingress controller's IP and issues the certificate. + +### Create the app registrations / OAuth clients + +**Microsoft / Entra** (single-tenant home directory): + +```bash +az ad app create --display-name "Memex Portal (portal.example.com)" \ + --sign-in-audience AzureADMyOrg \ + --web-redirect-uris "https://portal.example.com/signin-microsoft" +az ad app credential reset --id --display-name aks --years 1 # => client secret +``` + +**Google** (Cloud Console) and **LinkedIn** (Developer portal): create web OAuth +clients with redirect URIs `https://portal.example.com/signin-google` and +`https://portal.example.com/signin-linkedin`. --- @@ -856,6 +1207,208 @@ Postgres endpoint. --- +## Operating it β€” rolling a new image, and the traps + +```bash +# a) The manifest MUST carry every arch leg. A partial manifest list = ImagePullBackOff on the +# missing arch, which presents as "the deploy hung" (memex-cloud V46 outage, 2026-07-19). +az acr manifest show -r -n memex-portal-ai: \ + | jq -r '.manifests[]?.platform | "\(.os)/\(.architecture)"' # expect linux/amd64 AND linux/arm64 + +# b) Roll. +az aks command invoke -g -n --command \ + "kubectl set image deploy/memex-portal-deployment memex-portal=.azurecr.io/memex-portal-ai: -n \ + && kubectl rollout status deploy/memex-portal-deployment -n --timeout=600s" + +# c) Verify with REAL signals, looped. HTTP 200 is the Blazor shell and proves nothing. +for i in $(seq 1 10); do curl -s -o /dev/null -w "%{http_code} " https:///api/content//content/; done +``` + +### 🧱 NodeType bake β€” the deploy MUST compile every dynamic NodeType + +Fresh pods invalidate every dynamic NodeType's cached assembly (the framework MVID +changed), so they all need a recompile. Left lazy, that compile happens on user +requests after the pod is already serving β€” and a type nothing happens to touch +stays **"no definition"** until its pages hang with *"No response received … +`SubscribeRequest` β†’ target X"* (SocialMedia/Post on memex-cloud, 2026-07-30: +every post page burned the 60 s timeout all morning while the portal looked +healthy). **Managed envs therefore run the bake ON β€” these three knobs travel +together** (`values.aks.yaml` carries them; keep them in every env overlay so a +`helm upgrade` never reverts them): + +| Knob | What it does | +|---|---| +| `config.memex_portal.PreWarm__DynamicTypes: "true"` | every new pod sweeps + compiles ALL dynamic NodeTypes at start (resumes from the shared `/data` cache β€” warm restarts are cheap) | +| `config.memex_portal.PreWarm__GateReadiness: "true"` | `/health` stays red until the sweep is green; with `maxSurge 1 / maxUnavailable 0` a regressed type STALLS the rollout with the old image serving. **⚠️ Depends on the sweep, and `values.aks.yaml` currently sets BOTH to `"false"` β€” read the namespace, not this table (#1981).** The two keys are ONE setting: the gate reads state only the sweep writes, so gate-without-sweep is disarmed at startup (it used to register and stay permanently green) and logged at Critical. Turning the gate on means turning `PreWarm__DynamicTypes` on in the same change. It was tried 2026-08-02 and reverted the same day on 7 FALSE regressions β€” all cross-silo `SubscribeRequest` timeouts, not compile errors (#694 residue). The gate no longer reads "no answer" as "it broke": a `TimedOut` outcome is filed as *unevaluated* and can never gate, and that leniency now survives the cascade (a dependent of an unevaluated upstream is `UpstreamUnevaluated`, also non-gating). Only a `CompileError` β€” or an `UpstreamFailed` cascading from one β€” on a **previously-healthy** type stalls a roll | +| `config.memex_portal.PreWarm__AllowUnprovenBake: "false"` | **βœ… OFF (strict).** The gate also refuses readiness when the sweep *errored* β€” enumeration threw or timed out β€” because such a pod verified **nothing** and a gate that certifies "I verified nothing" is worse than no gate. That guard used to live in the pre-run bake Job (*"FINDING NOTHING IS NOT PASSING"*, exit 3, `Bake__AllowEmpty`) and was lost when #1357 retired the Job; it is now enforced on the surviving path as `BakePhase.Faulted`. ⚠️ **"Empty" is not "unproven"** β€” a mesh that genuinely has no dynamic NodeTypes enumerates fine, completes and serves; only the *inability to get an answer* gates. Set `"true"` only to roll forward past an environment that cannot answer the enumeration, accepting lazy compilation. It can never waive a real regression, and `/health` keeps reporting the bake as unproven | +| `probes.startup: {periodSeconds: 10, failureThreshold: 180}` (= 30 min) | ⚠️ REQUIRED with the gate: a cold bake is **~2.4 s/type**, sequential *(measured 2026-08-10, prod Loki, three portals)* β€” ~10 min on memex-cloud, the largest mesh β€” and the default 5 min budget kills the pod mid-bake forever. 30 min is that worst case plus a plain cold boot, x2. `progressDeadlineSeconds` is DERIVED from these two in the chart, so raising them can't leave it behind. **Was `1080` (3 h)** until 2026-08-10, sized from a "~90 s/type" estimate that was 37x too high β€” a window that long detected nothing | + +**🚨 Before you trust the gate, verify the namespace actually reads it.** The gate +protects a portal through exactly two deployment facts, and on 2026-08-10 two of +the three portals had drifted away from both. A gated config in a namespace +missing either is *false confidence* β€” it reports and the outage happens anyway. + +| Must be true | Why | Check | +|---|---|---| +| a `startupProbe` exists **on `/health`** | that probe is the ONLY reader of the gate; no startupProbe (or one on `/alive` / `/healthz`) ignores it entirely | `kubectl -n get deploy memex-portal-deployment -o jsonpath='{.spec.template.spec.containers[0].startupProbe}'` | +| `strategy.maxUnavailable: 0` | surge-first keeps the old pod serving until the new one passes; at `maxUnavailable:1` with `replicas:1` the only serving pod can be deleted BEFORE the replacement is ready | `kubectl -n get deploy memex-portal-deployment -o jsonpath='{.spec.strategy.rollingUpdate}'` | + +Both are rendered correctly by the chart β€” a `helm upgrade` restores them. A +per-env `portal-patch.json` or a hand `kubectl patch` can still override them, +which is how the drift happened. + +πŸͺ¦ **There is no pre-run bake Job any more (#1347) β€” the CI bake replaced it +(#1660 WS3).** The separate `memex-bake` image was removed after two weeks of +running in zero namespaces. On its only AKS run (memex-cloud, 2026-07-30) +`memex-bake:3.0.0-ci.1565` computed a **different framework fingerprint** than the +running `portal-ai:3.0.0-ci.1565` β€” same version, same commit, separately +published β€” so its framework-stale kickoff started flipping CURRENT NodeType +records to `Pending` and rebuilding them for a framework nothing serves. The cause +was the per-build `+build.` stamp `InformationalVersion` then carried +under `CIRun`: every `dotnet publish` minted a fresh `MeshWeaver.Graph` MVID. +#1660 WS3 removed that stamp and made CI builds deterministic β€” the framework +identity is now the **API-surface hash** (`s`, `FrameworkBuildIdentity`; +identical across CI builds AND across internal-only merges, moved only by breaking +surface changes or Graph changes) β€” so the Build-and-Test run's bake artifact IS +adoptable, and usually already published: `main-cd`'s `publish-bake` job copies it +onto the portals' `/data` share (`prebuilt-bundles///`), and each +booting pod seeds its own identity's bundles (`PreWarm__PrebuiltBundleRoot`) before +its sweep. + +**The pod-side sweep remains the enforcement.** It runs in the serving process, +adopts whatever CI published, and compiles the remainder β€” 76 s for 280 types cold +(memex-cloud, batch direct-compile), seconds when the CI bake covered the shipped +content. The "fail before prod" contract is given by `PreWarm__GateReadiness` + +`maxSurge:1` / `maxUnavailable:0`: the new pod refuses readiness until its OWN bake +is green while the old image keeps serving. + +**Verify after every roll** β€” the bake completing is the deploy signal, not HTTP 200: + +```bash +# The sweep's verdict (compiled=N alreadyBaked=M compileErrors=0 …) β€” read the NEWEST pod +# explicitly: `kubectl logs deploy/…` picks an arbitrary pod and mid-rollout that is often the OLD one. +NEW=$(kubectl -n get pods -l app.kubernetes.io/component=memex-portal \ + --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1:].metadata.name}') +kubectl -n logs "$NEW" | grep "DynamicTypePreWarmer: warm-up complete" +# The gate's verdict (Healthy "baked in …" / Unhealthy "regressed" β€” rollout stalled on purpose): +kubectl -n get pods # new pod 0/1 while baking is CORRECT; investigate only a Regressed log line +``` + +Live-env equivalent without a helm apply (what enabled memex-cloud on 2026-07-30): + +```bash +kubectl -n patch configmap memex-portal-config --type merge \ + -p '{"data":{"PreWarm__DynamicTypes":"true","PreWarm__GateReadiness":"false"}}' +kubectl -n patch deployment memex-portal-deployment --type json -p \ + '[{"op":"replace","path":"/spec/template/spec/containers/0/startupProbe/periodSeconds","value":10}, + {"op":"replace","path":"/spec/template/spec/containers/0/startupProbe/failureThreshold","value":180}]' +# the probe patch rolls the deployment; the new pod bakes behind the gate while the old serves +``` + +- **A pod sitting 0/1 for a long time is the gate doing its job** while a cold bake + runs; only a `REFUSING READINESS … Regressions:` log line means a type broke on + this image β€” fix the type, never widen the timeout. +- **Retries are cheap β€” the sweep resumes from the shared cache.** Every type a + sweep DID build is on `/data` for good; deleting a gate-red pod re-runs only what + is missing. Progress across attempts is monotonic. +- **⚠️ Known flake while core #694 is open (cross-silo reply routing):** during a + roll the mesh briefly runs TWO silos (old serving + new baking), and in that + window the pod-side sweep's shared-source resolution can fail + nondeterministically β€” the symptom is a `CompileError` full of unresolved names + that live in a SIBLING type's `Source/` (`shared=@…`), on a type that compiles + clean when triggered individually. First verify with a targeted compile (MCP + `compile @` β€” recycle the type's hub first if the subscribe times out; the + targeted subscribe rides the SAME cross-silo routing, so it completes reliably + only in a single-silo window β€” delete the baking pod and use the ~2 minutes before + its replacement joins), then let the replacement's re-sweep find the fixed types + `alreadyBaked`. (Observed live on the first gated roll, memex-cloud 2026-07-30: + the gate caught 2 types with stale-green records whose newest assemblies were 6 + days old β€” real, invisible breakage β€” plus sweep failures on shared-source types + that compiled clean when triggered individually in a single-silo window.) + +Mechanism: +[NodeTypeCompilation.md](../../src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md). + +### Scaling: KEDA wins + +`kubectl scale --replicas=0` (the documented heal for a wedged mesh) is **silently +reverted** by a `ScaledObject` with `minReplicaCount: 2` β€” you conclude the heal +failed when it never ran. `kubectl get scaledobject -n ` first, and check +`PAUSED`. + +#### 🚧 A PAUSED scaler is a FENCE β€” find out why before you remove it + +`autoscaling.keda.sh/paused-replicas: ""` pins the deployment at `n`, **deletes +the HPA**, and makes `minReplicaCount` inert. Nothing in any chart sets it, +`kubectl get deploy` does not show it, and it survives every roll β€” so it reads as +"this deployment is just single-replica" long after whoever applied it has moved on. + +**Nobody applies it for fun.** It is the most direct way to say "stop making new +silos", so assume it is suppressing a multi-silo defect until you have evidence +otherwise. `memex-cloud` carried one for **16 days** (2026-07-29 β†’ 2026-08-14): it +was applied 87 minutes after the second fix for **#694 β€” *cross-silo posts lose +AccessContext: static content 500s on ~50% of requests with 2 replicas*** β€” and 29 +seconds after KEDA scaled the namespace back up. Removing it without reading that +history is re-running the incident. + +Establish provenance before unpausing β€” the same way you would for any hand-applied +field: + +```bash +kubectl -n get scaledobject -o jsonpath='{range .metadata.managedFields[*]}{.manager} {.operation} {.time}{"\n"}{end}' +kubectl -n get scaledobject -o jsonpath='{.status.lastActiveTime}{"\n"}' # when it last scaled +``` + +`managedFields` gives you the writer (`kubectl-annotate`) and the timestamp; +`lastActiveTime` immediately before it is the tell that someone watched it scale and +stopped it. Then find what shipped or broke that day (`git log --since= +--until=`, closed issues), and only unpause once you can name the defect and +point at its fix. Treat the unpause as a **monitored experiment** with a rollback +trigger, not a config tidy-up β€” a fix merged while the namespace was pinned has only +ever been exercised in the transient two-silo window of a rollout, never in steady +state. `scripts/check-chart-drift.sh` reports the annotation as CLUSTER-ONLY drift so +it stops being invisible. + +### Crash dumps: verify the MOUNT + +`DOTNET_DbgEnableMiniDump` + `DOTNET_DbgMiniDumpName=/data/dumps/…` do nothing +without a volume at that path β€” `createdump` does not create directories, so the +crash destroys its own evidence. The chart mounts a dedicated `memex-dumps` +emptyDir; verify the LIVE pod actually has it: + +```bash +kubectl get deploy memex-portal-deployment -n \ + -o jsonpath='{range .spec.template.spec.containers[0].volumeMounts[*]}{.name} -> {.mountPath}{"\n"}{end}' \ + | grep dumps || echo "NO DUMP MOUNT β€” every crash will produce nothing" +``` + +⚠️ A per-env `portal-patch.json` replaces volumes **by index**, so adding or +reordering a chart volume silently misaligns an environment and a cluster can run an +older volume set while the chart in git looks right. Diff live mounts against the +chart after every deploy. + +### Degraded-but-Ready replica β€” intermittent hangs, `kubectl top` divergence + +Symptom: some requests hang or serve garbled state while most succeed, typically +after a burst of content syncs or a scheduled bake. Cause class (2026-08-25, +MeshWeaver#2194): each NodeType publication mints a new AssemblyLoadContext and +serving instances stay on the OLD build behind the Recycle banner, so every +superseded ALC stays rooted β€” the type-hosting silos climb to tens of GB and go +GC-bound while still answering `/alive` (both post-startup probes), so Kubernetes +never pulls them from rotation. + +```bash +az aks command invoke -g -n --command "kubectl top pods -n --no-headers" +``` + +One or two pods far above their siblings in BOTH memory and CPU = this. Remedy: +`kubectl delete pod` the outliers (grace-drain; the Deployment replaces them). +Prevention: enable `Modules:AutoRecycleOnStaleBuild` (MeshWeaver#2192) so instances +converge onto each newly published build and old ALCs collect β€” mechanism in +[NodeTypeCompilation.md](../../src/MeshWeaver.Documentation/Data/Architecture/NodeTypeCompilation.md). + +--- + ## Generating this from Aspire The repo already models the deployment in @@ -889,6 +1442,47 @@ Aspire and re-run Step 4 β€” `values.aks.yaml` and `manifests/` continue to appl on top unchanged (they only reference stable resource names like `memex-portal-deployment` / `memex-postgres-statefulset`). +### Why a written procedure and not pure `aspire up` + +Aspire's Kubernetes publisher generates the **workload** chart, but it does not +provision an AKS **cluster**, a private Postgres Flexible Server, a VPN, or +Let's Encrypt. Those platform pieces are the Bicep + the steps above. The AppHost +([`../aspire/Memex.Deploy.AppHost`](../aspire/Memex.Deploy.AppHost)) owns the app +model **and the deploy parameters** β€” including the OAuth providers; this document +stitches the platform around it. All config flows from deploy parameters β†’ env. + +--- + +## Known gaps / follow-ups + +- **Dump mount not applied to the live clusters** (2026-07-28): all three + namespaces carry the dump env vars while no pod mounts `/data/dumps`, so every + production `exit=139` so far produced no dump. Applying the current chart fixes + it; until then a `mkdir /data/dumps` on the `/data` PVC is a stopgap that puts + heap-sized dumps on a shared 16Gi share instead of the size-bounded emptyDir. +- ~~**Multi-replica HA**~~ (chart side done 2026-08-14): Orleans **AdoNet** + clustering is the HA provider (`Features:Orleans:Clustering`, legacy key + `Deployment:Orleans:Clustering`), backed by a dedicated `orleans` database the + migration's `OrleansClusteringSetup` creates. The chart now templates the replica + count and omits `spec.replicas` entirely under KEDA so the HPA owns it; + `scripts/check-chart-invariants.sh` asserts the prerequisites at render time. + What remains is per-environment and per-cluster, not chart work: each env's + overlay must set `keda.enabled`, and a paused `ScaledObject` still pins the count + regardless (see "Scaling: KEDA wins" above β€” check `PAUSED` before believing a + replica number). +- ~~**Chart connection string**~~ (fixed 2026-08-14): + `../helm/templates/memex-portal/secrets.yaml` and its migration counterpart now + take `secrets.memex_{portal,migration}.ConnectionStrings__memex` / `__orleans` + from values, falling back to the in-cluster host only when unset. An env that + supplies them no longer needs the post-`helm` secret patch in `scripts/deploy.sh`. + 🚨 Until an env's values file DOES supply them, `helm upgrade` still re-renders + the in-cluster host β€” and for `__orleans` nothing patches it back, so an AdoNet + namespace loses cluster membership. The chart `fail`s the render if an external + `ConnectionStrings__memex` is supplied without a matching `__orleans`, so the + half-configured case cannot ship silently. +- **Secrets β†’ Key Vault**: move the PG password, master key, and OAuth secrets into + Key Vault via the CSI Secrets Store add-on (enabled in `infra/modules/aks.bicep`). + --- ## Teardown @@ -910,10 +1504,13 @@ az group delete --name --yes --no-wait # cluster, VPN, ACR, storage, VNe - **Secrets**: move `memex_postgres_password` out of `--set`/values into Key Vault via the CSI Secrets Store add-on (already enabled). -- **API server**: private-only; `enablePrivateClusterPublicFQDN=false`. Local - accounts stay enabled so the VPN kubeconfig works β€” consider Entra-only +- **API server**: private-only; `enablePrivateClusterPublicFQDN=false`. + `infra/modules/aks.bicep` sets `disableLocalAccounts: false` with the comment + *"keep admin kubeconfig usable over the VPN"* β€” that is the **only** thing local + accounts are for here, since `az aks command invoke` does not need them. If you + skip the VPN (`deployVpnGateway: false`), consider Entra-only (`disableLocalAccounts=true` + AKS-managed Entra RBAC) for prod. -- **VPN auth**: this sample uses certificate auth for simplicity. Entra ID +- **VPN auth**: the optional P2S uses certificate auth for simplicity. Entra ID authentication on the P2S is stronger (revocation, conditional access). - **ACR**: `publicNetworkAccess` is Enabled for first-run `az acr import`. For a fully private cluster, switch ACR to Premium + Private Endpoint and disable @@ -921,4 +1518,86 @@ az group delete --name --yes --no-wait # cluster, VPN, ACR, storage, VNe - **Egress**: default `outboundType: loadBalancer` allows node egress to GHCR / Azure. For a locked-down network use `userDefinedRouting` + Azure Firewall and Option B (ACR import) so no pull traffic leaves the VNet. + +--- + +## Appendix A β€” Point-to-Site VPN (optional) + +**You do not need this to deploy.** Step 2's `az aks command invoke` covers every +apply / helm / logs / patch operation, and it is the only path anything automated +here uses. Provision the gateway when you want a **route into the VNet** for +interactive tooling that opens its own socket: + +- `kubectl port-forward` to in-cluster Grafana (it has no public endpoint); +- `psql` straight at the private Postgres Flexible Server; +- a native local `kubectl` β€” tab-completion, long-running `watch`/`logs -f` β€” without + a standing jumpbox VM. + +The gateway is still provisioned by default (`infra/main.bicep`, +`param deployVpnGateway bool = true`, module `vpn.bicep`) and it costs 20–45 minutes +of deploy time; `deployVpnGateway: false` is supported and documented on that +parameter. Because the cluster is private, the tunnel is what lets `kubectl` resolve +the API server: the linked private DNS zone `privatelink..azmk8s.io` maps +the API server FQDN to its private IP once your laptop is attached to the VNet. + +### A1. Create the P2S certificates (cert-based auth) + +```bash +# Root CA +openssl genrsa -out p2sRoot.key 2048 +openssl req -x509 -new -nodes -key p2sRoot.key -subj "/CN=Memex-P2S-Root" -days 3650 -out p2sRoot.crt + +# Client cert signed by the root +openssl genrsa -out p2sClient.key 2048 +openssl req -new -key p2sClient.key -subj "/CN=Memex-P2S-Client" -out p2sClient.csr +openssl x509 -req -in p2sClient.csr -CA p2sRoot.crt -CAkey p2sRoot.key -CAcreateserial -days 365 -out p2sClient.crt + +# Base64 of the ROOT public cert (single line, no PEM headers) β€” feed to Bicep +openssl x509 -in p2sRoot.crt -outform der | base64 -w0 ; echo +``` + +You can either: + +- paste that base64 string into `vpnClientRootCertData` and redeploy infra, **or** +- upload it after the fact without redeploying: + +```bash +az network vnet-gateway root-cert create \ + --resource-group --gateway-name -vpngw \ + --name P2SRootCert --public-cert-data "" ``` + +⚠️ **`--public-cert-data` is read as a FILE PATH by current `az` versions** β€” pass +the path, NOT the inline base64 string and NOT `@file`. On Windows the equivalent +certificate mint is: + +```powershell +$root = New-SelfSignedCertificate -Type Custom -KeySpec Signature -Subject "CN=MemexP2SRootCert" -KeyUsage CertSign -KeyExportPolicy Exportable -CertStoreLocation Cert:\CurrentUser\My -HashAlgorithm sha256 -KeyLength 2048 +$client = New-SelfSignedCertificate -Type Custom -DnsName MemexP2SChild -KeySpec Signature -Subject "CN=MemexP2SChildCert" -Signer $root -KeyExportPolicy Exportable -CertStoreLocation Cert:\CurrentUser\My -HashAlgorithm sha256 -KeyLength 2048 -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") +[IO.File]::WriteAllText("root.txt",[Convert]::ToBase64String($root.RawData)) +# then: az network vnet-gateway root-cert create -g --gateway-name \ +# --name MemexP2SRootCert --public-cert-data root.txt +``` + +### A2. Download + connect the VPN client + +```bash +az network vnet-gateway vpn-client generate \ + --resource-group --name -vpngw --output tsv # -> download URL (zip) +``` + +Download the returned zip, install the profile (the **Azure VPN Client** or +OpenVPN; the bundle ships an OpenVPN `.ovpn` you augment with `p2sClient.crt` + +`p2sClient.key`), and connect. + +### A3. Get credentials and run kubectl + +```bash +az aks get-credentials --resource-group --name +kubectl get nodes # resolves the PRIVATE API server over the tunnel +kubectl -n monitoring port-forward svc/loki-grafana 3000:80 # what the tunnel is actually for +``` + +If `kubectl` times out: confirm the VPN is connected, that the private DNS zone +`privatelink..azmk8s.io` is linked to the VNet (it is, via `network.bicep`), +and that your client gets a `172.16.201.x` address. diff --git a/deploy/aks/SELF-UPDATE.md b/deploy/aks/SELF-UPDATE.md index b5c84ddfb3..d6d2a05586 100644 --- a/deploy/aks/SELF-UPDATE.md +++ b/deploy/aks/SELF-UPDATE.md @@ -30,7 +30,7 @@ PATCH). Wired by `AddSelfUpdate()` in `MemexConfiguration.cs`. misses the whole `/data` assembly cache by design, and a lazily-compiled mesh leaves any un-visited type hanging its pages ("no definition", 60 s `SubscribeRequest` timeouts β€” memex-cloud 2026-07-30). The self-updater itself only patches the image; the compiling happens on the NEW pod at startup via -`PreWarm__DynamicTypes` (see `DEPLOY-RUNBOOK.md` Β§7): every new pod sweeps and compiles all dynamic +`PreWarm__DynamicTypes` (see `README.md` β†’ "Operating it β€” rolling a new image, and the traps"): every new pod sweeps and compiles all dynamic NodeTypes at start, so an un-visited type can no longer sit "no definition" until its pages hang. Managed envs run the sweep ON β€” an env with it off has no deploy-time compile at all on this channel. @@ -66,7 +66,7 @@ timeout still surfaces instead of hiding a genuine regression. ⚠️ The gate is only as real as the namespace it runs in: it needs a `startupProbe` **on `/health`** (nothing else reads it) and `strategy.maxUnavailable: 0` (otherwise the serving pod can be deleted -before the replacement is ready). Verify both before trusting it β€” see DEPLOY-RUNBOOK.md Β§7. And if +before the replacement is ready). Verify both before trusting it β€” see README.md, "Operating it". And if a roll DOES stall, investigate it: the old image keeps serving, so there is no outage, but self-update stops advancing until someone looks. @@ -88,7 +88,7 @@ annotation/label/env. The gaps are operational: 1. **Azure (once):** ensure the portal UAMI + per-namespace federated credentials exist (`deploy/aks/infra/modules/portal-identity.bicep`, default-on via `main.bicep`), and grant it - **AcrPull** on the registry (cross-RG, out-of-band β€” see `DEPLOY-RUNBOOK.md`): + **AcrPull** on the registry (cross-RG, out-of-band β€” see `README.md` β†’ "Grant AcrPull on a SHARED registry"): ```bash PORTAL_MI=$(az identity show -g -n --query principalId -o tsv) az role assignment create --assignee-object-id "$PORTAL_MI" --assignee-principal-type ServicePrincipal \ @@ -107,9 +107,20 @@ annotation/label/env. The gaps are operational: `[SelfUpdate] applying update `. A `403` on PATCH = missing RBAC (step 2); a token/ACR error = missing workload identity or AcrPull (steps 1–2). -> ⚠️ The chart's migration is a **Job**, but the updater/RBAC target `memex-migration-deployment`. Live -> AKS clusters still have that Deployment (scaled to 0), so the migration PATCH is a harmless no-op -> there; a chart-only env logs a 404 and skips it. +> ⚠️ The chart's migration is a **Job**, but the updater/RBAC still target `memex-migration-deployment` +> β€” a workload the chart does not render (verified: `helm template deploy/helm` emits `kind: Job` named +> `memex-migration-`, and the ONLY `memex-migration-deployment` in the output is the +> `resourceNames` grant in `templates/memex-portal/rbac.yaml`). Where a namespace still carries the legacy +> Deployment the PATCH lands on a pod spec that may or may not be scaled to 0; on a chart-only env it +> **404s and `PatchDeploymentImageAsync` THROWS** β€” it does not log-and-skip. The portal is patched first, +> so the roll happens and only the migration leg faults. +> +> 🚨 **Do not "clean this up" by dropping the RBAC entry first.** Removing the grant turns the 404 into a +> 403 β€” still a throw β€” and on a namespace where the legacy Deployment is what actually applies migrations +> it would stop the schema advancing while the portal rolls ahead of it, which `DbVersionGate` answers by +> refusing to start the portal at all. The order is: teach the updater to run the migration **Job** +> (`SelfUpdateOptions.MigrationDeployment` + `KubernetesDeploymentUpdater` in `MeshWeaver.Plugins`), then +> remove the orphan Deployments and this grant. Tracked in #1788. ## Follow-up: the "edge" (any / unverified) channel diff --git a/deploy/aks/envs/example/deploy.sh b/deploy/aks/envs/example/deploy.sh index eee87c8074..c51e48fe1e 100644 --- a/deploy/aks/envs/example/deploy.sh +++ b/deploy/aks/envs/example/deploy.sh @@ -3,12 +3,13 @@ # namespace `example`, on the D16s_v5 `silos` pool, against the `exampledb` # database (already loaded with the prod data β€” see migrate-db.sh). # -# The db-migration is NOT run here: the data was restored from prod at its -# current schema version, so we deploy against it as-is (run the migration -# deliberately only if a schema delta is needed). Stage like the prod env: +# The db-migration DOES run here, as the chart's run-once Job, and that is correct: the data was +# restored from prod at its current schema version, so phase 1 (idempotent schema init) re-applies +# cleanly and phase 2 (versioned repairs, gated on admin.mesh_nodes.db_version) fast-forwards +# without repeating anything. Stage like the prod env: # STAGE with this script + values.exampledb.yaml + portal-pvcs.yaml + # portal-ingress.yaml + secretproviderclass.yaml + portal-patch.json + ./helm -# export MEMEX_PG_CONN='Host=;...;Database=exampledb;SslMode=Require;Trust Server Certificate=true' +# export MEMEX_PG_CONN='Host=.postgres.database.azure.com;...;Database=exampledb;SslMode=Require;Trust Server Certificate=true' # export IMAGE_TAG= # az aks command invoke -g -n \ # --command "MEMEX_PG_CONN='$MEMEX_PG_CONN' IMAGE_TAG='$IMAGE_TAG' bash deploy.sh" --file . @@ -25,10 +26,15 @@ kubectl apply -f ./portal-pvcs.yaml helm upgrade --install "$RELEASE" ./helm \ -f ./helm/values.yaml -f ./values.exampledb.yaml -n "$NS" -# Use the shared Flexible Server (`exampledb`) β€” don't run the chart's in-cluster -# pg, and DON'T run the migration (data already restored from prod). +# Use the shared Flexible Server (`exampledb`) β€” don't run the chart's in-cluster pg. kubectl -n "$NS" scale statefulset memex-postgres-statefulset --replicas=0 || true -kubectl -n "$NS" scale deployment memex-migration-deployment --replicas=0 || true +# 🚨 The migration is NOT scaled away here any more, and it does not need to be. It is a run-once +# Job (deploy/helm/templates/memex-migration/job.yaml), so there is no Deployment to scale to zero β€” +# the line that tried named an object the chart does not define (#1788). Running it against a +# restored prod database is safe AND is what you want: phase 1 (schema init) is idempotent and +# always runs, and phase 2 (versioned repairs) is gated on admin.mesh_nodes.db_version, so a DB +# restored at the current version fast-forwards without repeating anything. See +# memex/aspire/Memex.Database.Migration/Program.cs. kubectl -n "$NS" set image deployment/memex-portal-deployment memex-portal="$ACR/memex-portal-ai:$IMAGE_TAG" diff --git a/deploy/aks/infra/modules/acr-role-assignment.bicep b/deploy/aks/infra/modules/acr-role-assignment.bicep index ed07b1a86b..88c77f6e93 100644 --- a/deploy/aks/infra/modules/acr-role-assignment.bicep +++ b/deploy/aks/infra/modules/acr-role-assignment.bicep @@ -11,7 +11,7 @@ // principal must have User Access Administrator (or Owner) on the registry's // resource group for this to succeed β€” that is why it is opt-in (default off): // where the deployer lacks that role, grant AcrPull out-of-band instead, exactly -// as the cluster kubelet's AcrPull is granted (DEPLOY-RUNBOOK.md step 2). +// as the cluster kubelet's AcrPull is granted (README.md, "Grant AcrPull on a SHARED registry"). // --------------------------------------------------------------------------- @description('Name of the EXISTING container registry in this resource group (e.g. "meshweaver").') diff --git a/deploy/aks/infra/modules/portal-identity.bicep b/deploy/aks/infra/modules/portal-identity.bicep index a354d05a25..b909cc9e20 100644 --- a/deploy/aks/infra/modules/portal-identity.bicep +++ b/deploy/aks/infra/modules/portal-identity.bicep @@ -30,7 +30,7 @@ // another RG, so acrId is left empty here and the grant is done either by the // subscription-scoped acr-role-assignment.bicep module (opt-in IaC) or // OUT-OF-BAND via `az role assignment create` β€” exactly as the cluster -// kubelet's AcrPull is granted (see DEPLOY-RUNBOOK.md / README.md). +// kubelet's AcrPull is granted (see README.md, "Grant AcrPull on a SHARED registry"). // --------------------------------------------------------------------------- @description('Azure region for the managed identity.') diff --git a/deploy/aks/manifests/portal-ha-patch.yaml b/deploy/aks/manifests/portal-ha-patch.yaml index 0a438263f6..4c7feec3db 100644 --- a/deploy/aks/manifests/portal-ha-patch.yaml +++ b/deploy/aks/manifests/portal-ha-patch.yaml @@ -3,13 +3,36 @@ # chart Deployment/StatefulSet into an HA, persistent # AKS workload. # -# The ../helm chart ships replicas: 1 with emptyDir volumes. Rather than fork the -# chart, apply these patches AFTER `helm install`: +# 🚨 LEGACY - DO NOT APPLY. Kept for the one thing the chart still does not cover +# (the zone-spread topologySpreadConstraints, below); everything else here is now +# rendered by the chart, and re-applying this patch RE-CREATES A PRODUCTION OUTAGE. +# +# What changed. The header below was written when ../helm shipped `replicas: 1` with +# emptyDir volumes, so the documented procedure was to `kubectl patch` afterwards: # # kubectl patch deployment memex-portal-deployment -n memex \ -# --type strategic --patch-file portal-ha-patch.yaml +# --type strategic --patch-file portal-ha-patch.yaml # <- DO NOT # kubectl patch statefulset memex-postgres-statefulset -n memex \ -# --type strategic --patch-file postgres-pvc-patch.yaml +# --type strategic --patch-file postgres-pvc-patch.yaml # <- DO NOT +# +# Neither is true any more, and both are now harmful: +# +# * VOLUMES: `persistence:` in deploy/aks/values.aks.yaml supplies a claimName per +# drive and the chart mounts each directly. The bolt-on patch was replaced +# precisely because any UNPATCHED apply silently reverted /data to emptyDir, +# wiping the assembly / NuGet / DataProtection caches on every restart in the gap. +# +# * REPLICAS: KEDA owns the count (keda.enabled: true, minReplicas: 2), and the +# chart therefore OMITS spec.replicas entirely so the HPA is the single writer. +# Writing `replicas: 2` back in makes helm and the HPA fight over one field - +# invariant #1 of deploy/aks/scripts/check-chart-invariants.sh ("spec.replicas is +# ABSENT when a ScaledObject exists"), a gate that exists because the chart +# described exactly this contradiction in git for a month and produced a +# PRODUCTION 503 on 2026-08-14. +# +# Want the zone spread? Put topologySpreadConstraints in your values overlay or make +# it a chart change. Never hand-patch the Deployment. See deploy/aks/README.md -> +# "Do NOT apply portal-ha-patch.yaml". # # This document is the PORTAL patch. (Postgres patch is the next file.) # diff --git a/deploy/aks/manifests/postgres-pvc-patch.yaml b/deploy/aks/manifests/postgres-pvc-patch.yaml index 1ef413d856..4896249187 100644 --- a/deploy/aks/manifests/postgres-pvc-patch.yaml +++ b/deploy/aks/manifests/postgres-pvc-patch.yaml @@ -2,8 +2,16 @@ # postgres-pvc-patch.yaml β€” Bind the Postgres StatefulSet to a real PVC and add # the pgBackRest WAL-archive volume + sidecar. # -# The chart's StatefulSet uses emptyDir for /var/lib/postgresql/data (DATA LOSS -# on restart). This patch: +# 🚨 LEGACY for the PVC half - DO NOT APPLY it for that reason. When this was +# written the chart's StatefulSet used emptyDir for /var/lib/postgresql/data (DATA +# LOSS on restart); it now declares `volumeClaimTemplates: memex-pgdata` (RWO, 10Gi) +# itself, so the binding below is redundant. And on the default AKS path the +# in-cluster Postgres is not used at all - every namespace runs against the Azure +# Postgres Flexible Server and the StatefulSet is scaled to 0. The pgBackRest +# sidecar half is still real, and is wired by deploy/aks/README.md -> "PITR backups +# with pgBackRest". See also "Do NOT apply portal-ha-patch.yaml" there. +# +# This patch: # - replaces the data emptyDir with the memex-pgdata PVC (managed-csi, RWO) # - mounts a shared "pgbackrest" emptyDir for the spool/socket so the sidecar # and the postgres container share pgBackRest's stanza working dir diff --git a/deploy/aks/scripts/deploy.sh b/deploy/aks/scripts/deploy.sh index 07a869be2c..137051cd7a 100644 --- a/deploy/aks/scripts/deploy.sh +++ b/deploy/aks/scripts/deploy.sh @@ -8,7 +8,7 @@ # cd # cp -r /deploy/helm ./helm # cp /deploy/aks/values.aks.yaml . -# export MEMEX_PG_CONN='Host=;Port=5432;Username=memexadmin;Password=;Database=memex;SslMode=Require;Trust Server Certificate=true' +# export MEMEX_PG_CONN='Host=.postgres.database.azure.com;Port=5432;Username=memexadmin;Password=;Database=memex;SslMode=Require;Trust Server Certificate=true' # az aks command invoke -g -n --command "MEMEX_PG_CONN='$MEMEX_PG_CONN' bash deploy.sh" --file . set -uo pipefail NS=memex @@ -22,25 +22,37 @@ helm upgrade --install memex ./helm -f ./helm/values.yaml -f ./values.aks.yaml - # External managed Postgres -> don't run the chart's in-cluster pg. kubectl -n "$NS" scale statefulset memex-postgres-statefulset --replicas=0 || true # The chart hardcodes the ghcr image path -> repoint to the shared ACR. +# 🚨 The PORTAL only. The migration is a run-once Job (deploy/helm/templates/memex-migration/job.yaml), +# created fresh by the helm upgrade above with the image from .Values.migration.image β€” there is no +# migration Deployment to set an image on. The two lines that used to do that here named an object +# the chart does not define: one was swallowed by `|| true`, the other printed an error on every +# documented deploy (#1788). Override the Job's image through the chart instead: +# helm upgrade ... --set migration.image="$ACR/memex-migration:latest" kubectl -n "$NS" set image deployment/memex-portal-deployment memex-portal="$ACR/memex-portal-ai:latest" -kubectl -n "$NS" set image deployment/memex-migration-deployment memex-migration="$ACR/memex-migration:latest" || true -# 1-replica baseline. The PVC volumes/mounts (data, users, content) render from the CHART itself -# (persistence: in values.aks.yaml) β€” no volume patching here any more: the old bolt-on patch meant -# any plain `helm upgrade` reverted /data to emptyDir until the patch re-ran, wiping the +# 🚨 NO replica patch here. The chart OMITS spec.replicas whenever a ScaledObject exists +# (deploy/helm/templates/memex-portal/deployment.yaml), and values.aks.yaml sets keda.enabled: true +# with minReplicas: 2 - so KEDA owns the count. This script used to +# `kubectl patch ... /spec/replicas 1` right afterwards, which re-created BY HAND exactly the +# helm-vs-HPA contradiction that check-chart-invariants.sh invariant #1 exists to catch, and which +# produced the 2026-08-14 production 503 (three files asking for HA, one line vetoing it). Change +# the floor in values.aks.yaml (`keda.minReplicas`) instead - never with a post-helm patch. +# +# The PVC volumes/mounts (data, users, content) render from the CHART too (persistence: in +# values.aks.yaml) - no volume patching here either: the old bolt-on patch meant any plain +# `helm upgrade` reverted /data to emptyDir until the patch re-ran, wiping the # assembly/nuget/DataProtection caches on every restart in the gap. -kubectl -n "$NS" patch deployment memex-portal-deployment --type=json -p '[{"op":"replace","path":"/spec/replicas","value":1}]' # Chart-gen gap: the secret template hardcodes the in-cluster pg connection string -> repoint -# both portal + migration at the external Flexible Server (private IP + password + SSL). +# both portal + migration at the external Flexible Server (server FQDN + password + SSL). for s in memex-portal-secrets memex-migration-secrets; do kubectl -n "$NS" patch secret "$s" --type merge -p "{\"stringData\":{\"ConnectionStrings__memex\":\"${MEMEX_PG_CONN}\"}}" done -kubectl -n "$NS" rollout restart deployment/memex-portal-deployment deployment/memex-migration-deployment +kubectl -n "$NS" rollout restart deployment/memex-portal-deployment echo "=== deployed ==="; kubectl -n "$NS" get deploy,pvc,svc -o wide # Observability (opt-in, folded into the standard deploy): set GRAFANA_PW to also bring up # Grafana + Loki + Promtail + Prometheus (Promtail scrapes every pod's stdout into Loki, so the # portal logs flow with no portal-side config). Stays private β€” reach it via the P2S VPN + -# `kubectl port-forward` (see DEPLOY-RUNBOOK.md Β§6). Non-fatal: a monitoring failure must not +# `kubectl port-forward` (see README.md, "Grafana + Loki + Prometheus"). Non-fatal: a monitoring failure must not # fail the app deploy. if [ -n "${GRAFANA_PW:-}" ] && [ -f ./install-observability.sh ]; then echo "=== observability (GRAFANA_PW set) ===" diff --git a/deploy/aks/values.aks.yaml b/deploy/aks/values.aks.yaml index b3eabf5a7a..3fba3c14f6 100644 --- a/deploy/aks/values.aks.yaml +++ b/deploy/aks/values.aks.yaml @@ -185,7 +185,7 @@ config: # the serving pod can be deleted first, so the gate reports and the outage # happens anyway β€” false confidence. # The chart renders 1 and 3 correctly; a per-env patch or a hand `kubectl` can - # still override them, so CHECK before trusting the gate (DEPLOY-RUNBOOK Β§7). + # still override them, so CHECK before trusting the gate (README.md, "Operating it"). # # Failure mode if the gate DOES fire on a real regression: a stalled rollout # with the old image still serving β€” no outage, but self-update stops diff --git a/deploy/helm/templates/memex-portal/secrets.yaml b/deploy/helm/templates/memex-portal/secrets.yaml index 8658847500..1ab0b6e271 100644 --- a/deploy/helm/templates/memex-portal/secrets.yaml +++ b/deploy/helm/templates/memex-portal/secrets.yaml @@ -13,7 +13,7 @@ stringData: # This line used to hard-code the in-cluster StatefulSet unconditionally. Every AKS namespace # runs against the Azure Postgres FLEXIBLE SERVER instead (`memex-postgres-statefulset` is scaled # to 0/0 there), so each env's deploy.sh had to `kubectl patch` the rendered value back out again - # afterwards β€” the "chart connection string" entry under DEPLOY-RUNBOOK's known gaps. + # afterwards β€” the "chart connection string" entry under deploy/aks/README.md's "Known gaps / follow-ups". # # The patch-afterwards dance is not merely untidy, it is a live outage waiting on the next # `helm upgrade`: helm REPLACES a Secret wholesale, so the render overwrites the working diff --git a/memex/Memex.Portal.Shared/Email/EmailDeliveryGuard.cs b/memex/Memex.Portal.Shared/Email/EmailDeliveryGuard.cs new file mode 100644 index 0000000000..31694b4e21 --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/EmailDeliveryGuard.cs @@ -0,0 +1,78 @@ +using MeshWeaver.Mesh; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Email; + +/// +/// 🚨 Email:Enabled=true with a sender that does not deliver is a REFUSED +/// configuration, not a working one (#2023). +/// +/// The defect this closes. While the Graph sender was compiled into the portal, +/// "a sender is registered" and "mail can be delivered" were the same question, and +/// 's report-success-without-sending contract was safe: it was only +/// ever reached with Email:Enabled=false, where no watcher calls it. The move into the +/// MeshWeaver.Mail.MicrosoftGraph module (#1737) added a SECOND path to the no-op β€” the +/// TryAddSingleton fallback when the module is not on the install β€” and on that path +/// Enabled IS true and the watchers DO run. then flipped +/// every queued mail New β†’ Sending β†’ Sent while nothing left the process: node data saying +/// delivered, no error, no log (the send lines are Information under a Warning default). Hours +/// were spent hunting inboxes, junk folders and message traces because Sent was trusted. +/// +/// Why refusing beats succeeding quietly. Mail that stays New is recoverable: +/// land the module and it goes out, with no data repair. Mail stamped Sent that was never +/// sent is unrecoverable, because nothing distinguishes it from mail that WAS sent. So the +/// watchers do not start at all on this configuration, and the no-op sender fails loudly for +/// every other caller instead of returning true. +/// +/// The refusal is logged at from StartAsync β€” inside the +/// startup window β€” so StartupErrorNotifier folds it into the Admin bell notification that +/// reports a degraded boot, rather than it sitting at Information under a Warning default where +/// nobody would ever see it. +/// +internal static class EmailDeliveryGuard +{ + /// The module that supplies a real sender. Named in every refusal so the operator is + /// told what to provision rather than that "email is broken". + internal const string SenderModule = "MeshWeaver.Mail.MicrosoftGraph"; + + /// + /// True when this install is configured to send mail but the resolved sender cannot deliver + /// it. A null sender counts: no sender at all certainly does not deliver, and treating + /// "could not resolve" as "probably fine" would be the skip-trapdoor this guard exists to + /// remove. + /// + internal static bool RefusesDelivery(EmailOptions options, IEmailSender? sender) + => options.Enabled && sender?.DeliversMail is not true; + + /// The one wording, so the sender, the watchers and the tests all say the same thing. + /// What is being refused (a component that will not start, or the send itself). + internal static string Explain(string what) + => $"Email:Enabled=true but the resolved {nameof(IEmailSender)} does NOT deliver mail β€” this " + + $"install has no mail sender. {what} Land the {SenderModule} module on this deployment " + + "(Modules:Assemblies / the plugin bundle) to restore delivery. Refusing rather than " + + "reporting success: mail left queued is recoverable once the module lands, mail stamped " + + "Sent that was never sent is not."; + + /// + /// The watcher-side gate: resolves the sender this install would actually use and, when it + /// cannot deliver, logs the refusal at Error and answers true so the caller returns + /// WITHOUT starting its watch. Queued mail then visibly stays New. + /// + /// The provider the host's sender singleton lives in. + /// The bound Email section. + /// Where the refusal is reported. + /// The watcher refusing to start, named in the log line. + internal static bool RefuseToStart( + IServiceProvider services, EmailOptions options, ILogger? logger, string component) + { + if (!RefusesDelivery(options, services.GetService())) + return false; + + logger?.LogError( + "{Component} is NOT starting. {Explanation}", + component, + Explain($"{component} will not claim or send anything, so outbound mail stays visibly queued.")); + return true; + } +} diff --git a/memex/Memex.Portal.Shared/Email/InvitationEmailSender.cs b/memex/Memex.Portal.Shared/Email/InvitationEmailSender.cs index 271ab9fd0f..d368ae32f3 100644 --- a/memex/Memex.Portal.Shared/Email/InvitationEmailSender.cs +++ b/memex/Memex.Portal.Shared/Email/InvitationEmailSender.cs @@ -61,6 +61,13 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } + // 🚨 Same refusal as OutboundEmailSender, and it matters more here: this watcher's claim is + // Invitation.EmailSentAt, which is PERMANENT β€” an invitation stamped "emailed" by a sender + // that delivered nothing can never be re-sent by any code path. Not starting leaves it + // un-stamped, so delivery resumes by itself once the module lands (#2023). + if (EmailDeliveryGuard.RefuseToStart(rootServices, options, logger, nameof(InvitationEmailSender))) + return Task.CompletedTask; + // Defer mesh access until the host is fully started (Orleans + mesh hub come up as // hosted services too) β€” same rationale as OutboundEmailSender. lifetime.ApplicationStarted.Register(BeginWatching); diff --git a/memex/Memex.Portal.Shared/Email/NoOpEmailSender.cs b/memex/Memex.Portal.Shared/Email/NoOpEmailSender.cs index f3d05bafe6..0271385972 100644 --- a/memex/Memex.Portal.Shared/Email/NoOpEmailSender.cs +++ b/memex/Memex.Portal.Shared/Email/NoOpEmailSender.cs @@ -5,18 +5,53 @@ namespace Memex.Portal.Shared.Email; /// -/// No-op registered when Email:Enabled is false (local dev, -/// tests, deployments without M365 creds). Logs the would-be send and reports success so -/// callers' reactive chains complete normally without any mail leaving the process. +/// The an install resolves when no real sender is registered β€” the +/// TryAddSingleton fallback in MemexConfiguration, taken whenever the +/// MeshWeaver.Mail.MicrosoftGraph module is not on this deployment. +/// +/// It reports false, and what it does on a send depends +/// entirely on which of TWO configurations reached it: +/// +/// +/// Email:Enabled=false β€” the intended case (local dev, tests, a +/// deployment with no mailbox). Nothing queues mail here, no watcher runs, and callers' reactive +/// chains complete normally: logs the would-be send and emits true. +/// 🚨 Email:Enabled=true β€” a MISCONFIGURED install: it is set up to send +/// mail and has nothing to send it with. Emitting true here is a lie that +/// converts into a durable one, stamping every queued mail +/// Sent while nothing leaves the process (#2023). So this path REFUSES: it logs the cause +/// at Error and surfaces the failure through OnError, which marks the mail +/// Failed β€” visible and re-queueable β€” instead of falsely +/// Sent. +/// +/// +/// The two are told apart by , which is why this type takes +/// the options rather than defaulting them: a no-op that cannot tell which case it is in cannot +/// report the right thing, and its old log line ("Email disabled (Email:Enabled=false)") stated a +/// falsehood on the second path. /// -public sealed class NoOpEmailSender(ILogger? logger = null) : IEmailSender +public sealed class NoOpEmailSender(EmailOptions options, ILogger? logger = null) : IEmailSender { + /// Always false β€” this sender's entire contract is to not deliver. See + /// for why the flag exists at all. + public bool DeliversMail => false; + public IObservable SendEmail(string toAddress, string subject, string htmlBody) => SendEmail(toAddress, subject, htmlBody, []); public IObservable SendEmail( string toAddress, string subject, string htmlBody, IReadOnlyCollection attachments) { + if (options.Enabled) + { + var explanation = EmailDeliveryGuard.Explain( + "This send is REFUSED rather than reported as delivered."); + logger?.LogError( + "Refusing to send to {To} (subject: {Subject}, attachments: {Attachments}). {Explanation}", + toAddress, subject, attachments.Count, explanation); + return Observable.Throw(new InvalidOperationException(explanation)); + } + logger?.LogInformation( "Email disabled (Email:Enabled=false) β€” skipping send to {To} (subject: {Subject}, attachments: {Attachments})", toAddress, subject, attachments.Count); diff --git a/memex/Memex.Portal.Shared/Email/OutboundEmailSender.cs b/memex/Memex.Portal.Shared/Email/OutboundEmailSender.cs index 1deb84a7a5..3aaac3d905 100644 --- a/memex/Memex.Portal.Shared/Email/OutboundEmailSender.cs +++ b/memex/Memex.Portal.Shared/Email/OutboundEmailSender.cs @@ -69,6 +69,15 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } + // 🚨 Enabled, but is there anything that can actually DELIVER? Asked HERE, in StartAsync, + // for two reasons. It is inside the startup window, so StartupErrorNotifier folds the + // refusal into the Admin bell that reports a degraded boot β€” whereas ApplicationStarted + // callbacks run after that buffer has already drained. And refusing before the watch is + // even armed is what keeps queued mail at New: a watcher that starts and then declines + // per-item has already claimed (New β†’ Sending) the first one. See EmailDeliveryGuard. + if (EmailDeliveryGuard.RefuseToStart(rootServices, options, logger, nameof(OutboundEmailSender))) + return Task.CompletedTask; + // Defer ALL mesh access until the host is fully started. The Orleans client and the mesh // hub come up as hosted services too; touching the hub here (or constructing // PortalApplication, whose ctor registers an Orleans stream) races that startup and NREs diff --git a/memex/Memex.Portal.Shared/MemexConfiguration.cs b/memex/Memex.Portal.Shared/MemexConfiguration.cs index acb603ebba..cac24c456c 100644 --- a/memex/Memex.Portal.Shared/MemexConfiguration.cs +++ b/memex/Memex.Portal.Shared/MemexConfiguration.cs @@ -167,6 +167,13 @@ public static void ConfigureMemexServices(this WebApplicationBuilder builder) // Graph sender wins; module absent β‡’ this no-op keeps OutboundEmailSender and // InvitationEmailSender β€” both of which GetRequiredService β€” resolvable // instead of throwing at startup. + // + // 🚨 Resolvable is NOT the same as able to send, and conflating the two was #2023: with + // Email:Enabled=true and the module absent, this fallback used to report success for every + // send, so OutboundEmailSender stamped queued mail New β†’ Sending β†’ Sent while nothing left + // the process. Both halves of that are now refused β€” the no-op fails loudly on this + // configuration (NoOpEmailSender), and the two watchers decline to start at all + // (EmailDeliveryGuard), leaving mail visibly queued instead of falsely delivered. services.TryAddSingleton(); // Inbound emailβ†’agent channel (intake). Mail is treated as a chat device: each inbound email diff --git a/src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md b/src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md index d87afe7fba..82e3988b61 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md @@ -121,7 +121,7 @@ The portal container is `memex-portal` in Deployment `memex-portal-deployment`. - **Migration ran:** find the Job first β€” `kubectl -n get jobs -l app.kubernetes.io/component=memex-migration` β€” then `kubectl -n logs job/memex-migration- --tail=40` β†’ expect `Database migration completed. Version: N`. A Job that reports `Complete` with that line is the success signal. - **A `CrashLoopBackOff` on a migration *Deployment* is NOT benign.** That is the legacy shape, and it is exactly the failure the Job replaced: the process exits 0, the Deployment restarts it, and every run rebuilds `public.top_level_index` across every partition schema. The chart records 310 restarts in a day pegging a full core. If you see it, the namespace is still on the legacy Deployment β€” do not wave it through. -- **Portal serves:** `curl -sS -o /dev/null -w '%{http_code}' https:///` β†’ `200`. **The host is not derivable from the namespace** β€” namespace `memex` serves `memex.systemorph.com` (the DNS record `deploy/aks/DEPLOY-RUNBOOK.md` creates), while `memex.meshweaver.cloud` is the *`memex-cloud`* namespace. Read the host off the namespace's own Ingress (`kubectl -n get ingress -o wide`) rather than templating it, or you will happily verify a portal you did not deploy to. +- **Portal serves:** `curl -sS -o /dev/null -w '%{http_code}' https:///` β†’ `200`. **The host is not derivable from the namespace** β€” namespace `memex` serves `memex.systemorph.com` (the DNS record `deploy/aks/README.md` β†’ "Public ingress + TLS + DNS" creates), while `memex.meshweaver.cloud` is the *`memex-cloud`* namespace. Read the host off the namespace's own Ingress (`kubectl -n get ingress -o wide`) rather than templating it, or you will happily verify a portal you did not deploy to. - **Schema/index applied** (when the change was a migration): spot-check via `az aks command invoke … "kubectl -n exec deployment/memex-portal-deployment -- …"` or an MCP query. ### The cluster runs what the chart describes β€” check it, don't assume it @@ -261,7 +261,7 @@ Steady state is **self-update** (see [ReleaseStrategy.md](/Doc/Architecture/Rele 1. **Provision the UAMI + federated credentials** β€” included in the infra deploy (`deployPortalIdentity` defaults `true`). Read the client id back: ```bash - # --name is whatever the infra deploy was created with; deploy/aks/DEPLOY-RUNBOOK.md uses memex-aks-infra-sc. + # --name is whatever the infra deploy was created with; deploy/aks/README.md uses memex-aks-infra. az deployment sub show --name memex-aks-infra-sc \ --query "properties.outputs.{clientId:portalIdentityClientId.value, principalId:portalIdentityPrincipalId.value}" -o jsonc ``` @@ -285,10 +285,15 @@ Deployments** in one pass β€” `memex-portal-deployment` (container `memex-portal `memex-migration-deployment` (container `memex-migration`), the names in `SelfUpdateOptions`. **🚨 That second patch does not currently run a migration, and this is a known gap, not a working -mechanism.** `deploy/aks/SELF-UPDATE.md` states it outright: the chart's migration is a Job, but the updater -and its RBAC target `memex-migration-deployment` β€” live AKS clusters have that Deployment scaled to 0, so -the PATCH is a no-op there, and a chart-only environment logs a 404 and skips it. **So a self-update rolls -the portal image while the schema stays where it was.** A release that carries a migration needs the Job to +mechanism.** The chart's migration is a **Job**; the updater and its RBAC target +`memex-migration-deployment`, which the chart does not render at all (`helm template deploy/helm` produces +`kind: Job` named `memex-migration-`, and the only occurrence of `memex-migration-deployment` in +the whole rendered output is the RBAC `resourceNames` grant). On a namespace that still carries the legacy +Deployment the PATCH lands on a pod spec; on a chart-only environment it **404s, and +`KubernetesDeploymentUpdater.PatchDeploymentImageAsync` THROWS on any non-success status** β€” it does not +"log and skip". The portal is patched FIRST, so the roll itself still happens and the fault surfaces +afterwards, on the migration leg only. **Either way a self-update rolls the portal image while the schema +stays where it was.** A release that carries a migration needs the Job to run (`helm upgrade`) β€” do not assume self-update covers it. Verify `admin.mesh_nodes.db_version` after any roll that included a schema change. diff --git a/src/MeshWeaver.Documentation/Data/Architecture/Instances.md b/src/MeshWeaver.Documentation/Data/Architecture/Instances.md index 4743a3d7a2..e619b03bb8 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/Instances.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/Instances.md @@ -72,9 +72,14 @@ provisioned with the deploy tooling, not from the running app β€” the company in 1. Add the namespace to `portalNamespaces` in `deploy/aks/infra/main.bicep` (creates its federated credential + AcrPull) and redeploy the identity module. 2. Create the instance's **database** on the shared `` server. -3. Author `deploy/aks/envs//values..yaml` (git-ignored: host, `MEMEX_DATABASENAME`, TLS - secret, AI + auth config, `selfUpdate.azureClientId`). -4. `deploy/aks/envs//deploy.sh` β€” helm install + PVCs + KV `SecretProviderClass` + ingress + TLS. +3. Author the env's `values..yaml` (host, `MEMEX_DATABASENAME`, TLS secret, AI + auth config, + `selfUpdate.azureClientId`). +4. Run the env's `deploy.sh` β€” helm install + PVCs + KV `SecretProviderClass` + ingress + TLS. + +> 🚨 **Env folders live in the PRIVATE `Systemorph/Memex` repository**, not under +> `deploy/aks/envs//` β€” they moved out on 2026-08-08/09 because their directory names are +> tenant identities. The maintained, ordered procedure is `docs/new-deployment.md` there; +> `deploy/aks/envs/example/` in this repository is the reference template only. 5. Wire sign-in redirect URIs + invitation/email config for the new domain. **Delete** an instance: diff --git a/src/MeshWeaver.Documentation/Data/Architecture/MemexCloudDeployment.md b/src/MeshWeaver.Documentation/Data/Architecture/MemexCloudDeployment.md index 5db7148aef..400d95e580 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/MemexCloudDeployment.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/MemexCloudDeployment.md @@ -11,7 +11,7 @@ This guide explains how to stand up the Memex portal on a **private** Azure Kube > **Conventions.** Examples use placeholder names β€” domain `portal.example.com`, registry `meshweaver`, resource group ``. Substitute your own values. **Sensitive values** (IP addresses, tenant/app GUIDs, passwords, client secrets) appear as ``; never commit real ones β€” keep them in Key Vault. > -> The exact, ordered command sequence lives in `deploy/aks/DEPLOY-RUNBOOK.md` (in the repository). This document is the architecture and operations layer around it. +> The exact, ordered command sequence lives in `deploy/aks/README.md` (in the repository) β€” the public, self-contained AKS deployment sample, which `DEPLOY-RUNBOOK.md` was folded into. This document is the architecture and operations layer around it. > **Deployment model.** One Aspire AppHost (`deploy/aspire/Memex.Deploy.AppHost`) describes the workload from published images. The Aspire **Kubernetes publisher** generates the Helm chart (`deploy/helm`). The AKS *platform* β€” cluster, Postgres, VPN, TLS β€” is Bicep plus a thin overlay. @@ -139,7 +139,9 @@ Parameters live in `deploy/aks/infra/main.parameters.json`: - `postgresHighAvailability` β€” enable for production. - `gatewaySku` β€” use an AZ SKU such as `VpnGw1AZ`. -The Postgres connection uses the **private IP + password + SSL** (`SslMode=Require;Trust Server Certificate=true`). Do not use the public FQDN form β€” a `database.azure.com` hostname routes the portal into its managed-identity-token branch, which does not match a password server. +The Postgres connection uses the **server FQDN + password + SSL** (`Host=.postgres.database.azure.com;…;SslMode=Require;Trust Server Certificate=true`). The servers moved to the FQDN on 2026-08-24. + +> 🚨 This page previously said the opposite β€” *"do not use the public FQDN form, a `database.azure.com` hostname routes the portal into its managed-identity-token branch"* β€” and that would now talk an operator out of the correct configuration. It stopped being true with `9e468aa70`: `AzurePostgres.UsesManagedIdentityAuth` (`src/MeshWeaver.Hosting.PostgreSql/AzurePostgres.cs`) takes the Entra-token path only when the host carries the Azure suffix **AND no password is present**, so FQDN + password deliberately takes plain Npgsql. `AzurePostgresAuthSelectionTests` pins exactly that case. --- @@ -150,13 +152,13 @@ The Postgres connection uses the **private IP + password + SSL** (`SslMode=Requi 1. Creates the namespace and RWX PVCs. 2. Runs `helm upgrade --install` with `deploy/helm` + `values.aks.yaml` + `values.deploy.yaml`. 3. Scales the chart's in-cluster Postgres to 0 (you use the Flexible Server). -4. Runs `kubectl set image` to the shared ACR. -5. Patches the portal to 1 replica with the Azure Files mounts. +4. Runs `kubectl set image` on the **portal** Deployment to the shared ACR. +5. Patches the portal to 1 replica (the Azure Files mounts render from the chart itself β€” `persistence:` in `values.aks.yaml` β€” so there is no volume patch any more). 6. **Patches the connection-string secret** to the external Postgres. > **Known chart-generation gaps** (fix at the AddMemex generator): > - The chart's `secrets.yaml` hardcodes the in-cluster Postgres connection string β†’ `deploy.sh` patches it post-install. -> - Step 4's `kubectl set image deployment/memex-migration-deployment` is a **legacy leftover** and is why `deploy.sh` guards it with `|| true`. The chart renders the migration as a run-once **Job** (`memex-migration-`), so that line targets a Deployment the chart no longer creates. The migration actually runs from the `helm upgrade` in step 2. +> - `deploy.sh` used to carry TWO commands against a `memex-migration-deployment` β€” a `set image` and a `rollout restart` β€” and only the first was guarded with `|| true`; the second printed an error on every documented deploy. Both are gone (#1788): the chart renders the migration as a run-once **Job** (`memex-migration-`), so there is no such Deployment. The migration runs from the `helm upgrade` in step 2; override its image through the chart (`--set migration.image=…`), never with `kubectl set image`. > 🚨 **A migration `CrashLoopBackOff` is NOT harmless.** Earlier revisions of this chart rendered the > migration as a Deployment, which restarted the process after each clean `exit 0`. Every run rebuilds diff --git a/src/MeshWeaver.Documentation/Data/Architecture/OnboardingNewEnvironment.md b/src/MeshWeaver.Documentation/Data/Architecture/OnboardingNewEnvironment.md index a73d8bb5a8..215480751c 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/OnboardingNewEnvironment.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/OnboardingNewEnvironment.md @@ -9,12 +9,20 @@ Category: Architecture A "new environment" is an additional Memex portal β€” its own domain, database, and sign-in β€” running on the **shared AKS cluster** (`` / ``, -swedencentral). Each environment lives under `deploy/aks/envs//`; the only one tracked in this -repo is the reference template `deploy/aks/envs/example/` β€” per-tenant env folders are git-ignored -because their directory names are tenant identities. -The shared platform (cluster, ingress, Postgres server, Key Vault, ACR) is brought up -once β€” see the AKS deploy runbook (`deploy/aks/DEPLOY-RUNBOOK.md` in the repository); this guide -adds an environment on top of it. +swedencentral). + +> 🚨 **This page is the MECHANISM, not the runbook.** Per-environment folders moved OUT of this +> repository on 2026-08-08/09 (commit `a69959165`) β€” their directory names are tenant identities. +> The maintained, ordered operator procedure lives in the **private `Systemorph/Memex` repository** +> (`docs/new-deployment.md`, with the inventory in `deployments/aks/envs.json`), and that copy is +> authoritative wherever this page and it disagree. What remains here is the part that is genuinely +> public: what is shared vs separate, the chart's config pass-through rule, the self-update model, +> and the gotchas. The only env folder still in this repository is the reference template +> `deploy/aks/envs/example/`, kept as a shape to copy β€” **not** as the place a real environment lives. + +The shared platform (cluster, ingress, Postgres server, Key Vault, ACR) is brought up once β€” see +the AKS deployment sample (`deploy/aks/README.md` in the repository); this guide adds an +environment on top of it. ## Shared vs. separate @@ -31,7 +39,9 @@ adds an environment on top of it. ## 1. Scaffold the env folder -Copy the template folder `deploy/aks/envs/example/` to `deploy/aks/envs//`: +Copy the reference template `deploy/aks/envs/example/` β€” the real env folder is created in the +**private ops repository**, not here (see the note above). These are the files it contains and what +each one carries: | File | What to change | |---|---| @@ -84,16 +94,23 @@ PORTAL_MI_CLIENT_ID=$(az identity show -g $RG -n --query clien ## 3. Deploy + issue TLS +> 🚨 **The connection host is the server FQDN, not a private IP.** The servers moved on 2026-08-24, +> and a stale private-IP address here would revert that migration. The FQDN is safe with a password: +> `AzurePostgres.UsesManagedIdentityAuth` (`src/MeshWeaver.Hosting.PostgreSql/AzurePostgres.cs`) +> takes the Entra-token path only when the host carries the Azure suffix **AND no password is +> present**, so FQDN + password deliberately takes plain Npgsql with `SslMode=Require` β€” pinned by +> `AzurePostgresAuthSelectionTests`. + ```bash -STAGE=$(mktemp -d); cp deploy/aks/envs//* "$STAGE"/; cp -r deploy/helm "$STAGE"/helm -export MEMEX_PG_CONN='Host=;Port=5432;Username=memexadmin;Password=;Database=;SslMode=Require;Trust Server Certificate=true' +STAGE=$(mktemp -d); cp /* "$STAGE"/; cp -r deploy/helm "$STAGE"/helm +export MEMEX_PG_CONN='Host=.postgres.database.azure.com;Port=5432;Username=memexadmin;Password=;Database=;SslMode=Require;Trust Server Certificate=true' export IMAGE_TAG= ( cd "$STAGE" && az aks command invoke -g -n \ --command "MEMEX_PG_CONN='$MEMEX_PG_CONN' IMAGE_TAG='$IMAGE_TAG' bash deploy.sh" --file . ) # Verify BEFORE DNS/TLS (host still unresolved or pointing elsewhere): curl -sS -k -o /dev/null -w "%{http_code}\n" --resolve :443:$INGRESS_IP https:/// # Then issue the cert (needs the A-record to resolve publicly): -( cd deploy/aks/envs/ && az aks command invoke -g -n --command "bash tls.sh" --file tls.sh ) +( cd && az aks command invoke -g -n --command "bash tls.sh" --file tls.sh ) ``` ## Self-update: first-install checklist @@ -307,6 +324,6 @@ for i in $(seq 1 10); do curl -s -o /dev/null -w "%{http_code} " https:/// ## Related -- AKS Deploy Runbook β€” `deploy/aks/DEPLOY-RUNBOOK.md` in the repository β€” the one-time shared-platform bring-up. +- AKS deployment sample β€” `deploy/aks/README.md` in the repository β€” the one-time shared-platform bring-up (the former `DEPLOY-RUNBOOK.md` was folded into it). - [Memex Cloud Deployment](/Doc/Architecture/MemexCloudDeployment) Β· [Deployment Options](/Doc/Architecture/DeploymentOptions) - [Invitation-Only Onboarding](/Doc/Architecture/InvitationOnlyOnboarding) Β· [Feature Flags](/Doc/Architecture/FeatureFlags) diff --git a/src/MeshWeaver.Documentation/Data/WhatsNew/2026-08-25-email-no-longer-reports-sent-without-sending.md b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-08-25-email-no-longer-reports-sent-without-sending.md new file mode 100644 index 0000000000..6bf5acd7e4 --- /dev/null +++ b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-08-25-email-no-longer-reports-sent-without-sending.md @@ -0,0 +1,21 @@ +--- +Name: Email never reports "Sent" without sending +Category: Fix +Description: A portal set up to send mail but missing its mail sender used to mark every message Sent while nothing was delivered; it now refuses loudly and leaves the mail queued. +Icon: Sparkle +Order: -20260825 +--- + +# Email never reports "Sent" without sending + +If a portal was switched on for e-mail but had no mail sender installed, outbound messages and +invitations were marked **Sent** while nothing actually left the portal. Nothing looked wrong: the +message record said delivered, and no error appeared anywhere. The only way to find out was that +the mail never arrived. + +That can no longer happen. On a portal in that state, e-mail now stops instead of pretending: the +message stays visibly **queued**, the failure is reported, and the cause is named. Once the mail +sender is installed, the queued messages go out on their own β€” nothing needs to be re-created or +re-sent by hand. + +Portals that send mail normally, and portals with e-mail switched off, are unaffected. diff --git a/src/MeshWeaver.Mesh.Contract/IEmailSender.cs b/src/MeshWeaver.Mesh.Contract/IEmailSender.cs index ad75e708dc..d368eb6ba0 100644 --- a/src/MeshWeaver.Mesh.Contract/IEmailSender.cs +++ b/src/MeshWeaver.Mesh.Contract/IEmailSender.cs @@ -16,6 +16,25 @@ namespace MeshWeaver.Mesh; /// public interface IEmailSender { + /// + /// Whether this sender ACTUALLY DELIVERS mail β€” false for a sender whose contract is to + /// report success without sending (the host's no-op). + /// + /// 🚨 This exists because "a sender is registered" and "mail can be delivered" were the + /// same question for exactly as long as the only real sender was compiled in. Once the Graph + /// sender moved into the MeshWeaver.Mail.MicrosoftGraph module, a deployment could have + /// Email:Enabled=true AND resolve the no-op β€” and every queued mail was then stamped + /// Sent while nothing left the process. Undeliverable mail marked delivered is silent + /// data loss, and the only thing that separates it from a working install is this flag + /// (#2023). + /// + /// Defaults to true: a sender says it cannot deliver only when it genuinely + /// cannot, so a third-party implementation that forgets this member is treated as a real + /// sender β€” the safe direction, because a real sender wrongly reported as a no-op would refuse + /// mail that would have gone out. + /// + bool DeliversMail => true; + /// /// Sends an HTML email. The returned observable is cold β€” the send runs on Subscribe and /// emits a single true on success, or surfaces the failure via OnError. diff --git a/test/Memex.Portal.Shared.Test/DrainEndpointTest.cs b/test/Memex.Portal.Shared.Test/DrainEndpointTest.cs new file mode 100644 index 0000000000..0836635c41 --- /dev/null +++ b/test/Memex.Portal.Shared.Test/DrainEndpointTest.cs @@ -0,0 +1,187 @@ +using System.Net; +using System.Net.Http; +using System.Text.Encodings.Web; +using System.Threading.Tasks; +using Memex.Portal.ServiceDefaults; +using MeshWeaver.Hosting; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Memex.Portal.Shared.Test; + +/// +/// 🚨 The session drain, pinned end to end (#1971 / #1342). +/// +/// What was NOT pinned. A rollout is non-disruptive only because the container's +/// preStop polls http://127.0.0.1:8080/drain until this pod's last Blazor circuit +/// closes. #2043 pinned the ARITHMETIC of that loop (DrainDeadlineGuard, a text guard over +/// the chart) and DrainProgressTest pins what each probe is worth logging β€” but nothing +/// pinned the thing in the middle: that the portal actually MAPS /drain, that it answers +/// 503-with-a-count while circuits are open and 200 when drained, and that it answers with no +/// session at all. Every one of those is a chart↔code agreement, and this chart already carries a +/// scar of exactly that class: preStop once probed with wget, which is absent from +/// the image, so the loop could never succeed and EVERY termination hung to the grace ceiling. It +/// was caught by reading, not by a test. +/// +/// Why a wrong answer here is silent. preStop probes with +/// curl -sf -m 5 -o /dev/null, which cannot tell a 404 from a 503 from a refused +/// connection β€” all three are "not drained yet, keep waiting". So an unmapped, renamed, or +/// login-gated /drain does not fail: it makes every roll sit out its whole drain window and +/// then cut every open session at the deadline, with nothing in the log that names the cause. That +/// is the same disease as the rest of this issue β€” silence indistinguishable from success β€” and +/// these cases are what make it fail loudly instead, at build time. +/// +/// Driven over a REAL HTTP pipeline through the REAL +/// composition (never a hand-mapped copy), behind +/// a deny-by-default authorization fallback so the anonymity assertion is not vacuous β€” the same +/// harness shape as beside it. +/// +public class DrainEndpointTest +{ + /// The route the chart's preStop hook polls. Changing it is a chart change too. + private const string DrainRoute = "/drain"; + + /// A route with no AllowAnonymous β€” the proof the fallback policy is real. + private const string GuardedRoute = "/guarded"; + + private const string TestScheme = "NoSession"; + + /// + /// The pipeline under test. is registered exactly as the portal + /// registers it, so the endpoint reads the same counter a live pod's circuits feed. + /// + private static WebApplication BuildApp(ActiveCircuitTracker? tracker) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Logging.ClearProviders(); + builder.AddDefaultHealthChecks(); + + if (tracker is not null) + builder.Services.AddSingleton(tracker); + + builder.Services.AddAuthentication(TestScheme) + .AddScheme(TestScheme, _ => { }); + + // Deny by default. AllowAnonymous on /drain is what must beat this β€” a preStop hook carries + // no cookie and no bearer token. + builder.Services.AddAuthorizationBuilder() + .SetFallbackPolicy(new AuthorizationPolicyBuilder() + .RequireAssertion(_ => false) + .Build()); + + var app = builder.Build(); + app.UseAuthentication(); + app.UseAuthorization(); + app.MapDefaultEndpoints(); + app.MapGet(GuardedRoute, () => "secret"); + return app; + } + + /// Authenticates nobody β€” the "no session" a kubelet preStop probe arrives with. + private sealed class NoSessionHandler( + IOptionsMonitor options, ILoggerFactory logger, UrlEncoder encoder) + : AuthenticationHandler(options, logger, encoder) + { + protected override Task HandleAuthenticateAsync() => + Task.FromResult(AuthenticateResult.NoResult()); + } + + private static async Task<(HttpResponseMessage Response, string Body)> GetAsync( + string route, ActiveCircuitTracker? tracker = null) + { + var app = BuildApp(tracker); + await using (app) + { + await app.StartAsync(TestContext.Current.CancellationToken); + using var client = app.GetTestClient(); + var response = await client.GetAsync(route, TestContext.Current.CancellationToken); + var body = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + return (response, body); + } + } + + /// + /// Non-vacuity guard: the pipeline really does refuse an unauthenticated caller. Without it the + /// anonymity assertion below proves nothing β€” every route answers when nothing is guarding. + /// + [Fact] + public async Task A_guarded_route_in_the_same_pipeline_refuses_an_anonymous_caller() + { + var (response, _) = await GetAsync(GuardedRoute); + + response.StatusCode.Should().BeOneOf(HttpStatusCode.Unauthorized, HttpStatusCode.Forbidden); + } + + /// + /// 🚨 The success signal preStop waits for. A pod with no circuits answers 200, which is the + /// ONLY thing that ends the drain loop early β€” every other outcome, including a 404 from an + /// unmapped route, reads to curl -sf as "still draining". + /// + [Fact] + public async Task WithNoCircuits_TheDrainRouteAnswers200_TheOnlyThingThatEndsThePreStopLoop() + { + var (response, body) = await GetAsync(DrainRoute, new ActiveCircuitTracker()); + + response.StatusCode.Should().Be(HttpStatusCode.OK, + "preStop polls until this route SUCCEEDS; a 404 or a 5xx is indistinguishable from " + + "'sessions are still open' and rides the whole drain window before cutting them off"); + body.Should().Be("drained"); + } + + /// + /// 🚨 The hold signal. A live circuit must produce a NON-success status, or a roll would tear + /// down a pod that is still serving someone β€” the regression #1342 fixed. + /// + [Fact] + public async Task WithALiveCircuit_TheDrainRouteAnswers503_AndNamesTheCount() + { + var tracker = new ActiveCircuitTracker(); + tracker.Opened(); + tracker.Opened(); + + var (response, body) = await GetAsync(DrainRoute, tracker); + + response.StatusCode.Should().Be(HttpStatusCode.ServiceUnavailable); + body.Should().Contain("2", + "the count is the diagnostic β€” DrainProgress reports it, and an operator reading a " + + "Terminating pod has nothing else to go on"); + } + + /// + /// 🚨 Anonymous, because the caller is a kubelet exec with no cookie and no token. A portal + /// that put this route behind its login would answer 302/401 to every probe β€” which + /// curl -sf reads as "not drained" β€” so every roll would ride to the deadline and cut + /// every session. Proved against the deny-by-default fallback above, which the control case + /// shows is real. + /// + [Fact] + public async Task TheDrainRoute_AnswersWithoutASession() + { + var (response, _) = await GetAsync(DrainRoute, new ActiveCircuitTracker()); + + response.StatusCode.Should().NotBe(HttpStatusCode.Unauthorized); + response.StatusCode.Should().NotBe(HttpStatusCode.Forbidden); + response.StatusCode.Should().Be(HttpStatusCode.OK); + } + + /// + /// A host with no Blazor tracker at all (a worker, a test host) is trivially drained. Pinned + /// because the alternative β€” a 500 from a missing service β€” is read by curl -sf as + /// "keep waiting", so the pod would sit out its whole window and then be killed anyway. + /// + [Fact] + public async Task WithNoCircuitTrackerRegistered_TheRouteStillAnswersDrained() + { + var (response, body) = await GetAsync(DrainRoute, tracker: null); + + response.StatusCode.Should().Be(HttpStatusCode.OK); + body.Should().Be("drained"); + } +} diff --git a/test/Memex.Portal.Shared.Test/NoOpEmailSenderRefusalTest.cs b/test/Memex.Portal.Shared.Test/NoOpEmailSenderRefusalTest.cs new file mode 100644 index 0000000000..2cf506f58a --- /dev/null +++ b/test/Memex.Portal.Shared.Test/NoOpEmailSenderRefusalTest.cs @@ -0,0 +1,250 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Threading; +using System.Threading.Tasks; +using Memex.Portal.Shared.Email; +using MeshWeaver.Mesh; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Xunit; + +namespace Memex.Portal.Shared.Test; + +/// +/// 🚨 Email:Enabled=true with a sender that cannot deliver must NEVER produce a +/// Sent record (#2023). +/// +/// The defect: once the Graph sender moved into the MeshWeaver.Mail.MicrosoftGraph +/// module, an install could have Email:Enabled=true and still resolve +/// β€” whose contract was to report success without sending. Every +/// queued mail was then stamped New β†’ Sending β†’ Sent while nothing left the process: node +/// data saying delivered, no error, and no log (the send lines are Information under a Warning +/// default). Hours went into inboxes, junk folders and message traces because Sent was +/// trusted. +/// +/// These cases pin the whole refusal, at the three places it has to hold: the sender itself, +/// the send state machine that writes the status, and the watcher that would have claimed the mail +/// in the first place. The last one is what keeps mail recoverable β€” mail left New goes out +/// by itself once the module lands, mail stamped Sent is indistinguishable from mail that +/// really was sent and can never be re-driven. +/// +public class NoOpEmailSenderRefusalTest +{ + private static readonly TimeSpan TestBudget = TimeSpan.FromSeconds(10); + + private static EmailOptions Configured(bool enabled) => new() + { + Enabled = enabled, + MailboxAddress = "no-reply@example.test", + TenantId = "tenant", + ClientId = "client", + ClientSecret = "secret", + }; + + /// + /// The one case the no-op is FOR: mail is switched off, nothing queues, and a caller's chain + /// completes normally. Kept as the control β€” without it "refuses everything" would satisfy the + /// test below while breaking every local dev run and test host. + /// + [Fact] + public async Task Disabled_TheNoOpStillReportsSuccess_SoLocalDevAndTestsAreUnaffected() + { + IEmailSender sender = new NoOpEmailSender(Configured(enabled: false)); + + Assert.False(sender.DeliversMail); + Assert.True(await sender.SendEmail("x@example.test", "s", "

b

").FirstAsync().ToTask()); + } + + /// + /// 🚨 The regression pin. Enabled + a sender that does not deliver is a MISCONFIGURATION, and + /// the send must surface as a failure rather than emit true β€” a caller that records an + /// outcome then records the truth. + /// + [Fact] + public async Task Enabled_ButTheSenderCannotDeliver_TheSendFails_ItNeverReportsSuccess() + { + IEmailSender sender = new NoOpEmailSender(Configured(enabled: true)); + + var failure = await Assert.ThrowsAsync( + () => sender.SendEmail("x@example.test", "s", "

b

").FirstAsync().ToTask()); + + // The message must name what to provision. "Email is broken" sends an operator hunting; + // the module name is the whole diagnosis. + Assert.Contains(EmailDeliveryGuard.SenderModule, failure.Message, StringComparison.Ordinal); + Assert.Contains("Email:Enabled=true", failure.Message, StringComparison.Ordinal); + } + + /// + /// 🚨 End to end through the code that writes the status: the real + /// on an Enabled=true install, driven by the real + /// . The mail must end Failed β€” visible and + /// re-queueable β€” and Sent must never appear in the status history at all. Asserting + /// the FINAL status alone would be weaker: the defect wrote Sent, and a later correction to + /// Failed would still have left a window in which the node claimed delivery. + /// + [Fact] + public async Task Enabled_ButTheSenderCannotDeliver_QueuedMailEndsFailed_AndIsNeverStampedSent() + { + var sender = new NoOpEmailSender(Configured(enabled: true)); + var store = new ConcurrentDictionary(); + var statusHistory = new List(); + + using var queue = new OutboundSendQueue( + readCurrent: path => Observable.Defer(() => Observable.Return( + store.TryGetValue(path, out var mail) ? mail : null)), + writeStatus: (node, current, status) => Observable.Defer(() => + { + lock (statusHistory) statusHistory.Add(status); + store[node.Path] = current with { Status = status }; + return Observable.Return(node); + }), + send: mail => sender.SendEmail(mail.To!, mail.Subject, mail.Body)); + + var email = new MeshWeaver.Mesh.Email + { + Direction = EmailDirection.Outbound, + Status = EmailStatus.New, + To = "recipient@example.test", + Subject = "s", + Body = "b", + }; + var node = new MeshNode("m1", "T") { NodeType = "Email", Name = "m1", Content = email }; + store[node.Path] = email; + + var processed = queue.Processed.Take(1).Timeout(TestBudget).FirstAsync().ToTask(); + queue.Enqueue(node, email); + await processed; + + Assert.Equal(EmailStatus.Failed, store[node.Path].Status); + lock (statusHistory) + Assert.DoesNotContain(EmailStatus.Sent, statusHistory); + } + + /// + /// 🚨 The watcher must not even ARM on this configuration. This is the half that keeps the + /// mail RECOVERABLE: a watcher that starts and declines per-item has already claimed + /// (New β†’ Sending) whatever it picked up, whereas one that never starts leaves every + /// queued mail at New, so delivery resumes on its own once the module lands β€” no data + /// repair, no re-queueing by hand. + /// + /// Observed through the provider, because "did it arm" has no other outward sign: + /// BeginWatching's first act is rootServices.CreateScope(), which asks the + /// provider for an . Zero requests after + /// ApplicationStarted has fired means the watch was never established. Its own body is + /// wrapped in a try/catch, so a "did it throw" assertion would prove nothing. + /// + [Fact] + public async Task Enabled_ButTheSenderCannotDeliver_TheOutboundWatcherDoesNotStart() + { + var options = Configured(enabled: true); + var services = new ServiceCollection() + .AddSingleton(new NoOpEmailSender(options)) + .BuildServiceProvider(); + var recording = new RecordingProvider(services); + using var lifetime = new StartableLifetime(); + + var watcher = new OutboundEmailSender(recording, lifetime, options); + await watcher.StartAsync(CancellationToken.None); + lifetime.NotifyStarted(); + + Assert.Equal(0, recording.ScopeFactoryRequests); + watcher.Dispose(); + } + + /// + /// The control for the case above: a sender that DOES deliver must still arm the watcher. + /// Without this the previous test is satisfied by a watcher that never starts at all, which is + /// the failure mode that would silently stop all outbound mail on a correctly configured + /// install. + /// + [Fact] + public async Task Enabled_WithADeliveringSender_TheOutboundWatcherStarts() + { + var options = Configured(enabled: true); + var services = new ServiceCollection() + .AddSingleton(new DeliveringSender()) + .BuildServiceProvider(); + var recording = new RecordingProvider(services); + using var lifetime = new StartableLifetime(); + + var watcher = new OutboundEmailSender(recording, lifetime, options); + await watcher.StartAsync(CancellationToken.None); + lifetime.NotifyStarted(); + + Assert.True(recording.ScopeFactoryRequests > 0, + "a delivering sender must leave the outbound watch armed β€” otherwise the refusal above " + + "would be satisfied by a watcher that never runs for anyone."); + watcher.Dispose(); + } + + /// The guard's verdict table, stated once so every caller reads the same rule. + [Theory] + [InlineData(false, false, false)] // disabled + no-op: the intended configuration + [InlineData(false, true, false)] // disabled + a real sender: nothing to refuse + [InlineData(true, true, false)] // enabled + a real sender: the working install + [InlineData(true, false, true)] // 🚨 enabled + no-op: the refused configuration + public void TheGuard_RefusesExactlyTheEnabledButUndeliverableCombination( + bool enabled, bool delivers, bool refused) + { + IEmailSender sender = delivers + ? new DeliveringSender() + : new NoOpEmailSender(Configured(enabled)); + + Assert.Equal(refused, EmailDeliveryGuard.RefusesDelivery(Configured(enabled), sender)); + } + + /// + /// A sender resolving to nothing at all certainly cannot deliver. Pinned because the tempting + /// reading β€” "could not resolve, assume it is fine" β€” is the skip-trapdoor the guard exists to + /// remove. + /// + [Fact] + public void TheGuard_TreatsAnAbsentSenderAsUndeliverable() + => Assert.True(EmailDeliveryGuard.RefusesDelivery(Configured(enabled: true), sender: null)); + + /// A stand-in for the module's Graph sender: it claims delivery, nothing more. + private sealed class DeliveringSender : IEmailSender + { + public IObservable SendEmail(string toAddress, string subject, string htmlBody) + => Observable.Return(true); + + public IObservable SendEmail( + string toAddress, string subject, string htmlBody, + IReadOnlyCollection attachments) + => Observable.Return(true); + } + + /// Counts scope-factory requests β€” the outward sign that the watch was established. + private sealed class RecordingProvider(IServiceProvider inner) : IServiceProvider + { + private int scopeFactoryRequests; + + public int ScopeFactoryRequests => Volatile.Read(ref scopeFactoryRequests); + + public object? GetService(Type serviceType) + { + if (serviceType == typeof(IServiceScopeFactory)) + Interlocked.Increment(ref scopeFactoryRequests); + return inner.GetService(serviceType); + } + } + + /// + /// A lifetime whose can be fired on + /// demand β€” the real token the watcher registers its BeginWatching on. + /// + private sealed class StartableLifetime : IHostApplicationLifetime, IDisposable + { + private readonly CancellationTokenSource started = new(); + + public CancellationToken ApplicationStarted => started.Token; + public CancellationToken ApplicationStopping => CancellationToken.None; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() { } + public void NotifyStarted() => started.Cancel(); + public void Dispose() => started.Dispose(); + } +} diff --git a/test/MeshWeaver.Auth.Test/InvitationServiceTests.cs b/test/MeshWeaver.Auth.Test/InvitationServiceTests.cs index aec222a76a..a06bb76bca 100644 --- a/test/MeshWeaver.Auth.Test/InvitationServiceTests.cs +++ b/test/MeshWeaver.Auth.Test/InvitationServiceTests.cs @@ -134,11 +134,16 @@ public async Task MarkAccepted_FlipsStatus_StampsAcceptedAt() stored.AcceptedAt.Should().NotBeNull(); } - /// The NoOp sender (Email:Enabled=false) reports success without sending. + /// + /// The NoOp sender reports success without sending β€” but ONLY on the configuration it is + /// meant for, Email:Enabled=false, where nothing queues mail and no watcher runs. The + /// Enabled=true half of the contract (a refusal, never a false success) is pinned by + /// Memex.Portal.Shared.Test.NoOpEmailSenderRefusalTest, next to the watchers it stops. + /// [Fact(Timeout = 10000)] public async Task NoOpEmailSender_ReturnsTrue_WithoutSending() { - IEmailSender sender = new NoOpEmailSender(); + IEmailSender sender = new NoOpEmailSender(new EmailOptions { Enabled = false }); (await sender.SendEmail("x@example.com", "hi", "

body

").Should().Emit()).Should().BeTrue(); } } diff --git a/test/MeshWeaver.Documentation.Test/DrainDeadlineGuard.cs b/test/MeshWeaver.Documentation.Test/DrainDeadlineGuard.cs index bfa253f599..cbb4ae8b54 100644 --- a/test/MeshWeaver.Documentation.Test/DrainDeadlineGuard.cs +++ b/test/MeshWeaver.Documentation.Test/DrainDeadlineGuard.cs @@ -36,6 +36,13 @@ public class DrainDeadlineGuard private const string Deployment = "deploy/helm/templates/memex-portal/deployment.yaml"; private const string Values = "deploy/helm/values.yaml"; + /// Where the drain endpoint the preStop loop probes is actually mapped. + private const string ServiceDefaults = "memex/aspire/Memex.Portal.ServiceDefaults/ServiceDefaults.cs"; + + /// The base image the chart's default portal image is built on β€” where the probe tool + /// preStop shells out to has to actually exist. + private const string PortalBaseImage = "deploy/base-images/portal-ai/Dockerfile"; + /// /// The host's own shutdown budget (Memex.Portal.Distributed/Program.cs). The margin /// exists to make room for exactly this, so it must not be smaller. @@ -114,6 +121,106 @@ public void TheGraceCeiling_IsStillDrainSeconds() + "source for the ceiling makes the margin meaningless."); } + /// + /// 🚨 The chart must probe a URL the code actually answers on. + /// + /// Everything above is arithmetic INSIDE the chart. This is the other half of #1971 β€” + /// the agreement between the chart and the process β€” and it is the half that has already + /// broken once: the preStop hook probed with wget, which is absent from the image, so + /// the loop could never succeed and EVERY termination hung to the grace ceiling. Nothing + /// failed; a chart and a container image simply disagreed. + /// + /// The same disagreement is available in two more places, and both are just as silent. + /// preStop probes with curl -sf -m 5 -o /dev/null, which cannot tell a 404 from a 503 + /// from a refused connection β€” all three read as "not drained yet, keep waiting". So renaming + /// the route, or moving the app off the probed port, does not error: it makes every roll sit + /// out its whole drain window and then cut every open session at the deadline. The path and + /// port are therefore part of the chart↔code contract, and this pins them to the one place + /// each is declared. (What the endpoint ANSWERS β€” 200 drained / 503 with the count, and + /// without a session β€” is pinned over real HTTP by DrainEndpointTest.) + /// + [Fact] + public void ThePreStopProbe_TargetsARouteTheCodeMaps_OnThePortTheContainerServes() + { + var root = FindRepoRoot(); + var deployment = ExecutableLinesOf(File.ReadAllText(Path.Combine(root, Deployment))); + + var probe = Regex.Match(deployment, @"curl[^\n]*?http://127\.0\.0\.1:(?\d+)(?/\S*)"); + Assert.True(probe.Success, + $"{Deployment}'s preStop must probe the drain endpoint over loopback HTTP so this " + + "guard can read the port and path it targets. If the probe shape changes, update " + + "this guard deliberately β€” do not let it stop matching."); + + var path = probe.Groups["path"].Value.TrimEnd(';'); + var port = probe.Groups["port"].Value; + + var serviceDefaults = File.ReadAllText(Path.Combine(root, ServiceDefaults)); + Assert.True( + serviceDefaults.Contains($"MapGet(\"{path}\"", StringComparison.Ordinal), + $"{Deployment}'s preStop probes '{path}', but {ServiceDefaults} does not map that " + + "route. curl -sf reads the resulting 404 as 'still draining', so the drain would " + + "silently run to its deadline on every roll and cut every open session (#1971)."); + + Assert.True( + serviceDefaults.Contains(".AllowAnonymous()", StringComparison.Ordinal), + $"the drain route must stay anonymous β€” a preStop exec carries no cookie and no " + + "token, and a 401/302 is indistinguishable from 'still draining' to curl -sf."); + + // The port the container actually serves on, from the chart's own declaration of it. + Assert.True( + deployment.Contains($"containerPort: {port}", StringComparison.Ordinal), + $"{Deployment}'s preStop probes port {port}, which is not a containerPort this " + + "deployment declares. A probe of a port nothing listens on is a refused connection, " + + "which curl -sf reports exactly like an open session."); + + var values = File.ReadAllText(Path.Combine(root, Values)); + Assert.True( + Regex.IsMatch(values, $@"ASPNETCORE_HTTP_PORTS:\s*""?{Regex.Escape(port)}""?"), + $"{Values} must configure the portal to listen on {port} (ASPNETCORE_HTTP_PORTS) β€” the " + + "port the preStop probe targets. Two independent numbers drift, and the drift is " + + "invisible until a roll cuts sessions."); + } + + /// + /// 🚨 The probe tool must be IN the image. This is the scar itself, pinned: preStop once + /// probed with wget on the reasoning "curl is not in the base image; wget is (Debian + /// aspnet)" β€” false for the image this chart actually deploys. `while ! wget …` with no wget can + /// never succeed, so the loop would spin to the grace ceiling and SIGKILL, turning every single + /// pod termination into a thirty-minute hang. It was found by reading a running container, not + /// by anything that could fail. + /// + /// The chart hedges with command -v curl … || exit 0, and that hedge is correct β€” + /// losing the session drain costs one rollout's circuits, while hanging every pod to the ceiling + /// costs every rollout. But a fail-open hedge is exactly the shape that makes a broken drain + /// SILENT, so the hedge must never be the only thing standing between the chart and the image. + /// This is the assertion that fails loudly instead. + /// + [Fact] + public void ThePreStopProbeTool_IsInstalledInThePortalBaseImage() + { + var root = FindRepoRoot(); + var preStop = ExecutableLinesOf(File.ReadAllText(Path.Combine(root, Deployment))); + + var guard = Regex.Match(preStop, @"command -v (?\w+)"); + Assert.True(guard.Success, + $"{Deployment}'s preStop must keep its `command -v ` guard β€” without it, a missing " + + "probe tool hangs every termination to the grace ceiling instead of skipping the drain."); + + var tool = guard.Groups["tool"].Value; + Assert.True(preStop.Contains($"{tool} -sf", StringComparison.Ordinal) + || Regex.IsMatch(preStop, $@"!\s*{Regex.Escape(tool)}\b"), + $"{Deployment}'s preStop guards on '{tool}' but polls with something else β€” the guard " + + "then protects the wrong binary and the loop can still spin to the ceiling."); + + var dockerfile = File.ReadAllText(Path.Combine(root, PortalBaseImage)); + Assert.True( + Regex.IsMatch(dockerfile, $@"apt-get install[^\n]*\b{Regex.Escape(tool)}\b"), + $"{Deployment}'s preStop probes with '{tool}', but {PortalBaseImage} does not install " + + $"it. The chart's `command -v {tool} || exit 0` hedge then SKIPS the session drain on " + + "every roll β€” silently, because a skipped drain and a fast one look identical from " + + "outside. This is the wget defect (#1971), and it shipped once already."); + } + /// /// 🚨 Comment lines are stripped before every probe above. Three of the strings this guard /// looks for also appear in the prose explaining them, so without stripping the guard would be diff --git a/test/MeshWeaver.Documentation.Test/MigrationWorkloadModelGuard.cs b/test/MeshWeaver.Documentation.Test/MigrationWorkloadModelGuard.cs new file mode 100644 index 0000000000..61c270fc1a --- /dev/null +++ b/test/MeshWeaver.Documentation.Test/MigrationWorkloadModelGuard.cs @@ -0,0 +1,213 @@ +using System; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Xunit; + +namespace MeshWeaver.Documentation.Test; + +/// +/// 🚨 The database migration is a RUN-ONCE workload, and nothing in this repo may say otherwise +/// (#1788). +/// +/// What went wrong. The migration container runs the migration, prints +/// Database migration completed. Version: N, and exits 0. Modelled as a Deployment +/// that is a crash loop by construction: the kubelet restarts it, forever. Three production +/// namespaces sat at 50/53/38 restarts (memex-cloud reached 87 in nine hours), and each restart +/// rebuilt public.top_level_index across every partition schema β€” so the "benign +/// CrashLoopBackOff" the docs told everyone to expect was a CPU storm. +/// +/// Why it is a correctness problem, not tidiness. A standing crash loop makes "is +/// anything crash-looping in prod?" β€” the first question anyone asks in an incident β€” permanently +/// unanswerable: a genuine migration failure looks exactly like the documented noise. That is this +/// repo's central failure mode (silence indistinguishable from success), and the cure is to model +/// the workload correctly so the ambiguity cannot exist. +/// +/// The chart is already right; the PROSE was what kept the wrong model alive. +/// deploy/helm/templates/memex-migration/job.yaml has rendered a Job since #145 β€” +/// there is no Deployment template and there never was one to port. What survived was a set of +/// commands, in AGENTS.md and the deploy scripts, telling operators to +/// kubectl set image / rollout restart a memex-migration-deployment the chart +/// does not define. Every one of those either fails or keeps a cluster-only orphan alive. This +/// guard is what stops them coming back β€” a text guard because the invariant lives across a Helm +/// template, several shell scripts and a markdown file, where no unit test can observe it. +/// +/// 🚨 What this guard deliberately does NOT assert: that no file MENTIONS +/// memex-migration-deployment. Several must β€” deploy/aks/SELF-UPDATE.md and +/// Doc/Architecture/DeploymentAKS.md carry the warning that the self-updater still targets +/// that name, and the chart's own RBAC still grants it (removing that grant before the updater +/// stops patching it would turn a harmless call into a 403 mid-roll). Naming the problem is the +/// opposite of repeating it. The assertion is on COMMANDS. +/// +public class MigrationWorkloadModelGuard +{ + private const string MigrationTemplates = "deploy/helm/templates/memex-migration"; + private const string Job = "deploy/helm/templates/memex-migration/job.yaml"; + + /// + /// The workload itself: a Job that runs to completion and stops. restartPolicy: Never + /// is the line that makes "it exited 0" mean finished rather than "restart it". + /// + [Fact] + public void TheMigration_IsAJobThatRunsToCompletion_NeverADeployment() + { + var dir = Path.Combine(FindRepoRoot(), MigrationTemplates); + var templates = Directory.GetFiles(dir, "*.yaml"); + + foreach (var template in templates) + { + var body = ExecutableLinesOf(File.ReadAllText(template)); + Assert.False( + Regex.IsMatch(body, @"^kind:\s*""?Deployment""?", RegexOptions.Multiline), + $"{Path.GetRelativePath(FindRepoRoot(), template)} declares a Deployment. The " + + "migration runs once and exits 0, so a Deployment restarts it forever β€” 310 " + + "restarts a day pegging a core, and a permanent CrashLoopBackOff that makes a " + + "REAL migration failure unreadable (#1788)."); + } + + var job = ExecutableLinesOf(File.ReadAllText(Path.Combine(FindRepoRoot(), Job))); + + Assert.True(Regex.IsMatch(job, @"^kind:\s*""?Job""?", RegexOptions.Multiline), + $"{Job} must declare kind: Job β€” the workload model is the fix."); + + Assert.True(job.Contains("restartPolicy: \"Never\"", StringComparison.Ordinal), + $"{Job} must set restartPolicy: Never. Without it the pod is restarted on exit and the " + + "Job form buys nothing."); + + Assert.True(job.Contains(".Release.Revision", StringComparison.Ordinal), + $"{Job}'s name must embed .Release.Revision so a new migration actually RUNS on the " + + "next helm upgrade β€” a Job with a stable name is created once and then silently " + + "skipped, which would leave the schema behind while everything looked fine."); + + Assert.True(job.Contains("ttlSecondsAfterFinished", StringComparison.Ordinal), + $"{Job} must set ttlSecondsAfterFinished so completed Jobs clean themselves up rather " + + "than accumulating one object per upgrade."); + } + + /// + /// 🚨 And no command anywhere in the repo may roll the migration as a Deployment. Each of + /// these was live until #1788: AGENTS.md β€” the file every agent loads first β€” carried + /// both a set image and a rollout restart against + /// deployment/memex-migration-deployment, plus the instruction to treat the resulting + /// crash loop as benign. deploy/aks/scripts/deploy.sh carried the same two, one of them + /// unguarded, so a documented deploy always printed an error nobody read. + /// + /// The scan is over COMMANDS: a kubectl verb applied to that object name. Prose + /// that names it in order to warn about it is untouched, and must stay that way β€” the + /// self-updater and the chart's RBAC still target it, and the day that changes is the day the + /// warnings can go. + /// + [Fact] + public void NoCommandInThisRepo_RollsTheMigrationAsADeployment() + { + var root = FindRepoRoot(); + + // kubectl ... deployment/memex-migration-deployment (or "deployment memex-…") + var command = new Regex( + @"kubectl[^\n]*?\bdeployment[/ ]memex-migration-deployment\b", + RegexOptions.IgnoreCase); + + var offenders = ScannedFiles(root) + .Where(file => command.IsMatch(File.ReadAllText(file))) + .Select(file => Path.GetRelativePath(root, file)) + .OrderBy(p => p, StringComparer.Ordinal) + .ToArray(); + + Assert.True(offenders.Length == 0, + "these files still issue a kubectl command against 'memex-migration-deployment', a " + + "workload the chart does not define (it renders a Job β€” see " + + $"{Job}): {string.Join(", ", offenders)}. Such a command either errors, or keeps a " + + "cluster-only orphan Deployment alive that re-runs the migration forever (#1788)."); + } + + /// + /// 🚨 And nothing may teach the reader to EXPECT the crash loop. "Benign `CrashLoopBackOff`" is + /// how the wrong model survived so many deploys: it told every operator and every agent that a + /// crash-looping migration pod is the normal state, which is exactly what made a real migration + /// failure invisible. The pod restarting is not benign and, with the Job model, does not happen. + /// + /// 🚨 A claim, not a rebuttal. The phrase must stay quotable, because several + /// files exist specifically to refute it β€” job.yaml's own header ("the 'benign + /// CrashLoopBackOff' the docs assumed was actually a CPU storm") and the verify steps in + /// DeploymentAKS.md / MemexCloudDeployment.md. So a match is an offence only when + /// the SENTENCE carrying it does not also negate or historicise it. That is a tripwire on the + /// wording, deliberately weaker than the command scan above: a determined rewording gets + /// through, but the operative regression β€” a command that rolls the migration as a Deployment β€” + /// is caught structurally either way. + /// + [Fact] + public void NoDocument_TeachesThatACrashLoopingMigrationIsNormal() + { + var root = FindRepoRoot(); + + // "benign"/"harmless"/"expected" within the same sentence as the crash loop. + var excuse = new Regex( + @"[^.\n]*?(?:(?:benign|harmless|expected|normal)[^.\n]{0,80}CrashLoopBackOff" + + @"|CrashLoopBackOff[^.\n]{0,80}(?:is|are)\s+(?:benign|harmless|expected|normal))[^.\n]*", + RegexOptions.IgnoreCase); + + // A sentence that says the phrase in order to deny it, or to report that it USED to be + // said, is the opposite of the defect and must keep working. + string[] rebuttals = + ["not ", "n't", "never", "no longer", "used to", "assumed", "previously", "was ", "were ", + "replaced", "legacy", "wrongly", "falsely", "instead of"]; + + var offenders = ScannedFiles(root) + .Select(file => (File: file, Claims: excuse.Matches(File.ReadAllText(file)) + .Select(m => m.Value) + .Where(sentence => !rebuttals.Any(r => + sentence.Contains(r, StringComparison.OrdinalIgnoreCase))) + .ToArray())) + .Where(x => x.Claims.Length > 0) + .Select(x => $"{Path.GetRelativePath(root, x.File)} ({x.Claims[0].Trim()})") + .OrderBy(p => p, StringComparer.Ordinal) + .ToArray(); + + Assert.True(offenders.Length == 0, + "these files describe a CrashLoopBackOff as benign/expected: " + + $"{string.Join(" Β· ", offenders)}. A standing crash loop makes 'is anything " + + "crash-looping in prod?' permanently unanswerable, so a genuine failure reads as the " + + "documented noise. Model the workload correctly instead of documenting the symptom " + + "(#1788)."); + } + + /// + /// The files this guard reads: the operator-facing prose and scripts that can put a command in + /// front of a human. Deliberately NOT the whole tree β€” bin/, obj/ and the test + /// sources (this file names both strings) would make it match itself. + /// + private static string[] ScannedFiles(string root) => + [ + .. new[] { "AGENTS.md", "CLAUDE.md" } + .Select(name => Path.Combine(root, name)) + .Where(File.Exists), + .. Directory.EnumerateFiles(Path.Combine(root, "deploy"), "*", SearchOption.AllDirectories) + .Where(f => f.EndsWith(".md", StringComparison.Ordinal) + || f.EndsWith(".sh", StringComparison.Ordinal) + || f.EndsWith(".yaml", StringComparison.Ordinal)), + .. Directory.EnumerateFiles( + Path.Combine(root, "src", "MeshWeaver.Documentation", "Data"), + "*.md", SearchOption.AllDirectories), + .. Directory.EnumerateFiles(Path.Combine(root, "content"), "*.md", SearchOption.AllDirectories), + ]; + + /// + /// Comment lines are stripped before probing the templates, for the same reason + /// strips them: job.yaml's header explains the + /// Deployment it REPLACED, so an unstripped scan would find "Deployment" in the explanation + /// and fail on the very comment that documents the fix. + /// + private static string ExecutableLinesOf(string yaml) => + string.Join("\n", yaml.Split('\n') + .Where(line => !line.TrimStart().StartsWith('#'))); + + private static string FindRepoRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "MeshWeaver.slnx"))) + dir = dir.Parent; + return dir?.FullName + ?? throw new InvalidOperationException( + "Could not locate the repo root (MeshWeaver.slnx) from " + AppContext.BaseDirectory); + } +}