From 22ef1ab87478426be2623a100a7b198340882943 Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 14 Oct 2026 12:57:34 -0700 Subject: [PATCH 1/2] kind: deploy apiserver uses the kind overlay On kind, the system bundle patches ate-api-server so its object-store plugin uses the in-cluster rustfs. deploy apiserver rendered the base ate-api-server.yaml instead. A kind redeploy therefore reset the plugin to GCS, and the new pod never became ready. Move the patch into a kind/ate-api-server overlay, the same way kind/atelet is set up. The kind bundle builds on that overlay and DeployAteAPIServer applies it on kind. The rendered kind bundles are unchanged. --- cmd/ate-setup/internal/steps/deploy.go | 10 ++- cmd/ate-setup/internal/steps/deploy_test.go | 38 ++++++++++++ cmd/ate-setup/internal/steps/overlay_test.go | 6 ++ docs/dev/best-practices/otel-collector.md | 5 +- .../kind/ate-api-server/kustomization.yaml | 61 +++++++++++++++++++ manifests/ate-install/kind/kustomization.yaml | 53 ++-------------- 6 files changed, 122 insertions(+), 51 deletions(-) create mode 100644 manifests/ate-install/kind/ate-api-server/kustomization.yaml diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 5d4ae0cb53..6d1489a782 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -306,7 +306,7 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error { return err } } - manifest, err := e.renderResolve(ctx, e.Cfg.Manifest("ate-api-server.yaml")) + manifest, err := e.renderResolve(ctx, e.apiServerManifestPath()) if err != nil { return err } @@ -320,6 +320,14 @@ func (e *Env) DeployAteAPIServer(ctx context.Context) error { return e.Kube.RolloutStatus(ctx, kube.KindDeployment, e.Namespace(), "ate-api-server", e.Cfg.RolloutTimeout) } +// apiServerManifestPath is the ate-api-server manifest for the environment. +func (e *Env) apiServerManifestPath() string { + if e.Cfg.Kind { + return e.Cfg.Manifest("kind", "ate-api-server") + } + return e.Cfg.Manifest("ate-api-server.yaml") +} + // DeployAteController redeploys only ate-controller. func (e *Env) DeployAteController(ctx context.Context) error { log.Step("deploy_ate_controller") diff --git a/cmd/ate-setup/internal/steps/deploy_test.go b/cmd/ate-setup/internal/steps/deploy_test.go index ceebebb89a..32470be276 100644 --- a/cmd/ate-setup/internal/steps/deploy_test.go +++ b/cmd/ate-setup/internal/steps/deploy_test.go @@ -20,6 +20,7 @@ import ( "path/filepath" "testing" + "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -183,3 +184,40 @@ func TestMissingCRDsLeavesPresentOnesAlone(t *testing.T) { t.Errorf("missingCRDs = %v, want only csidriverconfigs.ate.dev", names) } } + +// deploy apiserver has to apply the ate-api-server that deploy ate-system +// does. On kind the bundle points its object-store plugin at rustfs, and a +// redeploy from the base file would fall back to GCS and never start. +func TestAPIServerManifestMatchesSystemBundle(t *testing.T) { + for _, tc := range []struct { + router string + kind bool + }{ + {config.RouterEnvoy, false}, + {config.RouterEnvoy, true}, + {config.RouterAgentgateway, false}, + {config.RouterAgentgateway, true}, + } { + e := &Env{Cfg: &config.Config{Root: repoRoot(t), Router: tc.router, Kind: tc.kind}} + deployment := func(path string) *unstructured.Unstructured { + manifest, err := e.render(path) + if err != nil { + t.Fatalf("render(%s): %v", path, err) + } + objs, err := kube.DecodeManifestBytes(manifest) + if err != nil { + t.Fatalf("decoding %s: %v", path, err) + } + dep := findObject(objs, "Deployment", "ate-api-server") + if dep == nil { + t.Fatalf("%s has no deployment/ate-api-server", path) + } + return dep + } + bundle := deployment(e.Cfg.Path(SystemOverlay(e.Cfg))) + standalone := deployment(e.apiServerManifestPath()) + if diff := cmp.Diff(bundle.Object, standalone.Object); diff != "" { + t.Errorf("router=%s kind=%v: deploy apiserver renders a different ate-api-server than the system bundle (-bundle +standalone):\n%s", tc.router, tc.kind, diff) + } + } +} diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index 8934509f42..c22ea9da58 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -440,6 +440,12 @@ func TestRenderCordonControlPlane(t *testing.T) { path: func(e *Env) string { return e.Cfg.Manifest("ate-api-server.yaml") }, want: []string{"ate-api-server"}, }, + { + name: "kind api server overlay", + cfg: config.Config{Kind: true}, + path: func(e *Env) string { return e.apiServerManifestPath() }, + want: []string{"ate-api-server"}, + }, { name: "podcert file", path: func(e *Env) string { return e.Cfg.Manifest("pod-certificate-controller.yaml") }, diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index 81d584b809..c533bf9aa1 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -428,8 +428,9 @@ the worker pods it creates, and `atenet-router`'s `--otlp-collector-address`, which is what its Envoy is given over xDS. Rather than editing the base manifests, prefer a kustomize overlay that -patches the variable — see `manifests/ate-install/kind/kustomization.yaml` -for a worked example. +replaces the `ate-otel-config` ConfigMap — see +`manifests/ate-install/kind/ate-otel-config.yaml`, which the kind overlay +lists instead of the base file, for a worked example. ### A Note on Logs diff --git a/manifests/ate-install/kind/ate-api-server/kustomization.yaml b/manifests/ate-install/kind/ate-api-server/kustomization.yaml new file mode 100644 index 0000000000..c362bb6b00 --- /dev/null +++ b/manifests/ate-install/kind/ate-api-server/kustomization.yaml @@ -0,0 +1,61 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../ate-api-server.yaml + +# OTEL_TRACES_SAMPLER pins ate-api-server to parentbased_always_on so every API +# call it roots is traced; the other components keep their binary defaults and +# follow its sampled flag via their ParentBased samplers. It stays an inline +# patch rather than a key in ate-otel-config.yaml because that ConfigMap is +# shared by every component via envFrom -- putting always_on there would pin the +# router and the control plane to 100% too, undoing their per-component ratios. +# +# The storage settings point ate-api-server's object-store plugin at the same +# rustfs the ../atelet overlay points atelet's at: both ends of an external +# snapshot's life have to agree on where it lives. +patches: + - patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ate-api-server + namespace: ate-system + spec: + template: + spec: + containers: + - name: ate-api-server + env: + - name: OTEL_TRACES_SAMPLER + value: parentbased_always_on + initContainers: + - name: objectstore-plugin + env: + - name: ATE_STORAGE_BACKEND + value: s3 + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + # TODO: use a secret / identity management + - name: AWS_ACCESS_KEY_ID + value: rustfsadmin + - name: AWS_SECRET_ACCESS_KEY + value: rustfsadmin diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 6562870924..ff65fefdd9 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -17,17 +17,17 @@ kind: Kustomization # ate-otel-config.yaml replaces the base ../ate-otel-config.yaml (same ConfigMap # name) so every component's `envFrom` resolves to the in-cluster collector -# instead of the GKE one. It is listed here rather than in ./atelet because that -# overlay is also built standalone; listing it in both would be a duplicate -# resource. hack/install-ate.sh applies the ConfigMap directly for the targeted -# single-component redeploys. +# instead of the GKE one. It is listed here rather than in ./atelet or +# ./ate-api-server because those overlays are also built standalone; listing it +# in the bundle and an overlay would be a duplicate resource. ate-setup applies +# the ConfigMap directly for the single-component redeploys. # # pod-certificate-controller.yaml is deliberately absent, as in ../base: the # installers (hack/install-ate.sh and cmd/ate-setup) apply and delete it on # their own, and re-applying the base file from this bundle would revert the # WORKERS_PER_SIGNER override. resources: - - ../ate-api-server.yaml + - ./ate-api-server - ../ate-controller.yaml - ./atelet - ../atenet-router.yaml @@ -36,46 +36,3 @@ resources: - rustfs.yaml - ./otel-collector.yaml - ./prometheus.yaml - -# OTEL_TRACES_SAMPLER pins ate-api-server to parentbased_always_on so every API -# call it roots is traced; the other components keep their binary defaults and -# follow its sampled flag via their ParentBased samplers. It stays an inline -# patch rather than a key in ate-otel-config.yaml because that ConfigMap is -# shared by every component via envFrom -- putting always_on there would pin the -# router and the control plane to 100% too, undoing their per-component ratios. -# -# The storage settings point ate-api-server's object-store plugin at the same -# rustfs the ./atelet overlay points atelet's at: both ends of an external -# snapshot's life have to agree on where it lives. -patches: - - patch: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: ate-api-server - namespace: ate-system - spec: - template: - spec: - containers: - - name: ate-api-server - env: - - name: OTEL_TRACES_SAMPLER - value: parentbased_always_on - initContainers: - - name: objectstore-plugin - env: - - name: ATE_STORAGE_BACKEND - value: s3 - - name: AWS_REGION - value: us-east-1 - - name: AWS_ENDPOINT_URL - value: http://rustfs.ate-system.svc:9000 - - name: AWS_S3_USE_PATH_STYLE - value: "true" - # TODO: use a secret / identity management - - name: AWS_ACCESS_KEY_ID - value: rustfsadmin - - name: AWS_SECRET_ACCESS_KEY - value: rustfsadmin - From 9a3adf2076dad5980198206f5755fe7afe2a824b Mon Sep 17 00:00:00 2001 From: Haven Xia Date: Wed, 14 Oct 2026 21:01:30 -0700 Subject: [PATCH 2/2] ate-setup: compare deploy atelet with the system bundle too Give atelet the same manifest path helper as ate-api-server, and compare its DaemonSet against the system bundle in the same test. If a kind redeploy of atelet ever went back to the base file, it would hit the bug ate-api-server had, and the test would catch it. --- cmd/ate-setup/internal/steps/deploy.go | 15 +++++---- cmd/ate-setup/internal/steps/deploy_test.go | 35 ++++++++++++-------- cmd/ate-setup/internal/steps/overlay_test.go | 2 +- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 6d1489a782..dd015eb2bb 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -473,12 +473,7 @@ func (e *Env) DeployAtelet(ctx context.Context) error { return err } - path := e.Cfg.Manifest("atelet.yaml") - if e.Cfg.Kind { - // The kind overlay patches the DaemonSet for the local node layout. - path = e.Cfg.Path(installDir + "/kind/atelet") - } - manifest, err := e.render(path) + manifest, err := e.render(e.ateletManifestPath()) if err != nil { return err } @@ -495,6 +490,14 @@ func (e *Env) DeployAtelet(ctx context.Context) error { return e.waitFakeWorkersync(ctx) } +// ateletManifestPath is the atelet manifest for the environment. +func (e *Env) ateletManifestPath() string { + if e.Cfg.Kind { + return e.Cfg.Manifest("kind", "atelet") + } + return e.Cfg.Manifest("atelet.yaml") +} + // DeployAtenet redeploys the atenet dataplane: router and egress. func (e *Env) DeployAtenet(ctx context.Context) error { log.Step("deploy_atenet") diff --git a/cmd/ate-setup/internal/steps/deploy_test.go b/cmd/ate-setup/internal/steps/deploy_test.go index 32470be276..4edffc0012 100644 --- a/cmd/ate-setup/internal/steps/deploy_test.go +++ b/cmd/ate-setup/internal/steps/deploy_test.go @@ -185,10 +185,10 @@ func TestMissingCRDsLeavesPresentOnesAlone(t *testing.T) { } } -// deploy apiserver has to apply the ate-api-server that deploy ate-system -// does. On kind the bundle points its object-store plugin at rustfs, and a -// redeploy from the base file would fall back to GCS and never start. -func TestAPIServerManifestMatchesSystemBundle(t *testing.T) { +// A single-component redeploy has to apply what deploy ate-system does. On +// kind the bundle points the object-store plugins at rustfs, and a redeploy +// from a base file would fall back to GCS and never start. +func TestComponentManifestsMatchSystemBundle(t *testing.T) { for _, tc := range []struct { router string kind bool @@ -199,7 +199,7 @@ func TestAPIServerManifestMatchesSystemBundle(t *testing.T) { {config.RouterAgentgateway, true}, } { e := &Env{Cfg: &config.Config{Root: repoRoot(t), Router: tc.router, Kind: tc.kind}} - deployment := func(path string) *unstructured.Unstructured { + render := func(path string) []*unstructured.Unstructured { manifest, err := e.render(path) if err != nil { t.Fatalf("render(%s): %v", path, err) @@ -208,16 +208,23 @@ func TestAPIServerManifestMatchesSystemBundle(t *testing.T) { if err != nil { t.Fatalf("decoding %s: %v", path, err) } - dep := findObject(objs, "Deployment", "ate-api-server") - if dep == nil { - t.Fatalf("%s has no deployment/ate-api-server", path) - } - return dep + return objs } - bundle := deployment(e.Cfg.Path(SystemOverlay(e.Cfg))) - standalone := deployment(e.apiServerManifestPath()) - if diff := cmp.Diff(bundle.Object, standalone.Object); diff != "" { - t.Errorf("router=%s kind=%v: deploy apiserver renders a different ate-api-server than the system bundle (-bundle +standalone):\n%s", tc.router, tc.kind, diff) + bundle := render(e.Cfg.Path(SystemOverlay(e.Cfg))) + for _, component := range []struct { + kind, name, path string + }{ + {"Deployment", "ate-api-server", e.apiServerManifestPath()}, + {"DaemonSet", "atelet", e.ateletManifestPath()}, + } { + want := findObject(bundle, component.kind, component.name) + got := findObject(render(component.path), component.kind, component.name) + if want == nil || got == nil { + t.Fatalf("router=%s kind=%v: %s/%s missing from the system bundle or %s", tc.router, tc.kind, component.kind, component.name, component.path) + } + if diff := cmp.Diff(want.Object, got.Object); diff != "" { + t.Errorf("router=%s kind=%v: %s renders a different %s/%s than the system bundle (-bundle +standalone):\n%s", tc.router, tc.kind, component.path, component.kind, component.name, diff) + } } } } diff --git a/cmd/ate-setup/internal/steps/overlay_test.go b/cmd/ate-setup/internal/steps/overlay_test.go index c22ea9da58..d3ec4ad261 100644 --- a/cmd/ate-setup/internal/steps/overlay_test.go +++ b/cmd/ate-setup/internal/steps/overlay_test.go @@ -635,7 +635,7 @@ func TestRenderFakeDataPlane(t *testing.T) { { name: "kind atelet overlay", cfg: config.Config{Kind: true}, - path: func(e *Env) string { return e.Cfg.Path(installDir + "/kind/atelet") }, + path: func(e *Env) string { return e.ateletManifestPath() }, wantAtelet: true, wantStorage: "s3", }, {