From 32e2a5967303e0ec799e3296d790aad65be2c5f4 Mon Sep 17 00:00:00 2001 From: jramkissoon Date: Mon, 26 Oct 2026 12:41:39 -0400 Subject: [PATCH 1/2] Reject conflicting DRA kubelet-plugin healthcheck ports When ComputeDomains are enabled, the kubelet-plugin DaemonSet runs two containers that each bind a gRPC health service in the shared pod network namespace. With both configured on one port the second bind fails with EADDRINUSE and that container crash-loops; which one loses is a startup race. Validate the ports in the DRA driver state before rendering. The check runs on the resolved ports, so setting only one field to the other container's default is caught too, and a disabled health service is skipped. The error names both fields and the port and surfaces through the existing getManifestObjects path as a ReconcileFailed condition, leaving the DaemonSet untouched. Fixes #2979 Signed-off-by: jramkissoon --- internal/state/dra_driver.go | 44 ++++++++++++++----- internal/state/dra_driver_test.go | 71 +++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 11 deletions(-) diff --git a/internal/state/dra_driver.go b/internal/state/dra_driver.go index 53e82c95bc..f5f18d2fa6 100644 --- a/internal/state/dra_driver.go +++ b/internal/state/dra_driver.go @@ -56,6 +56,21 @@ func resolveHealthcheckPort(hc *nvidiav1alpha1.DRADriverHealthcheckSpec, default return defaultPort } +// validateHealthcheckPorts rejects a GPUCluster whose two kubelet-plugin containers +// would bind the same gRPC health port. Both containers share the pod's network +// namespace, so the second bind fails with EADDRINUSE and that container crash-loops; +// which one loses is a startup race. The check runs on resolved ports so a user who +// sets only one port to the other container's default is caught too. A port <= 0 +// means the health service is disabled, so no bind happens. +func validateHealthcheckPorts(computeDomainsEnabled bool, gpusPort, computeDomainsPort int32) error { + if !computeDomainsEnabled || gpusPort <= 0 || gpusPort != computeDomainsPort { + return nil + } + return fmt.Errorf("spec.draDriver.gpus.kubeletPlugin.healthcheck.port and "+ + "spec.draDriver.computeDomains.kubeletPlugin.healthcheck.port must differ "+ + "when computeDomains is enabled; both resolve to %d", gpusPort) +} + type stateDRADriver struct { stateSkel } @@ -118,18 +133,25 @@ func (s *stateDRADriver) getManifestObjects(ctx context.Context, cr *nvidiav1alp return nil, fmt.Errorf("failed to get OpenShift version: %w", err) } + gpusHealthcheckPort := resolveHealthcheckPort( + cr.Spec.DRADriver.GPUs.KubeletPlugin.Healthcheck, defaultGPUsHealthcheckPort) + computeDomainsHealthcheckPort := resolveHealthcheckPort( + cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Healthcheck, defaultComputeDomainsHealthcheckPort) + if err := validateHealthcheckPorts(cr.Spec.DRADriver.IsComputeDomainsEnabled(), + gpusHealthcheckPort, computeDomainsHealthcheckPort); err != nil { + return nil, fmt.Errorf("invalid DRA driver healthcheck configuration: %w", err) + } + renderData := &draDriverRenderData{ - DRADriver: draDriverSpec, - HostPaths: &hostPaths, - Daemonsets: &daemonsets, - Namespace: s.namespace, - OpenshiftVersion: openshiftVersion, - DeviceClassAPIVersion: apiVersion, - FeatureGates: cr.Spec.DRADriver.FeatureGates, - GPUsHealthcheckPort: resolveHealthcheckPort( - cr.Spec.DRADriver.GPUs.KubeletPlugin.Healthcheck, defaultGPUsHealthcheckPort), - ComputeDomainsHealthcheckPort: resolveHealthcheckPort( - cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Healthcheck, defaultComputeDomainsHealthcheckPort), + DRADriver: draDriverSpec, + HostPaths: &hostPaths, + Daemonsets: &daemonsets, + Namespace: s.namespace, + OpenshiftVersion: openshiftVersion, + DeviceClassAPIVersion: apiVersion, + FeatureGates: cr.Spec.DRADriver.FeatureGates, + GPUsHealthcheckPort: gpusHealthcheckPort, + ComputeDomainsHealthcheckPort: computeDomainsHealthcheckPort, } return s.renderObjects(ctx, renderData) diff --git a/internal/state/dra_driver_test.go b/internal/state/dra_driver_test.go index 1598a68bb4..598e49c1ed 100644 --- a/internal/state/dra_driver_test.go +++ b/internal/state/dra_driver_test.go @@ -331,6 +331,77 @@ func TestDRADriverHealthcheckDisabled(t *testing.T) { assert.Nil(t, gpus.LivenessProbe) } +func TestDRADriverHealthcheckPortCollision(t *testing.T) { + for name, tc := range map[string]struct { + computeDomainsEnabled bool + gpus *nvidiav1alpha1.DRADriverHealthcheckSpec + computeDomains *nvidiav1alpha1.DRADriverHealthcheckSpec + wantErr string + }{ + "same explicit port": { + computeDomainsEnabled: true, + gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))}, + computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))}, + wantErr: "both resolve to 52000", + }, + // Only one field is set, yet it collides with the other container's default; + // this is why the check runs on resolved ports rather than the raw spec. + "gpus set to the computeDomains default": { + computeDomainsEnabled: true, + gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(51515))}, + wantErr: "both resolve to 51515", + }, + "computeDomains set to the gpus default": { + computeDomainsEnabled: true, + computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(51516))}, + wantErr: "both resolve to 51516", + }, + "different ports": { + computeDomainsEnabled: true, + gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))}, + computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52001))}, + }, + "defaults do not collide": { + computeDomainsEnabled: true, + }, + "computeDomains disabled ignores collision": { + gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))}, + computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))}, + }, + "both health services disabled": { + computeDomainsEnabled: true, + gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Enabled: new(false)}, + computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Enabled: new(false)}, + }, + // The enabled side deliberately uses the disabled side's default port: it must + // pass because a disabled service never binds. + "one health service disabled": { + computeDomainsEnabled: true, + gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Enabled: new(false)}, + computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(51516))}, + }, + } { + t.Run(name, func(t *testing.T) { + s := newTestDRAState(t) + cr := sampleGPUCluster() + cr.Spec.DRADriver.ComputeDomains.Enabled = new(tc.computeDomainsEnabled) + cr.Spec.DRADriver.GPUs.KubeletPlugin.Healthcheck = tc.gpus + cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Healthcheck = tc.computeDomains + + objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + assert.ErrorContains(t, err, "spec.draDriver.gpus.kubeletPlugin.healthcheck.port") + assert.ErrorContains(t, err, "spec.draDriver.computeDomains.kubeletPlugin.healthcheck.port") + assert.Nil(t, objs) + return + } + require.NoError(t, err) + assert.NotEmpty(t, objs) + }) + } +} + func TestDRADriverRenderDRAUnsupported(t *testing.T) { s := newTestDRAState(t) From 152cc10ffbb7363783278a3b1a905e87de10a172 Mon Sep 17 00:00:00 2001 From: jramkissoon Date: Thu, 29 Oct 2026 17:10:30 -0400 Subject: [PATCH 2/2] Reject HEALTHCHECK_PORT in DRA kubelet-plugin env kubeletPlugin.env is appended after the HEALTHCHECK_PORT entry the template renders from healthcheck.port, and the last duplicate wins in the container. A user entry therefore moves the health service behind the back of the port validation: with ComputeDomains enabled, gpus.kubeletPlugin.env HEALTHCHECK_PORT=51515 passes the check yet both containers bind 51515. Even without a collision the override cannot work, since the probes keep the spec port. Reject HEALTHCHECK_PORT in kubeletPlugin.env for the gpus container, and for the compute-domains container when it is rendered, pointing the user at the healthcheck.port field. Fixes #2979 (cherry picked from commit 709e7d313758fb341c5a710dee3cfa5f25ee7768) Signed-off-by: jramkissoon --- internal/state/dra_driver.go | 35 ++++++++++++++++++++ internal/state/dra_driver_test.go | 55 +++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/internal/state/dra_driver.go b/internal/state/dra_driver.go index f5f18d2fa6..d8419be766 100644 --- a/internal/state/dra_driver.go +++ b/internal/state/dra_driver.go @@ -42,6 +42,12 @@ const ( // upstream k8s-dra-driver-gpu Helm chart. defaultGPUsHealthcheckPort = int32(51516) defaultComputeDomainsHealthcheckPort = int32(51515) + + // healthcheckPortEnvName is rendered from healthcheck.port. kubeletPlugin.env is + // appended after it and the last duplicate wins, so a user entry would move the + // health service away from both the port validation and its probes, which keep + // the spec port. + healthcheckPortEnvName = "HEALTHCHECK_PORT" ) // resolveHealthcheckPort returns the spec-provided health service port, or the @@ -71,6 +77,31 @@ func validateHealthcheckPorts(computeDomainsEnabled bool, gpusPort, computeDomai "when computeDomains is enabled; both resolve to %d", gpusPort) } +// validateKubeletPluginEnv rejects a GPUCluster whose kubeletPlugin.env sets +// HEALTHCHECK_PORT (see healthcheckPortEnvName). The gpus container is always +// rendered, so its env is always checked: the probe mismatch breaks it even without +// a collision. The compute-domains container is only rendered when enabled. +func validateKubeletPluginEnv(computeDomainsEnabled bool, gpusEnv, computeDomainsEnv []nvidiav1.EnvVar) error { + type envList struct { + field string + env []nvidiav1.EnvVar + } + lists := []envList{{"spec.draDriver.gpus.kubeletPlugin.env", gpusEnv}} + if computeDomainsEnabled { + lists = append(lists, envList{"spec.draDriver.computeDomains.kubeletPlugin.env", computeDomainsEnv}) + } + + for _, l := range lists { + for _, e := range l.env { + if e.Name == healthcheckPortEnvName { + return fmt.Errorf("%s must not set %s; it is managed by the operator "+ + "(use the healthcheck.port field for the health service port)", l.field, healthcheckPortEnvName) + } + } + } + return nil +} + type stateDRADriver struct { stateSkel } @@ -141,6 +172,10 @@ func (s *stateDRADriver) getManifestObjects(ctx context.Context, cr *nvidiav1alp gpusHealthcheckPort, computeDomainsHealthcheckPort); err != nil { return nil, fmt.Errorf("invalid DRA driver healthcheck configuration: %w", err) } + if err := validateKubeletPluginEnv(cr.Spec.DRADriver.IsComputeDomainsEnabled(), + cr.Spec.DRADriver.GPUs.KubeletPlugin.Env, cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Env); err != nil { + return nil, fmt.Errorf("invalid DRA driver kubelet-plugin env configuration: %w", err) + } renderData := &draDriverRenderData{ DRADriver: draDriverSpec, diff --git a/internal/state/dra_driver_test.go b/internal/state/dra_driver_test.go index 598e49c1ed..cf5e3c2cd2 100644 --- a/internal/state/dra_driver_test.go +++ b/internal/state/dra_driver_test.go @@ -402,6 +402,61 @@ func TestDRADriverHealthcheckPortCollision(t *testing.T) { } } +func TestDRADriverKubeletPluginReservedEnv(t *testing.T) { + for name, tc := range map[string]struct { + computeDomainsEnabled bool + gpusEnv []nvidiav1.EnvVar + computeDomainsEnv []nvidiav1.EnvVar + wantErr []string + }{ + "gpus overrides HEALTHCHECK_PORT": { + gpusEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51515"}}, + wantErr: []string{"spec.draDriver.gpus.kubeletPlugin.env", "HEALTHCHECK_PORT"}, + }, + // The typed ports stay at their distinct defaults, so validateHealthcheckPorts + // passes, yet the env entry moves gpus onto the computeDomains port. + "gpus overrides HEALTHCHECK_PORT onto the computeDomains port": { + computeDomainsEnabled: true, + gpusEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51515"}}, + wantErr: []string{"spec.draDriver.gpus.kubeletPlugin.env", "HEALTHCHECK_PORT"}, + }, + "computeDomains overrides HEALTHCHECK_PORT": { + computeDomainsEnabled: true, + computeDomainsEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51516"}}, + wantErr: []string{"spec.draDriver.computeDomains.kubeletPlugin.env", "HEALTHCHECK_PORT"}, + }, + // The second container is not rendered, so its env is never applied. + "computeDomains env ignored when disabled": { + computeDomainsEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51516"}}, + }, + "unrelated env is allowed": { + computeDomainsEnabled: true, + gpusEnv: []nvidiav1.EnvVar{{Name: "LOG_VERBOSITY", Value: "6"}}, + computeDomainsEnv: []nvidiav1.EnvVar{{Name: "ALT_PROC_DEVICES_PATH", Value: "/host/proc-devices"}}, + }, + } { + t.Run(name, func(t *testing.T) { + s := newTestDRAState(t) + cr := sampleGPUCluster() + cr.Spec.DRADriver.ComputeDomains.Enabled = new(tc.computeDomainsEnabled) + cr.Spec.DRADriver.GPUs.KubeletPlugin.Env = tc.gpusEnv + cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Env = tc.computeDomainsEnv + + objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) + if len(tc.wantErr) > 0 { + require.Error(t, err) + for _, want := range tc.wantErr { + assert.ErrorContains(t, err, want) + } + assert.Nil(t, objs) + return + } + require.NoError(t, err) + assert.NotEmpty(t, objs) + }) + } +} + func TestDRADriverRenderDRAUnsupported(t *testing.T) { s := newTestDRAState(t)