Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 68 additions & 11 deletions internal/state/dra_driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ const (
// upstream k8s-dra-driver-gpu Helm chart.
defaultGPUsHealthcheckPort = int32(51516)
defaultComputeDomainsHealthcheckPort = int32(51515)

// healthcheckPortEnvName is rendered from healthcheck.port. kubeletPlugin.env is
// appended after it and the last duplicate wins, so a user entry would move the
// health service away from both the port validation and its probes, which keep
// the spec port.
healthcheckPortEnvName = "HEALTHCHECK_PORT"
)

// resolveHealthcheckPort returns the spec-provided health service port, or the
Expand All @@ -56,6 +62,46 @@ func resolveHealthcheckPort(hc *nvidiav1alpha1.DRADriverHealthcheckSpec, default
return defaultPort
}

// validateHealthcheckPorts rejects a GPUCluster whose two kubelet-plugin containers
// would bind the same gRPC health port. Both containers share the pod's network
// namespace, so the second bind fails with EADDRINUSE and that container crash-loops;
// which one loses is a startup race. The check runs on resolved ports so a user who
// sets only one port to the other container's default is caught too. A port <= 0
// means the health service is disabled, so no bind happens.
func validateHealthcheckPorts(computeDomainsEnabled bool, gpusPort, computeDomainsPort int32) error {
if !computeDomainsEnabled || gpusPort <= 0 || gpusPort != computeDomainsPort {
return nil
}
return fmt.Errorf("spec.draDriver.gpus.kubeletPlugin.healthcheck.port and "+
"spec.draDriver.computeDomains.kubeletPlugin.healthcheck.port must differ "+
"when computeDomains is enabled; both resolve to %d", gpusPort)
}

// validateKubeletPluginEnv rejects a GPUCluster whose kubeletPlugin.env sets
// HEALTHCHECK_PORT (see healthcheckPortEnvName). The gpus container is always
// rendered, so its env is always checked: the probe mismatch breaks it even without
// a collision. The compute-domains container is only rendered when enabled.
func validateKubeletPluginEnv(computeDomainsEnabled bool, gpusEnv, computeDomainsEnv []nvidiav1.EnvVar) error {
type envList struct {
field string
env []nvidiav1.EnvVar
}
lists := []envList{{"spec.draDriver.gpus.kubeletPlugin.env", gpusEnv}}
if computeDomainsEnabled {
lists = append(lists, envList{"spec.draDriver.computeDomains.kubeletPlugin.env", computeDomainsEnv})
}

for _, l := range lists {
for _, e := range l.env {
if e.Name == healthcheckPortEnvName {
return fmt.Errorf("%s must not set %s; it is managed by the operator "+
"(use the healthcheck.port field for the health service port)", l.field, healthcheckPortEnvName)
}
}
}
return nil
}

type stateDRADriver struct {
stateSkel
}
Expand Down Expand Up @@ -118,18 +164,29 @@ func (s *stateDRADriver) getManifestObjects(ctx context.Context, cr *nvidiav1alp
return nil, fmt.Errorf("failed to get OpenShift version: %w", err)
}

gpusHealthcheckPort := resolveHealthcheckPort(
cr.Spec.DRADriver.GPUs.KubeletPlugin.Healthcheck, defaultGPUsHealthcheckPort)
computeDomainsHealthcheckPort := resolveHealthcheckPort(
cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Healthcheck, defaultComputeDomainsHealthcheckPort)
if err := validateHealthcheckPorts(cr.Spec.DRADriver.IsComputeDomainsEnabled(),
Comment thread
jramkissoon marked this conversation as resolved.
Resolved
gpusHealthcheckPort, computeDomainsHealthcheckPort); err != nil {
return nil, fmt.Errorf("invalid DRA driver healthcheck configuration: %w", err)
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
Resolved
}
if err := validateKubeletPluginEnv(cr.Spec.DRADriver.IsComputeDomainsEnabled(),
cr.Spec.DRADriver.GPUs.KubeletPlugin.Env, cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Env); err != nil {
return nil, fmt.Errorf("invalid DRA driver kubelet-plugin env configuration: %w", err)
}

renderData := &draDriverRenderData{
DRADriver: draDriverSpec,
HostPaths: &hostPaths,
Daemonsets: &daemonsets,
Namespace: s.namespace,
OpenshiftVersion: openshiftVersion,
DeviceClassAPIVersion: apiVersion,
FeatureGates: cr.Spec.DRADriver.FeatureGates,
GPUsHealthcheckPort: resolveHealthcheckPort(
cr.Spec.DRADriver.GPUs.KubeletPlugin.Healthcheck, defaultGPUsHealthcheckPort),
ComputeDomainsHealthcheckPort: resolveHealthcheckPort(
cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Healthcheck, defaultComputeDomainsHealthcheckPort),
DRADriver: draDriverSpec,
HostPaths: &hostPaths,
Daemonsets: &daemonsets,
Namespace: s.namespace,
OpenshiftVersion: openshiftVersion,
DeviceClassAPIVersion: apiVersion,
FeatureGates: cr.Spec.DRADriver.FeatureGates,
GPUsHealthcheckPort: gpusHealthcheckPort,
ComputeDomainsHealthcheckPort: computeDomainsHealthcheckPort,
}

return s.renderObjects(ctx, renderData)
Expand Down
126 changes: 126 additions & 0 deletions internal/state/dra_driver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,132 @@ func TestDRADriverHealthcheckDisabled(t *testing.T) {
assert.Nil(t, gpus.LivenessProbe)
}

func TestDRADriverHealthcheckPortCollision(t *testing.T) {
for name, tc := range map[string]struct {
computeDomainsEnabled bool
gpus *nvidiav1alpha1.DRADriverHealthcheckSpec
computeDomains *nvidiav1alpha1.DRADriverHealthcheckSpec
wantErr string
}{
"same explicit port": {
computeDomainsEnabled: true,
gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))},
computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))},
wantErr: "both resolve to 52000",
},
// Only one field is set, yet it collides with the other container's default;
// this is why the check runs on resolved ports rather than the raw spec.
"gpus set to the computeDomains default": {
Comment thread
jramkissoon marked this conversation as resolved.
Resolved
computeDomainsEnabled: true,
gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(51515))},
wantErr: "both resolve to 51515",
},
"computeDomains set to the gpus default": {
computeDomainsEnabled: true,
computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(51516))},
wantErr: "both resolve to 51516",
},
"different ports": {
computeDomainsEnabled: true,
gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))},
computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52001))},
},
"defaults do not collide": {
computeDomainsEnabled: true,
},
"computeDomains disabled ignores collision": {
gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))},
computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(52000))},
},
"both health services disabled": {
computeDomainsEnabled: true,
gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Enabled: new(false)},
computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Enabled: new(false)},
},
// The enabled side deliberately uses the disabled side's default port: it must
// pass because a disabled service never binds.
"one health service disabled": {
computeDomainsEnabled: true,
gpus: &nvidiav1alpha1.DRADriverHealthcheckSpec{Enabled: new(false)},
computeDomains: &nvidiav1alpha1.DRADriverHealthcheckSpec{Port: new(int32(51516))},
},
} {
t.Run(name, func(t *testing.T) {
s := newTestDRAState(t)
cr := sampleGPUCluster()
cr.Spec.DRADriver.ComputeDomains.Enabled = new(tc.computeDomainsEnabled)
cr.Spec.DRADriver.GPUs.KubeletPlugin.Healthcheck = tc.gpus
cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Healthcheck = tc.computeDomains

objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog())
if tc.wantErr != "" {
require.ErrorContains(t, err, tc.wantErr)
assert.ErrorContains(t, err, "spec.draDriver.gpus.kubeletPlugin.healthcheck.port")
assert.ErrorContains(t, err, "spec.draDriver.computeDomains.kubeletPlugin.healthcheck.port")
assert.Nil(t, objs)
return
}
require.NoError(t, err)
assert.NotEmpty(t, objs)
})
}
}

func TestDRADriverKubeletPluginReservedEnv(t *testing.T) {
for name, tc := range map[string]struct {
computeDomainsEnabled bool
gpusEnv []nvidiav1.EnvVar
computeDomainsEnv []nvidiav1.EnvVar
wantErr []string
}{
"gpus overrides HEALTHCHECK_PORT": {
gpusEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51515"}},
wantErr: []string{"spec.draDriver.gpus.kubeletPlugin.env", "HEALTHCHECK_PORT"},
},
// The typed ports stay at their distinct defaults, so validateHealthcheckPorts
// passes, yet the env entry moves gpus onto the computeDomains port.
"gpus overrides HEALTHCHECK_PORT onto the computeDomains port": {
computeDomainsEnabled: true,
gpusEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51515"}},
wantErr: []string{"spec.draDriver.gpus.kubeletPlugin.env", "HEALTHCHECK_PORT"},
},
"computeDomains overrides HEALTHCHECK_PORT": {
computeDomainsEnabled: true,
computeDomainsEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51516"}},
wantErr: []string{"spec.draDriver.computeDomains.kubeletPlugin.env", "HEALTHCHECK_PORT"},
},
// The second container is not rendered, so its env is never applied.
"computeDomains env ignored when disabled": {
computeDomainsEnv: []nvidiav1.EnvVar{{Name: "HEALTHCHECK_PORT", Value: "51516"}},
},
"unrelated env is allowed": {
computeDomainsEnabled: true,
gpusEnv: []nvidiav1.EnvVar{{Name: "LOG_VERBOSITY", Value: "6"}},
computeDomainsEnv: []nvidiav1.EnvVar{{Name: "ALT_PROC_DEVICES_PATH", Value: "/host/proc-devices"}},
},
} {
t.Run(name, func(t *testing.T) {
s := newTestDRAState(t)
cr := sampleGPUCluster()
cr.Spec.DRADriver.ComputeDomains.Enabled = new(tc.computeDomainsEnabled)
cr.Spec.DRADriver.GPUs.KubeletPlugin.Env = tc.gpusEnv
cr.Spec.DRADriver.ComputeDomains.KubeletPlugin.Env = tc.computeDomainsEnv

objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog())
if len(tc.wantErr) > 0 {
require.Error(t, err)
for _, want := range tc.wantErr {
assert.ErrorContains(t, err, want)
}
assert.Nil(t, objs)
return
}
require.NoError(t, err)
assert.NotEmpty(t, objs)
})
}
}

func TestDRADriverRenderDRAUnsupported(t *testing.T) {
s := newTestDRAState(t)

Expand Down