Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 176 additions & 0 deletions controllers/daemonset_pod_security_context_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/

package controllers

import (
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"

gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
)

// namedContainerSecurity is one container securityContext, copied so later mutation
// of the DaemonSet does not change the snapshot.
type namedContainerSecurity struct {
name string
sc *corev1.SecurityContext
}

func snapshotContainerSecurity(spec corev1.PodSpec) []namedContainerSecurity {
out := make([]namedContainerSecurity, 0, len(spec.InitContainers)+len(spec.Containers))
for i := range spec.InitContainers {
c := spec.InitContainers[i]
out = append(out, namedContainerSecurity{name: "init/" + c.Name, sc: c.SecurityContext.DeepCopy()})
}
for i := range spec.Containers {
c := spec.Containers[i]
out = append(out, namedContainerSecurity{name: c.Name, sc: c.SecurityContext.DeepCopy()})
}
return out
}

func loadDaemonSetAsset(t *testing.T, rel string) *appsv1.DaemonSet {
t.Helper()
buf, err := os.ReadFile(filepath.Join(cfg.root, rel))
require.NoError(t, err)
ds := &appsv1.DaemonSet{}
ser := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme,
json.SerializerOptions{Yaml: true, Pretty: false, Strict: false})
_, _, err = ser.Decode(buf, nil, ds)
require.NoError(t, err)
return ds
}

// TestClusterPolicyDaemonsetsPodSecurityContext is the control for the GPUCluster renders.
// ClusterPolicy copies daemonsets.podSecurityContext onto the pod and still forces the
// validator containers to run as root, leaving privileged and container SELinux options in place.
func TestClusterPolicyDaemonsetsPodSecurityContext(t *testing.T) {
t.Setenv("DCGM_IMAGE", "example.com/dcgm:test")
t.Setenv("DCGM_EXPORTER_IMAGE", "example.com/dcgm-exporter:test")
t.Setenv("VALIDATOR_IMAGE", "example.com/gpu-operator:test")

podSecurityContext := &corev1.PodSecurityContext{
SELinuxOptions: &corev1.SELinuxOptions{
Type: "spc_t",
Level: "s0",
},
RunAsUser: new(int64(1000)),
}

cases := []struct {
name string
asset string
runAsRoot []string
}{
{
name: "dcgm",
asset: "assets/state-dcgm/0400_dcgm.yml",
runAsRoot: []string{"init/toolkit-validation"},
},
{
name: "dcgm-exporter",
asset: "assets/state-dcgm-exporter/0800_daemonset.yaml",
runAsRoot: []string{"init/toolkit-validation"},
},
{
name: "validator",
asset: "assets/state-operator-validation/0500_daemonset.yaml",
runAsRoot: []string{
"init/driver-validation",
"init/toolkit-validation",
"init/cuda-validation",
"init/plugin-validation",
"nvidia-operator-validator",
},
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ds := loadDaemonSetAsset(t, tc.asset)
before := snapshotContainerSecurity(ds.Spec.Template.Spec)

n := ClusterPolicyController{
singleton: &gpuv1.ClusterPolicy{
Spec: gpuv1.ClusterPolicySpec{
Daemonsets: gpuv1.DaemonsetsSpec{PodSecurityContext: podSecurityContext},
},
},
logger: ctrl.Log.WithName("test"),
runtime: gpuv1.Containerd,
}
require.NoError(t, preProcessDaemonSet(ds, n))

require.Equal(t, podSecurityContext, ds.Spec.Template.Spec.SecurityContext)

after := snapshotContainerSecurity(ds.Spec.Template.Spec)
require.Equal(t, len(before), len(after))
runAsRoot := map[string]bool{}
for _, name := range tc.runAsRoot {
runAsRoot[name] = true
}
for i := range before {
require.Equal(t, before[i].name, after[i].name)
require.Equal(t, privileged(before[i].sc), privileged(after[i].sc), before[i].name)
require.Equal(t, seLinux(before[i].sc), seLinux(after[i].sc), before[i].name)
if runAsRoot[before[i].name] {
require.NotNil(t, after[i].sc)
require.NotNil(t, after[i].sc.RunAsUser)
require.Equal(t, int64(0), *after[i].sc.RunAsUser)
require.NotNil(t, before[i].sc)
require.NotNil(t, before[i].sc.Privileged)
require.True(t, *before[i].sc.Privileged)
} else if before[i].sc != nil {
require.Equal(t, before[i].sc.RunAsUser, after[i].sc.RunAsUser)
}
}
})
}

t.Run("unset", func(t *testing.T) {
ds := loadDaemonSetAsset(t, "assets/state-dcgm/0400_dcgm.yml")
n := ClusterPolicyController{
singleton: &gpuv1.ClusterPolicy{},
logger: ctrl.Log.WithName("test"),
runtime: gpuv1.Containerd,
}
require.NoError(t, preProcessDaemonSet(ds, n))
require.Nil(t, ds.Spec.Template.Spec.SecurityContext)
})
}

func privileged(sc *corev1.SecurityContext) *bool {
if sc == nil {
return nil
}
return sc.Privileged
}

func seLinux(sc *corev1.SecurityContext) *corev1.SELinuxOptions {
if sc == nil {
return nil
}
return sc.SELinuxOptions
}
105 changes: 105 additions & 0 deletions internal/state/gpucluster_render_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"testing"

"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand Down Expand Up @@ -119,6 +120,110 @@ func TestGPUClusterDaemonsetsPriorityClass(t *testing.T) {
}
}

// containerSecurity is the securityContext of one container in a rendered pod.
type containerSecurity struct {
name string
sc *corev1.SecurityContext
}

func containerSecurityContexts(spec corev1.PodSpec) []containerSecurity {
out := make([]containerSecurity, 0, len(spec.InitContainers)+len(spec.Containers))
for i := range spec.InitContainers {
c := spec.InitContainers[i]
out = append(out, containerSecurity{name: "init/" + c.Name, sc: c.SecurityContext})
}
for i := range spec.Containers {
c := spec.Containers[i]
out = append(out, containerSecurity{name: c.Name, sc: c.SecurityContext})
}
return out
}

// TestGPUClusterDaemonsetsPodSecurityContext renders each GPUCluster DaemonSet with
// spec.daemonsets.podSecurityContext. ClusterPolicy already copies that field onto operand
// pods (see TestClusterPolicyDaemonsetsPodSecurityContext); these renders must do the same
// while leaving the container securityContext, including privileged and runAsUser, unchanged.
func TestGPUClusterDaemonsetsPodSecurityContext(t *testing.T) {
podSecurityContext := &corev1.PodSecurityContext{
SELinuxOptions: &corev1.SELinuxOptions{
Type: "spc_t",
Level: "s0",
},
// Distinct from the container runAsUser the templates pin, so a pod-level UID
// cannot replace the root user those containers require.
RunAsUser: new(int64(1000)),
}

type renderer interface {
getManifestObjects(context.Context, *nvidiav1alpha1.GPUCluster, InfoCatalog) ([]*unstructured.Unstructured, error)
}
states := map[string]renderer{
"dcgm": newTestDCGMState(t),
"dcgm-exporter": newTestDCGMExporterState(t, false),
"dra-validator": newTestDRAValidationState(t),
"dra-driver": newTestDRAState(t),
}

// Container securityContext values the operand needs regardless of the pod context.
type wantContainer struct {
privileged bool
runAsUser *int64
}
required := map[string]map[string]wantContainer{
"dcgm": {
"nvidia-dcgm-ctr": {privileged: true},
},
"dcgm-exporter": {
"nvidia-dcgm-exporter-dra": {privileged: true},
},
"dra-driver": {
"init/driver-validation": {privileged: true, runAsUser: new(int64(0))},
"gpus": {privileged: true},
"compute-domains": {privileged: true},
},
}

render := func(t *testing.T, s renderer, psc *corev1.PodSecurityContext) *appsv1.DaemonSet {
t.Helper()
cr := sampleGPUCluster()
cr.Spec.DCGM = &nvidiav1.DCGMSpec{Enabled: new(true)}
cr.Spec.DCGMExporter = &nvidiav1.DCGMExporterSpec{Enabled: new(true)}
cr.Spec.DRADriver.ComputeDomains.Enabled = new(true)
cr.Spec.Daemonsets.PodSecurityContext = psc

objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog())
require.NoError(t, err)
return findDaemonSet(t, objs)
}

for stateName, s := range states {
t.Run(stateName, func(t *testing.T) {
base := render(t, s, nil)
withContext := render(t, s, podSecurityContext)

require.Nil(t, base.Spec.Template.Spec.SecurityContext)
require.Equal(t, podSecurityContext, withContext.Spec.Template.Spec.SecurityContext)
require.Equal(t, containerSecurityContexts(base.Spec.Template.Spec), containerSecurityContexts(withContext.Spec.Template.Spec))

got := map[string]*corev1.SecurityContext{}
for _, c := range containerSecurityContexts(withContext.Spec.Template.Spec) {
got[c.name] = c.sc
}
for name, want := range required[stateName] {
sc, ok := got[name]
require.Truef(t, ok, "container %s not rendered", name)
require.NotNil(t, sc)
require.NotNil(t, sc.Privileged)
require.Equal(t, want.privileged, *sc.Privileged)
if want.runAsUser != nil {
require.NotNil(t, sc.RunAsUser)
require.Equal(t, *want.runAsUser, *sc.RunAsUser)
}
}
})
}
}

// TestGPUClusterRenderGolden renders each GPUCluster operand's manifests end to end and
// byte-compares the full YAML stream against fixtures in testdata/golden, so unintended
// template changes surface as diffs (mirroring the NVIDIADriver renderer tests).
Expand Down
4 changes: 4 additions & 0 deletions manifests/state-dcgm-exporter/0700_daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ spec:
- {{ . | toJson }}
{{- end }}
{{- end }}
{{- if .Daemonsets.PodSecurityContext }}
securityContext:
{{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }}
{{- end }}
resourceClaims:
- name: admin-gpus
resourceClaimTemplateName: nvidia-dcgm-exporter-admin
Expand Down
4 changes: 4 additions & 0 deletions manifests/state-dcgm/0500_daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ spec:
- {{ . | toJson }}
{{- end }}
{{- end }}
{{- if .Daemonsets.PodSecurityContext }}
securityContext:
{{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }}
{{- end }}
resourceClaims:
- name: admin-gpus
resourceClaimTemplateName: nvidia-dcgm-admin
Expand Down
4 changes: 4 additions & 0 deletions manifests/state-dra-driver/0500_daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@ spec:
- {{ . | toJson }}
{{- end }}
{{- end }}
{{- if .Daemonsets.PodSecurityContext }}
securityContext:
{{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }}
{{- end }}
Comment on lines +85 to +88

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Major · Stability & Availability · Heavy lift

Supported by static analysis

Script executed:

git diff b7d2c34dc112d879b5fee84e694d84a601c02041 76ae44278b4baecb6605bd26c6e7bc0abf47b85b -- manifests/state-dra-driver/0500_daemonset.yaml
cat -n manifests/state-dra-driver/0500_daemonset.yaml
rg -n -F -e 'compute-domains' -e 'mount --bind' -e 'podSecurityContext' -- manifests/state-dra-driver internal | sed -n '1,220p'

Repository: NVIDIA/gpu-operator

Length of output: 19513


Script executed:

rg -n -F -e 'DRA_DRIVER_IMAGE' -e 'MASK_NVIDIA_DRIVER_PARAMS' -e 'PodSecurityContext' -- internal/state api/nvidia/v1alpha1 deployments/gpu-operator | sed -n '1,200p'
cat -n internal/state/gpucluster_render_test.go | sed -n '135,220p'
rg -n -F -e 'dra-driver-nvidia-gpu' -e 'DRA_DRIVER_IMAGE' -- deployments/gpu-operator/values.yaml bundle/manifests/gpu-operator-certified.clusterserviceversion.yaml .github | sed -n '1,100p'

Repository: NVIDIA/gpu-operator

Length of output: 5891


Web query:

Kubernetes privileged container non root UID all capabilities CAP_DAC_OVERRIDE CAP_SYS_ADMIN Kubernetes security context

Result:

Sources consulted:
- Linux kernel security constraints for Pods and containers | Kubernetes — https://kubernetes.io/docs/concepts/security/linux-kernel-security-constraints/
- Pod securityContext Internals for the CKS Exam: Linux Capability Sets, Runtime Default Capabilities, Why Added Capabilities Vanish for Non-Root Users, allowPrivilegeEscalation & no_new_privs, privileged Containers, fsGroup, fsGroupChangePolicy, supplementalGroupsPolicy, Sysctls, procMoun…[truncated]
- Configure a Security Context for a Pod or Container | Kubernetes — https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
- lib/IO/K8s/Api/Core/V1/SecurityContext.pm - metacpan.org — https://web.do.metacpan.org/release/GETTY/IO-K8s-1.100/source/lib/IO/K8s/Api/Core/V1/SecurityContext.pm
- SecurityContext class - io.k8s.api.core.v1 library - Dart API — https://pub.dev/documentation/kubernetes/latest/io.k8s.api.core.v1/SecurityContext-class.html

Web query:

github kubernetes-sigs dra-driver-nvidia-gpu v0.5.1 gpu-kubelet-plugin non root kubelet registration socket permissions

Result:

Sources consulted:
- Releases · kubernetes-sigs/dra-driver-nvidia-gpu · GitHub — https://github.com/kubernetes-sigs/dra-driver-nvidia-gpu/releases
- kubernetes-sigs/dra-driver-nvidia-gpu v0.5.1 on GitHub — https://newreleases.io/project/github/kubernetes-sigs/dra-driver-nvidia-gpu/release/v0.5.1
- cmd/gpu-kubelet-plugin/driver.go — https://github.com/NVIDIA/k8s-dra-driver-gpu/blob/8effb048f94b3f18338a6f93527352cda82ee385/cmd/gpu-kubelet-plugin/driver.go
- cmd/gpu-kubelet-plugin/driver.go — https://github.com/NVIDIA/k8s-dra-driver-gpu/blob/main/cmd/gpu-kubelet-plugin/driver.go
- cmd/gpu-kubelet-plugin/main.go — https://github.com/NVIDIA/k8s-dra-driver-gpu/blob/main/cmd/gpu-kubelet-plugin/main.go

Web query:

containerd privileged non root exec ambient capabilities CAP_SYS_ADMIN bash kubernetes runc UID 1000 privileged mount

Result:

Sources consulted:
- Chapter 7: Security Boundaries - Containerd From The Bottom Up — https://thecontainerdbook.com/chapters/part-2/07-security
- Kubernetes Linux Capabilities [Add, Drop and Examples] — https://www.golinuxcloud.com/kubernetes-securitycontext-capabilities/
- Docker and Container Capabilities | Luke Hebert — https://lukehebert.com/post/docker-and-container-capabilities/
- libcontainer/SPEC.md at main · opencontainers/runc — https://github.com/opencontainers/runc/blob/main/libcontainer/SPEC.md
- Capabilities and Privilege Management | open-beagle/runc | DeepWiki — https://deepwiki.com/open-beagle/runc/6.3-capabilities-and-privilege-management

Web query:

site:github.com/kubernetes-sigs/dra-driver-nvidia-gpu "runAsUser" "privileged" "non-root" kubelet plugin

Result:

Sources consulted:
- site/content/docs/reference/helm-values.md — https://github.com/kubernetes-sigs/dra-driver-nvidia-gpu/blob/main/site/content/docs/reference/helm-values.md
- deployments/helm/dra-driver-nvidia-gpu/values.yaml — https://github.com/kubernetes-sigs/dra-driver-nvidia-gpu/blob/main/deployments/helm/dra-driver-nvidia-gpu/values.yaml
- deployments/helm/dra-driver-nvidia-gpu/values.yaml — https://github.com/kubernetes-sigs/dra-driver-nvidia-gpu/blob/v0.4.1/deployments/helm/dra-driver-nvidia-gpu/values.yaml
- 616fe03 Add SCC to service accounts on OpenShift — https://github.com/kubernetes-sigs/dra-driver-nvidia-gpu/commit/616fe03385145ca5dfeb8db5482d32e19b0e0b4e
- Add MPS config option to configure nvidia-cuda-mps-control to run in multi user mode — GitHub pull request 1153 in kubernetes-sigs/dra-driver-nvidia-gpu (link omitted to avoid creating a cross-reference)

Script executed:

cat -n deployments/gpu-operator/values.yaml | sed -n '605,640p'
cat -n internal/state/dra_driver.go | sed -n '130,195p'
rg -n -F -e 'KubeletPlugin' -e 'DaemonsetsSpec' -- api/nvidia/v1 api/nvidia/v1alpha1 | sed -n '1,140p'
rg -n -F -- 'MASK_NVIDIA_DRIVER_PARAMS' . --glob '!vendor/**' || test "$?" -eq 1

Repository: NVIDIA/gpu-operator

Length of output: 6531


Web query:

containerd CRI privileged runAsUser non root ambient capabilities 2.3 exec all capabilities Kubernetes

Result:

Sources consulted:
- Kubernetes should configure the ambient capability set — GitHub issue 56374 in kubernetes/kubernetes (link omitted to avoid creating a cross-reference)
- How to Restrict Container Capabilities Using securityContext in Kubernetes — https://oneuptime.com/blog/post/2026-02-09-restrict-container-capabilities-security-context/view
- Chapter 7: Security Boundaries - Containerd From The Bottom Up — https://thecontainerdbook.com/chapters/part-2/07-security
- Capabilities - Building a Container from Scratch in Rust — https://brianshih1.github.io/mini-container/capabilities.html
- containerd should set ambient capabilities — GitHub issue 5644 in containerd/containerd (link omitted to avoid creating a cross-reference)

Script executed:

python3 - <<'PY'
import urllib.request
urls = [
'https://raw.githubusercontent.com/containerd/containerd/v2.3.5/internal/cri/server/container_create.go',
'https://raw.githubusercontent.com/kubernetes-sigs/dra-driver-nvidia-gpu/v0.5.1/cmd/gpu-kubelet-plugin/driver.go',
'https://raw.githubusercontent.com/kubernetes-sigs/dra-driver-nvidia-gpu/v0.5.1/deployments/helm/dra-driver-nvidia-gpu/templates/kubeletplugin.yaml',
]
for url in urls:
    print('\n###', url)
    try:
        text = urllib.request.urlopen(url, timeout=12).read().decode()
        lines = text.splitlines()
        needles = ('WithoutAmbientCaps', 'WithAllCurrentCapabilities', 'Privileged', 'kubeletplugin.Start(', 'Mkdir', 'runAsUser', 'podSecurityContext')
        hits = [i for i, line in enumerate(lines) if any(n in line for n in needles)]
        for i in hits:
            print('\n'.join(f'{j+1}: {lines[j]}' for j in range(max(0,i-4), min(len(lines),i+7))))
            print('---')
        if not hits:
            print('No selected symbols found')
    except Exception as exc:
        print(type(exc).__name__, str(exc))
PY

Repository: NVIDIA/gpu-operator

Length of output: 8306


Script executed:

python3 - <<'PY'
import json, urllib.request
base='https://raw.githubusercontent.com/kubernetes-sigs/dra-driver-nvidia-gpu/v0.5.1/'
def read(url):
    return urllib.request.urlopen(url, timeout=15).read().decode()
try:
    tree=json.loads(read('https://api.github.com/repos/kubernetes-sigs/dra-driver-nvidia-gpu/git/trees/v0.5.1?recursive=1'))['tree']
    paths=[x['path'] for x in tree if x['type']=='blob' and ((x['path'].startswith('cmd/gpu-kubelet-plugin/') and x['path'].endswith('.go')) or x['path'].endswith('Dockerfile') or x['path']=='go.mod')]
    print('Candidate files:', ', '.join(paths))
    for path in paths:
        if path.endswith('Dockerfile') and 'gpu' not in path and path != 'Dockerfile':
            continue
        try:
            lines=read(base+path).splitlines()
        except Exception as e:
            print(path, 'read failed:', e)
            continue
        needles=('func NewDeviceState(', 'os.Mkdir', 'MkdirAll(', 'k8s.io/dynamic-resource-allocation', 'USER ', 'FROM ', 'kubeletplugin.Start(', 'NewDriver(ctx')
        hits=[i for i,l in enumerate(lines) if any(n in l for n in needles)]
        if hits:
            print('\n###', base+path)
            for i in hits:
                print('\n'.join(f'{j+1}: {lines[j]}' for j in range(max(0,i-5),min(len(lines),i+12))))
                print('---')
except Exception as e:
    print('Inspection failed:', type(e).__name__, e)
PY

Repository: NVIDIA/gpu-operator

Length of output: 30528


Script executed:

python3 - <<'PY'
import urllib.request
base='https://raw.githubusercontent.com/kubernetes-sigs/dra-driver-nvidia-gpu/v0.5.1/'
for path, ranges in {
'cmd/gpu-kubelet-plugin/main.go':[(44,80),(110,150),(280,370),(375,430)],
'deployments/container/Dockerfile':[(1,200)],
}.items():
    print('\n###',base+path)
    try:
        lines=urllib.request.urlopen(base+path,timeout=15).read().decode().splitlines()
        for a,b in ranges:
            print('\n'.join(f'{i+1}: {lines[i]}' for i in range(a-1,min(b,len(lines)))))
    except Exception as e:
        print(type(e).__name__,e)
PY
cat -n tests/holodeck.yaml | grep -n -C 2 'containerd'
cat -n manifests/state-dra-driver/0500_daemonset.yaml | sed -n '192,205p;227,245p;363,370p'

Repository: NVIDIA/gpu-operator

Length of output: 18786


Script executed:

python3 - <<'PY'
import urllib.request
url='https://raw.githubusercontent.com/kubernetes-sigs/dra-driver-nvidia-gpu/v0.5.1/cmd/gpu-kubelet-plugin/main.go'
lines=urllib.request.urlopen(url,timeout=15).read().decode().splitlines()
for a,b in [(78,88),(190,220),(245,280)]:
 print(f'\n### {url}:{a}-{b}')
 print('\n'.join(f'{i+1}: {lines[i]}' for i in range(a-1,min(b,len(lines)))))
for i,l in enumerate(lines):
 if 'func main(' in l or 'RunPlugin(' in l or 'app.Run' in l:
  print(f'\n### {url}:{max(1,i-5)}-{min(len(lines),i+12)}')
  print('\n'.join(f'{j+1}: {lines[j]}' for j in range(max(0,i-6),min(len(lines),i+12))))
PY

Repository: NVIDIA/gpu-operator

Length of output: 5688


Web query:

man7 capabilities execve nonroot ambient file capabilities effective permitted Linux calculation

Result:

Sources consulted:
- capabilities(7) - Linux manual page — https://man7.org/linux/man-pages/man7/capabilities.7.html
- Linux capabilities: what the five sets actually do — RodHat — https://rodhat.com/tips/2026-09-20-linux-capabilities-set-model/
- How to execve a process, retaining capabilities in spite of missing filesystem-based capabilities? — https://stackoverflow.com/questions/14636359/how-to-execve-a-process-retaining-capabilities-in-spite-of-missing-filesystem-b
- Beyond Container Capabilities: Understanding Linux Capability Sets | utam0k — https://www.utam0k.jp/en/blog/2025/12/14/linux-capability-sets/
- capabilities: Ambient capabilities [LWN.net] — https://lwn.net/Articles/636533/

Script executed:

python3 - <<'PY'
from urllib.request import urlopen
url='https://raw.githubusercontent.com/kubernetes-sigs/dra-driver-nvidia-gpu/v0.5.1/cmd/gpu-kubelet-plugin/main.go'
lines=urlopen(url,timeout=15).read().decode().splitlines()
for i,line in enumerate(lines):
 if 'KUBELET_PLUGINS_DIRECTORY_PATH' in line:
  print(url)
  print('\n'.join(f'{j+1}: {lines[j]}' for j in range(max(0,i-5),min(len(lines),i+3))))
PY

Repository: NVIDIA/gpu-operator

Length of output: 642


Ensure the GPU plugin can start with a non-root pod UID.

With runAsUser: 1000 on containerd 2.3.5, the plugin can fail to create its directory under a root-owned kubelet plugins directory. The gpus container then exits and can crash-loop. Privileged mode does not preserve ambient capabilities across the non-root executable.

The render test does not detect this startup failure.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this only changes anything when daemonsets.podSecurityContext is set, and that matches clusterpolicy today: applyCommonDaemonsetConfig copies the same field onto every operand daemonset, including the device plugin that also writes under /var/lib/kubelet. the init container here already pins runAsUser: 0, and i left the plugin containers as they are so the change stays a straight port of the clusterpolicy behavior. @rahulait if you'd prefer the kubelet plugin containers to pin runAsUser: 0 as well so a non-root pod uid can't reach them, i can add that here.

initContainers:
- name: driver-validation
image: {{ .DRADriver.InitImagePath }}
Expand Down
4 changes: 4 additions & 0 deletions manifests/state-dra-validation/0500_daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ spec:
- {{ . | toJson }}
{{- end }}
{{- end }}
{{- if .Daemonsets.PodSecurityContext }}
securityContext:
{{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }}
{{- end }}
resourceClaims:
- name: validation-gpu
resourceClaimTemplateName: nvidia-dra-validator-gpu
Expand Down