diff --git a/controllers/daemonset_pod_security_context_test.go b/controllers/daemonset_pod_security_context_test.go new file mode 100644 index 0000000000..009a2bb016 --- /dev/null +++ b/controllers/daemonset_pod_security_context_test.go @@ -0,0 +1,176 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package controllers + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + + gpuv1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1" +) + +// namedContainerSecurity is one container securityContext, copied so later mutation +// of the DaemonSet does not change the snapshot. +type namedContainerSecurity struct { + name string + sc *corev1.SecurityContext +} + +func snapshotContainerSecurity(spec corev1.PodSpec) []namedContainerSecurity { + out := make([]namedContainerSecurity, 0, len(spec.InitContainers)+len(spec.Containers)) + for i := range spec.InitContainers { + c := spec.InitContainers[i] + out = append(out, namedContainerSecurity{name: "init/" + c.Name, sc: c.SecurityContext.DeepCopy()}) + } + for i := range spec.Containers { + c := spec.Containers[i] + out = append(out, namedContainerSecurity{name: c.Name, sc: c.SecurityContext.DeepCopy()}) + } + return out +} + +func loadDaemonSetAsset(t *testing.T, rel string) *appsv1.DaemonSet { + t.Helper() + buf, err := os.ReadFile(filepath.Join(cfg.root, rel)) + require.NoError(t, err) + ds := &appsv1.DaemonSet{} + ser := json.NewSerializerWithOptions(json.DefaultMetaFactory, scheme.Scheme, scheme.Scheme, + json.SerializerOptions{Yaml: true, Pretty: false, Strict: false}) + _, _, err = ser.Decode(buf, nil, ds) + require.NoError(t, err) + return ds +} + +// TestClusterPolicyDaemonsetsPodSecurityContext is the control for the GPUCluster renders. +// ClusterPolicy copies daemonsets.podSecurityContext onto the pod and still forces the +// validator containers to run as root, leaving privileged and container SELinux options in place. +func TestClusterPolicyDaemonsetsPodSecurityContext(t *testing.T) { + t.Setenv("DCGM_IMAGE", "example.com/dcgm:test") + t.Setenv("DCGM_EXPORTER_IMAGE", "example.com/dcgm-exporter:test") + t.Setenv("VALIDATOR_IMAGE", "example.com/gpu-operator:test") + + podSecurityContext := &corev1.PodSecurityContext{ + SELinuxOptions: &corev1.SELinuxOptions{ + Type: "spc_t", + Level: "s0", + }, + RunAsUser: new(int64(1000)), + } + + cases := []struct { + name string + asset string + runAsRoot []string + }{ + { + name: "dcgm", + asset: "assets/state-dcgm/0400_dcgm.yml", + runAsRoot: []string{"init/toolkit-validation"}, + }, + { + name: "dcgm-exporter", + asset: "assets/state-dcgm-exporter/0800_daemonset.yaml", + runAsRoot: []string{"init/toolkit-validation"}, + }, + { + name: "validator", + asset: "assets/state-operator-validation/0500_daemonset.yaml", + runAsRoot: []string{ + "init/driver-validation", + "init/toolkit-validation", + "init/cuda-validation", + "init/plugin-validation", + "nvidia-operator-validator", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ds := loadDaemonSetAsset(t, tc.asset) + before := snapshotContainerSecurity(ds.Spec.Template.Spec) + + n := ClusterPolicyController{ + singleton: &gpuv1.ClusterPolicy{ + Spec: gpuv1.ClusterPolicySpec{ + Daemonsets: gpuv1.DaemonsetsSpec{PodSecurityContext: podSecurityContext}, + }, + }, + logger: ctrl.Log.WithName("test"), + runtime: gpuv1.Containerd, + } + require.NoError(t, preProcessDaemonSet(ds, n)) + + require.Equal(t, podSecurityContext, ds.Spec.Template.Spec.SecurityContext) + + after := snapshotContainerSecurity(ds.Spec.Template.Spec) + require.Equal(t, len(before), len(after)) + runAsRoot := map[string]bool{} + for _, name := range tc.runAsRoot { + runAsRoot[name] = true + } + for i := range before { + require.Equal(t, before[i].name, after[i].name) + require.Equal(t, privileged(before[i].sc), privileged(after[i].sc), before[i].name) + require.Equal(t, seLinux(before[i].sc), seLinux(after[i].sc), before[i].name) + if runAsRoot[before[i].name] { + require.NotNil(t, after[i].sc) + require.NotNil(t, after[i].sc.RunAsUser) + require.Equal(t, int64(0), *after[i].sc.RunAsUser) + require.NotNil(t, before[i].sc) + require.NotNil(t, before[i].sc.Privileged) + require.True(t, *before[i].sc.Privileged) + } else if before[i].sc != nil { + require.Equal(t, before[i].sc.RunAsUser, after[i].sc.RunAsUser) + } + } + }) + } + + t.Run("unset", func(t *testing.T) { + ds := loadDaemonSetAsset(t, "assets/state-dcgm/0400_dcgm.yml") + n := ClusterPolicyController{ + singleton: &gpuv1.ClusterPolicy{}, + logger: ctrl.Log.WithName("test"), + runtime: gpuv1.Containerd, + } + require.NoError(t, preProcessDaemonSet(ds, n)) + require.Nil(t, ds.Spec.Template.Spec.SecurityContext) + }) +} + +func privileged(sc *corev1.SecurityContext) *bool { + if sc == nil { + return nil + } + return sc.Privileged +} + +func seLinux(sc *corev1.SecurityContext) *corev1.SELinuxOptions { + if sc == nil { + return nil + } + return sc.SELinuxOptions +} diff --git a/internal/state/gpucluster_render_test.go b/internal/state/gpucluster_render_test.go index 6b130b4cc2..5fe6a7d5d1 100644 --- a/internal/state/gpucluster_render_test.go +++ b/internal/state/gpucluster_render_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -119,6 +120,110 @@ func TestGPUClusterDaemonsetsPriorityClass(t *testing.T) { } } +// containerSecurity is the securityContext of one container in a rendered pod. +type containerSecurity struct { + name string + sc *corev1.SecurityContext +} + +func containerSecurityContexts(spec corev1.PodSpec) []containerSecurity { + out := make([]containerSecurity, 0, len(spec.InitContainers)+len(spec.Containers)) + for i := range spec.InitContainers { + c := spec.InitContainers[i] + out = append(out, containerSecurity{name: "init/" + c.Name, sc: c.SecurityContext}) + } + for i := range spec.Containers { + c := spec.Containers[i] + out = append(out, containerSecurity{name: c.Name, sc: c.SecurityContext}) + } + return out +} + +// TestGPUClusterDaemonsetsPodSecurityContext renders each GPUCluster DaemonSet with +// spec.daemonsets.podSecurityContext. ClusterPolicy already copies that field onto operand +// pods (see TestClusterPolicyDaemonsetsPodSecurityContext); these renders must do the same +// while leaving the container securityContext, including privileged and runAsUser, unchanged. +func TestGPUClusterDaemonsetsPodSecurityContext(t *testing.T) { + podSecurityContext := &corev1.PodSecurityContext{ + SELinuxOptions: &corev1.SELinuxOptions{ + Type: "spc_t", + Level: "s0", + }, + // Distinct from the container runAsUser the templates pin, so a pod-level UID + // cannot replace the root user those containers require. + RunAsUser: new(int64(1000)), + } + + type renderer interface { + getManifestObjects(context.Context, *nvidiav1alpha1.GPUCluster, InfoCatalog) ([]*unstructured.Unstructured, error) + } + states := map[string]renderer{ + "dcgm": newTestDCGMState(t), + "dcgm-exporter": newTestDCGMExporterState(t, false), + "dra-validator": newTestDRAValidationState(t), + "dra-driver": newTestDRAState(t), + } + + // Container securityContext values the operand needs regardless of the pod context. + type wantContainer struct { + privileged bool + runAsUser *int64 + } + required := map[string]map[string]wantContainer{ + "dcgm": { + "nvidia-dcgm-ctr": {privileged: true}, + }, + "dcgm-exporter": { + "nvidia-dcgm-exporter-dra": {privileged: true}, + }, + "dra-driver": { + "init/driver-validation": {privileged: true, runAsUser: new(int64(0))}, + "gpus": {privileged: true}, + "compute-domains": {privileged: true}, + }, + } + + render := func(t *testing.T, s renderer, psc *corev1.PodSecurityContext) *appsv1.DaemonSet { + t.Helper() + cr := sampleGPUCluster() + cr.Spec.DCGM = &nvidiav1.DCGMSpec{Enabled: new(true)} + cr.Spec.DCGMExporter = &nvidiav1.DCGMExporterSpec{Enabled: new(true)} + cr.Spec.DRADriver.ComputeDomains.Enabled = new(true) + cr.Spec.Daemonsets.PodSecurityContext = psc + + objs, err := s.getManifestObjects(context.Background(), cr, draSupportedCatalog()) + require.NoError(t, err) + return findDaemonSet(t, objs) + } + + for stateName, s := range states { + t.Run(stateName, func(t *testing.T) { + base := render(t, s, nil) + withContext := render(t, s, podSecurityContext) + + require.Nil(t, base.Spec.Template.Spec.SecurityContext) + require.Equal(t, podSecurityContext, withContext.Spec.Template.Spec.SecurityContext) + require.Equal(t, containerSecurityContexts(base.Spec.Template.Spec), containerSecurityContexts(withContext.Spec.Template.Spec)) + + got := map[string]*corev1.SecurityContext{} + for _, c := range containerSecurityContexts(withContext.Spec.Template.Spec) { + got[c.name] = c.sc + } + for name, want := range required[stateName] { + sc, ok := got[name] + require.Truef(t, ok, "container %s not rendered", name) + require.NotNil(t, sc) + require.NotNil(t, sc.Privileged) + require.Equal(t, want.privileged, *sc.Privileged) + if want.runAsUser != nil { + require.NotNil(t, sc.RunAsUser) + require.Equal(t, *want.runAsUser, *sc.RunAsUser) + } + } + }) + } +} + // TestGPUClusterRenderGolden renders each GPUCluster operand's manifests end to end and // byte-compares the full YAML stream against fixtures in testdata/golden, so unintended // template changes surface as diffs (mirroring the NVIDIADriver renderer tests). diff --git a/manifests/state-dcgm-exporter/0700_daemonset.yaml b/manifests/state-dcgm-exporter/0700_daemonset.yaml index 32be819354..4e007cc839 100644 --- a/manifests/state-dcgm-exporter/0700_daemonset.yaml +++ b/manifests/state-dcgm-exporter/0700_daemonset.yaml @@ -69,6 +69,10 @@ spec: - {{ . | toJson }} {{- end }} {{- end }} + {{- if .Daemonsets.PodSecurityContext }} + securityContext: + {{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }} + {{- end }} resourceClaims: - name: admin-gpus resourceClaimTemplateName: nvidia-dcgm-exporter-admin diff --git a/manifests/state-dcgm/0500_daemonset.yaml b/manifests/state-dcgm/0500_daemonset.yaml index d06948103f..196059662a 100644 --- a/manifests/state-dcgm/0500_daemonset.yaml +++ b/manifests/state-dcgm/0500_daemonset.yaml @@ -62,6 +62,10 @@ spec: - {{ . | toJson }} {{- end }} {{- end }} + {{- if .Daemonsets.PodSecurityContext }} + securityContext: + {{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }} + {{- end }} resourceClaims: - name: admin-gpus resourceClaimTemplateName: nvidia-dcgm-admin diff --git a/manifests/state-dra-driver/0500_daemonset.yaml b/manifests/state-dra-driver/0500_daemonset.yaml index 673ff70ed2..31fe40fa16 100644 --- a/manifests/state-dra-driver/0500_daemonset.yaml +++ b/manifests/state-dra-driver/0500_daemonset.yaml @@ -82,6 +82,10 @@ spec: - {{ . | toJson }} {{- end }} {{- end }} + {{- if .Daemonsets.PodSecurityContext }} + securityContext: + {{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }} + {{- end }} initContainers: - name: driver-validation image: {{ .DRADriver.InitImagePath }} diff --git a/manifests/state-dra-validation/0500_daemonset.yaml b/manifests/state-dra-validation/0500_daemonset.yaml index 8775ba3be4..4cdec318cf 100644 --- a/manifests/state-dra-validation/0500_daemonset.yaml +++ b/manifests/state-dra-validation/0500_daemonset.yaml @@ -86,6 +86,10 @@ spec: - {{ . | toJson }} {{- end }} {{- end }} + {{- if .Daemonsets.PodSecurityContext }} + securityContext: + {{- .Daemonsets.PodSecurityContext | yaml | nindent 8 }} + {{- end }} resourceClaims: - name: validation-gpu resourceClaimTemplateName: nvidia-dra-validator-gpu