From 90e757a5699e917bf9aaac0b29e1b46affa5eb30 Mon Sep 17 00:00:00 2001 From: Christopher Desiniotis Date: Tue, 25 Nov 2025 21:46:05 -0800 Subject: [PATCH] Run nvidia-smi after modules are loaded in driver ds startup probe This commit eliminates the race condition where the startup probe in the driver daemonset runs after the kernel modules are built (and installed) but before the modules are loaded into the kernel. In this case, the invocation of nvidia-smi (by the startup probe) is what is actually loading the nvidia kernel module and not the modprobe we perform in our driver container scripts. As a result, the nvidia driver will be loaded with a default configuration -- none of the custom kernel module parameters provided by users (via a configmap) or set by our driver container will get applied. Signed-off-by: Christopher Desiniotis --- assets/state-driver/0500_daemonset.yaml | 2 +- internal/state/testdata/golden/driver-additional-configs.yaml | 2 +- internal/state/testdata/golden/driver-full-spec.yaml | 2 +- internal/state/testdata/golden/driver-gdrcopy-openshift.yaml | 2 +- internal/state/testdata/golden/driver-gdrcopy.yaml | 2 +- internal/state/testdata/golden/driver-gds.yaml | 2 +- internal/state/testdata/golden/driver-minimal.yaml | 2 +- .../state/testdata/golden/driver-openshift-drivertoolkit.yaml | 2 +- internal/state/testdata/golden/driver-precompiled.yaml | 2 +- internal/state/testdata/golden/driver-rdma-hostmofed.yaml | 2 +- internal/state/testdata/golden/driver-rdma.yaml | 2 +- internal/state/testdata/golden/driver-secret-env.yaml | 2 +- .../state/testdata/golden/driver-vgpu-licensing-secret.yaml | 2 +- internal/state/testdata/golden/driver-vgpu-licensing.yaml | 2 +- manifests/state-driver/0500_daemonset.yaml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/assets/state-driver/0500_daemonset.yaml b/assets/state-driver/0500_daemonset.yaml index 3b9e1437ce..6456a70547 100644 --- a/assets/state-driver/0500_daemonset.yaml +++ b/assets/state-driver/0500_daemonset.yaml @@ -137,7 +137,7 @@ spec: startupProbe: exec: command: - [sh, -c, 'nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'] + [sh, -c, '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'] initialDelaySeconds: 60 failureThreshold: 120 successThreshold: 1 diff --git a/internal/state/testdata/golden/driver-additional-configs.yaml b/internal/state/testdata/golden/driver-additional-configs.yaml index f323600e6c..9ac2e3666b 100644 --- a/internal/state/testdata/golden/driver-additional-configs.yaml +++ b/internal/state/testdata/golden/driver-additional-configs.yaml @@ -168,7 +168,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-full-spec.yaml b/internal/state/testdata/golden/driver-full-spec.yaml index 79d17caa65..6eb6bed8d3 100644 --- a/internal/state/testdata/golden/driver-full-spec.yaml +++ b/internal/state/testdata/golden/driver-full-spec.yaml @@ -182,7 +182,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml b/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml index 61e466c815..81bcfa59ab 100644 --- a/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml +++ b/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml @@ -238,7 +238,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-gdrcopy.yaml b/internal/state/testdata/golden/driver-gdrcopy.yaml index 551ea3912d..b9b8971fb8 100644 --- a/internal/state/testdata/golden/driver-gdrcopy.yaml +++ b/internal/state/testdata/golden/driver-gdrcopy.yaml @@ -168,7 +168,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-gds.yaml b/internal/state/testdata/golden/driver-gds.yaml index 841b00a38e..43b930c2a8 100644 --- a/internal/state/testdata/golden/driver-gds.yaml +++ b/internal/state/testdata/golden/driver-gds.yaml @@ -168,7 +168,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-minimal.yaml b/internal/state/testdata/golden/driver-minimal.yaml index 47417ef8e2..f2b314dca5 100644 --- a/internal/state/testdata/golden/driver-minimal.yaml +++ b/internal/state/testdata/golden/driver-minimal.yaml @@ -168,7 +168,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml b/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml index 6fa36bc940..04b00b7d20 100644 --- a/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml +++ b/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml @@ -238,7 +238,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-precompiled.yaml b/internal/state/testdata/golden/driver-precompiled.yaml index 4146524bd3..40c6bcf9aa 100644 --- a/internal/state/testdata/golden/driver-precompiled.yaml +++ b/internal/state/testdata/golden/driver-precompiled.yaml @@ -170,7 +170,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-rdma-hostmofed.yaml b/internal/state/testdata/golden/driver-rdma-hostmofed.yaml index a75afa1231..917bafec6b 100644 --- a/internal/state/testdata/golden/driver-rdma-hostmofed.yaml +++ b/internal/state/testdata/golden/driver-rdma-hostmofed.yaml @@ -172,7 +172,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-rdma.yaml b/internal/state/testdata/golden/driver-rdma.yaml index 8beb12b090..70e2b5b269 100644 --- a/internal/state/testdata/golden/driver-rdma.yaml +++ b/internal/state/testdata/golden/driver-rdma.yaml @@ -170,7 +170,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-secret-env.yaml b/internal/state/testdata/golden/driver-secret-env.yaml index 88b33bbb54..17fd861943 100644 --- a/internal/state/testdata/golden/driver-secret-env.yaml +++ b/internal/state/testdata/golden/driver-secret-env.yaml @@ -171,7 +171,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml index 8350172593..19a8726a84 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml @@ -168,7 +168,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/internal/state/testdata/golden/driver-vgpu-licensing.yaml b/internal/state/testdata/golden/driver-vgpu-licensing.yaml index 8d6f8804b4..0286d488d1 100644 --- a/internal/state/testdata/golden/driver-vgpu-licensing.yaml +++ b/internal/state/testdata/golden/driver-vgpu-licensing.yaml @@ -168,7 +168,7 @@ spec: command: - sh - -c - - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready + - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready' failureThreshold: 120 initialDelaySeconds: 60 periodSeconds: 10 diff --git a/manifests/state-driver/0500_daemonset.yaml b/manifests/state-driver/0500_daemonset.yaml index f32bfc9ce8..e3d0dc1f6f 100644 --- a/manifests/state-driver/0500_daemonset.yaml +++ b/manifests/state-driver/0500_daemonset.yaml @@ -331,7 +331,7 @@ spec: startupProbe: exec: command: - [sh, -c, 'nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'] + [sh, -c, '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'] initialDelaySeconds: {{ .Driver.Spec.StartupProbe.InitialDelaySeconds }} failureThreshold: {{ .Driver.Spec.StartupProbe.FailureThreshold }} successThreshold: {{ .Driver.Spec.StartupProbe.SuccessThreshold }}