diff --git a/cmd/nvidia-validator/main.go b/cmd/nvidia-validator/main.go index 0035a386a7..78c5402b07 100644 --- a/cmd/nvidia-validator/main.go +++ b/cmd/nvidia-validator/main.go @@ -684,16 +684,10 @@ func runCommandWithWait(command string, args []string, sleepSeconds int, silent } // For driver container installs, check existence of .driver-ctr-ready to confirm running driver -// container has completed and is in Ready state. +// container has completed and is in Ready state. This is a single attempt: waitForDriver owns +// the retries so that it can keep re-probing the host driver while the container is not ready. func assertDriverContainerReady(silent bool) error { - command := shell - args := []string{"-c", fmt.Sprintf("stat %s", driverContainerStatusFilePath)} - - if withWaitFlag { - return runCommandWithWait(command, args, sleepIntervalSecondsFlag, silent) - } - - return runCommand(command, args, silent) + return runCommand(shell, []string{"-c", fmt.Sprintf("stat %s", driverContainerStatusFilePath)}, silent) } // isDriverManagedByOperator determines if the NVIDIA driver is managed by the GPU Operator. @@ -787,18 +781,32 @@ func validateDriverContainer(silent bool, driverManagedByOperator bool) error { return cmd.Run() } + log.Info("Attempting to validate a driver container installation") + if err := validateDriver(silent); err != nil { + return fmt.Errorf("error validating driver: %w", err) + } + return nil +} + +// waitForDriver waits for a usable driver installation, retrying the driver container and +// re-probing the host until one of them succeeds. It reports whether the driver was found on +// the host rather than in the driver container. +func waitForDriver(wait bool, intervalSeconds int, probeContainer, probeHost func() error) (bool, error) { for { - log.Info("Attempting to validate a driver container installation") - err := validateDriver(silent) - if err != nil { - if !withWaitFlag { - return fmt.Errorf("error validating driver: %w", err) - } - log.Warningf("failed to validate the driver, retrying after %d seconds\n", sleepIntervalSecondsFlag) - time.Sleep(time.Duration(sleepIntervalSecondsFlag) * time.Second) - continue + err := probeContainer() + if err == nil { + return false, nil + } + if !wait { + return false, err + } + + log.Warningf("failed to validate the driver, retrying after %d seconds\n", intervalSeconds) + time.Sleep(time.Duration(intervalSeconds) * time.Second) + + if err := probeHost(); err == nil { + return true, nil } - return nil } } @@ -818,11 +826,21 @@ func (d *Driver) runValidation(silent bool) (driverInfo, error) { return driverInfo{}, fmt.Errorf("error checking if driver is managed by GPU Operator: %w", err) } - err = validateDriverContainer(silent, driverManagedByOperator) + hostDriver, err := waitForDriver( + withWaitFlag, + sleepIntervalSecondsFlag, + func() error { return validateDriverContainer(silent, driverManagedByOperator) }, + func() error { return validateHostDriver(true) }, + ) if err != nil { return driverInfo{}, err } + if hostDriver { + log.Info("Detected a pre-installed driver on the host") + return getDriverInfo(true, hostRootFlag, hostRootFlag, "/host"), nil + } + if driverManagedByOperator { err = validateAdditionalDriverComponents(d.ctx, driverContainerStatusFilePath) if err != nil { diff --git a/cmd/nvidia-validator/main_test.go b/cmd/nvidia-validator/main_test.go index abbd73eec5..75b3a5c8c3 100644 --- a/cmd/nvidia-validator/main_test.go +++ b/cmd/nvidia-validator/main_test.go @@ -18,12 +18,14 @@ package main import ( "context" + "errors" "fmt" "io/fs" "os" "path/filepath" "strings" "testing" + "time" "github.com/NVIDIA/go-nvlib/pkg/nvmdev" "github.com/NVIDIA/go-nvlib/pkg/nvpci" @@ -494,3 +496,125 @@ func TestCountNvidiaDevices(t *testing.T) { }) } } + +func Test_waitForDriver(t *testing.T) { + // failUntil returns a probe that fails on its first n calls and succeeds afterwards, together + // with a pointer to its call count. A negative n never succeeds. + failUntil := func(n int) (func() error, *int) { + calls := 0 + return func() error { + calls++ + if n < 0 || calls <= n { + return errors.New("probe failed") + } + return nil + }, &calls + } + + testCases := []struct { + name string + wait bool + containerFailures int + hostFailures int + expectedHost bool + expectedErr bool + expectedHostCalls int + }{ + { + name: "driver container ready on the first attempt", + wait: true, + containerFailures: 0, + hostFailures: -1, + expectedHost: false, + expectedHostCalls: 0, + }, + { + name: "driver container becomes ready while waiting", + wait: true, + containerFailures: 3, + hostFailures: -1, + expectedHost: false, + expectedHostCalls: 3, + }, + { + // Regression: on a node whose driver ships with the OS image there is never going to + // be a driver container, so a host driver that only finishes loading after the + // validator started must still be detected. The container probe is given a finite + // number of failures so that this case fails rather than hangs if the host is no + // longer re-probed. + name: "host driver appears after the validator started", + wait: true, + containerFailures: 50, + hostFailures: 2, + expectedHost: true, + expectedHostCalls: 3, + }, + { + name: "container error is returned when not waiting", + wait: false, + containerFailures: -1, + hostFailures: 0, + expectedHost: false, + expectedErr: true, + expectedHostCalls: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + probeContainer, _ := failUntil(tc.containerFailures) + probeHost, hostCalls := failUntil(tc.hostFailures) + + hostDriver, err := waitForDriver(tc.wait, 0, probeContainer, probeHost) + + if tc.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.expectedHost, hostDriver) + require.Equal(t, tc.expectedHostCalls, *hostCalls) + }) + } +} + +// Regression: with the driver DaemonSet present in the cluster but not running on this node, the +// driver container readiness file never appears. The readiness check must not block on it, or the +// host driver is never re-probed. +func Test_waitForDriver_driverManagedByOperatorWithLateHostDriver(t *testing.T) { + _, err := os.Stat(driverContainerStatusFilePath) + require.True(t, os.IsNotExist(err), "test requires %s to be absent", driverContainerStatusFilePath) + + origWait, origInterval := withWaitFlag, sleepIntervalSecondsFlag + withWaitFlag, sleepIntervalSecondsFlag = true, 0 + t.Cleanup(func() { withWaitFlag, sleepIntervalSecondsFlag = origWait, origInterval }) + + hostCalls := 0 + probeHost := func() error { + hostCalls++ + if hostCalls <= 2 { + return errors.New("host driver not loaded yet") + } + return nil + } + + type result struct { + hostDriver bool + err error + } + done := make(chan result, 1) + go func() { + hostDriver, err := waitForDriver(withWaitFlag, sleepIntervalSecondsFlag, + func() error { return validateDriverContainer(true, true) }, probeHost) + done <- result{hostDriver, err} + }() + + select { + case r := <-done: + require.NoError(t, r.err) + require.True(t, r.hostDriver) + require.Equal(t, 3, hostCalls) + case <-time.After(10 * time.Second): + t.Fatal("waitForDriver blocked on the driver container readiness check and never re-probed the host") + } +}