Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 38 additions & 20 deletions cmd/nvidia-validator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -684,16 +684,10 @@ func runCommandWithWait(command string, args []string, sleepSeconds int, silent
}

// For driver container installs, check existence of .driver-ctr-ready to confirm running driver
// container has completed and is in Ready state.
// container has completed and is in Ready state. This is a single attempt: waitForDriver owns
// the retries so that it can keep re-probing the host driver while the container is not ready.
func assertDriverContainerReady(silent bool) error {
command := shell
args := []string{"-c", fmt.Sprintf("stat %s", driverContainerStatusFilePath)}

if withWaitFlag {
return runCommandWithWait(command, args, sleepIntervalSecondsFlag, silent)
}

return runCommand(command, args, silent)
return runCommand(shell, []string{"-c", fmt.Sprintf("stat %s", driverContainerStatusFilePath)}, silent)
}

// isDriverManagedByOperator determines if the NVIDIA driver is managed by the GPU Operator.
Expand Down Expand Up @@ -787,18 +781,32 @@ func validateDriverContainer(silent bool, driverManagedByOperator bool) error {
return cmd.Run()
}

log.Info("Attempting to validate a driver container installation")
if err := validateDriver(silent); err != nil {
return fmt.Errorf("error validating driver: %w", err)
}
return nil
}

// waitForDriver waits for a usable driver installation, retrying the driver container and
// re-probing the host until one of them succeeds. It reports whether the driver was found on
// the host rather than in the driver container.
func waitForDriver(wait bool, intervalSeconds int, probeContainer, probeHost func() error) (bool, error) {
for {
log.Info("Attempting to validate a driver container installation")
err := validateDriver(silent)
if err != nil {
if !withWaitFlag {
return fmt.Errorf("error validating driver: %w", err)
}
log.Warningf("failed to validate the driver, retrying after %d seconds\n", sleepIntervalSecondsFlag)
time.Sleep(time.Duration(sleepIntervalSecondsFlag) * time.Second)
continue
err := probeContainer()
if err == nil {
return false, nil
}
if !wait {
return false, err
Comment thread
joaopaulosr95 marked this conversation as resolved.
Resolved
}

log.Warningf("failed to validate the driver, retrying after %d seconds\n", intervalSeconds)
time.Sleep(time.Duration(intervalSeconds) * time.Second)

if err := probeHost(); err == nil {
return true, nil
}
return nil
}
}

Expand All @@ -818,11 +826,21 @@ func (d *Driver) runValidation(silent bool) (driverInfo, error) {
return driverInfo{}, fmt.Errorf("error checking if driver is managed by GPU Operator: %w", err)
}

err = validateDriverContainer(silent, driverManagedByOperator)
hostDriver, err := waitForDriver(
withWaitFlag,
sleepIntervalSecondsFlag,
func() error { return validateDriverContainer(silent, driverManagedByOperator) },
func() error { return validateHostDriver(true) },
Comment thread
joaopaulosr95 marked this conversation as resolved.
Resolved
)
if err != nil {
return driverInfo{}, err
}

if hostDriver {
log.Info("Detected a pre-installed driver on the host")
return getDriverInfo(true, hostRootFlag, hostRootFlag, "/host"), nil
}

if driverManagedByOperator {
err = validateAdditionalDriverComponents(d.ctx, driverContainerStatusFilePath)
if err != nil {
Expand Down
124 changes: 124 additions & 0 deletions cmd/nvidia-validator/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ package main

import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"time"

"github.com/NVIDIA/go-nvlib/pkg/nvmdev"
"github.com/NVIDIA/go-nvlib/pkg/nvpci"
Expand Down Expand Up @@ -494,3 +496,125 @@ func TestCountNvidiaDevices(t *testing.T) {
})
}
}

func Test_waitForDriver(t *testing.T) {
// failUntil returns a probe that fails on its first n calls and succeeds afterwards, together
// with a pointer to its call count. A negative n never succeeds.
failUntil := func(n int) (func() error, *int) {
calls := 0
return func() error {
calls++
if n < 0 || calls <= n {
return errors.New("probe failed")
}
return nil
}, &calls
}

testCases := []struct {
name string
wait bool
containerFailures int
hostFailures int
expectedHost bool
expectedErr bool
expectedHostCalls int
}{
{
name: "driver container ready on the first attempt",
wait: true,
containerFailures: 0,
hostFailures: -1,
expectedHost: false,
expectedHostCalls: 0,
},
{
name: "driver container becomes ready while waiting",
wait: true,
containerFailures: 3,
hostFailures: -1,
expectedHost: false,
expectedHostCalls: 3,
},
{
// Regression: on a node whose driver ships with the OS image there is never going to
// be a driver container, so a host driver that only finishes loading after the
// validator started must still be detected. The container probe is given a finite
// number of failures so that this case fails rather than hangs if the host is no
// longer re-probed.
name: "host driver appears after the validator started",
wait: true,
containerFailures: 50,
hostFailures: 2,
expectedHost: true,
expectedHostCalls: 3,
},
{
name: "container error is returned when not waiting",
wait: false,
containerFailures: -1,
hostFailures: 0,
expectedHost: false,
expectedErr: true,
expectedHostCalls: 0,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
probeContainer, _ := failUntil(tc.containerFailures)
probeHost, hostCalls := failUntil(tc.hostFailures)

hostDriver, err := waitForDriver(tc.wait, 0, probeContainer, probeHost)

if tc.expectedErr {
require.Error(t, err)
} else {
require.NoError(t, err)
}
require.Equal(t, tc.expectedHost, hostDriver)
require.Equal(t, tc.expectedHostCalls, *hostCalls)
})
}
}

// Regression: with the driver DaemonSet present in the cluster but not running on this node, the
// driver container readiness file never appears. The readiness check must not block on it, or the
// host driver is never re-probed.
func Test_waitForDriver_driverManagedByOperatorWithLateHostDriver(t *testing.T) {
_, err := os.Stat(driverContainerStatusFilePath)
require.True(t, os.IsNotExist(err), "test requires %s to be absent", driverContainerStatusFilePath)

origWait, origInterval := withWaitFlag, sleepIntervalSecondsFlag
withWaitFlag, sleepIntervalSecondsFlag = true, 0
t.Cleanup(func() { withWaitFlag, sleepIntervalSecondsFlag = origWait, origInterval })

hostCalls := 0
probeHost := func() error {
hostCalls++
if hostCalls <= 2 {
return errors.New("host driver not loaded yet")
}
return nil
}

type result struct {
hostDriver bool
err error
}
done := make(chan result, 1)
go func() {
hostDriver, err := waitForDriver(withWaitFlag, sleepIntervalSecondsFlag,
func() error { return validateDriverContainer(true, true) }, probeHost)
done <- result{hostDriver, err}
}()

select {
case r := <-done:
require.NoError(t, r.err)
require.True(t, r.hostDriver)
require.Equal(t, 3, hostCalls)
case <-time.After(10 * time.Second):
t.Fatal("waitForDriver blocked on the driver container readiness check and never re-probed the host")
}
}