diff --git a/internal/gpuconfig/loadid/driver.go b/internal/gpuconfig/loadid/driver.go new file mode 100644 index 000000000..c12335ad9 --- /dev/null +++ b/internal/gpuconfig/loadid/driver.go @@ -0,0 +1,166 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package loadid + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvidiav1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1" + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" +) + +// These names match the ones nvidia-validator and the driver container use. +const ( + // DriverReadyFile is written by every driver-validation step once the + // driver is ready. + DriverReadyFile = "driver-ready" + // DriverCtrReadyFile is written by the operator's driver container when + // its install is done, and removed by the driver pod's preStop hook. + DriverCtrReadyFile = ".driver-ctr-ready" + + // DriverDaemonSetSelector selects the driver DaemonSets. + DriverDaemonSetSelector = "app.kubernetes.io/component=nvidia-driver" +) + +// DriverInfo is the content of DriverReadyFile. +type DriverInfo struct { + IsHostDriver bool + // DriverRoot is the driver's install location on the host. + DriverRoot string + // DriverRootCtrPath is where the driver root is mounted in the container: + // /host for a host driver, /driver-root for a containerized one. + DriverRootCtrPath string + DevRoot string + DevRootCtrPath string +} + +// readDriverInfo reads DriverReadyFile in dir. ok is false when the file +// doesn't exist, which includes the moment between a validation step +// deleting it and writing it again. +func readDriverInfo(dir string) (info *DriverInfo, ok bool, err error) { + data, err := os.ReadFile(filepath.Join(dir, DriverReadyFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("reading %s: %w", DriverReadyFile, err) + } + info, err = parseDriverInfo(data) + if err != nil { + return nil, false, fmt.Errorf("%s: %w", DriverReadyFile, err) + } + return info, true, nil +} + +func parseDriverInfo(data []byte) (*DriverInfo, error) { + info := &DriverInfo{} + sawHostDriver := false + s := bufio.NewScanner(bytes.NewReader(data)) + for s.Scan() { + key, value, found := strings.Cut(s.Text(), "=") + if !found { + continue + } + switch key { + case "IS_HOST_DRIVER": + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("IS_HOST_DRIVER: %w", err) + } + info.IsHostDriver = b + sawHostDriver = true + case "NVIDIA_DRIVER_ROOT": + info.DriverRoot = value + case "DRIVER_ROOT_CTR_PATH": + info.DriverRootCtrPath = value + case "NVIDIA_DEV_ROOT": + info.DevRoot = value + case "DEV_ROOT_CTR_PATH": + info.DevRootCtrPath = value + } + } + if err := s.Err(); err != nil { + return nil, fmt.Errorf("scanning: %w", err) + } + // Telling a host driver from a containerized one decides whether + // DriverCtrReadyFile is needed, so its absence can't default to either. + if !sawHostDriver { + return nil, errors.New("missing IS_HOST_DRIVER") + } + return info, nil +} + +// Ready reports whether the driver is ready and returns its DriverReadyFile +// content. A containerized driver the operator manages (see +// ManagedByOperator) also needs DriverCtrReadyFile. +func Ready(dir string, operatorManaged bool) (*DriverInfo, bool, error) { + info, ok, err := readDriverInfo(dir) + if err != nil || !ok { + return nil, false, err + } + // A driver container the operator doesn't manage never writes + // DriverCtrReadyFile, so waiting for it there would wait forever. + if info.IsHostDriver || !operatorManaged { + return info, true, nil + } + _, err = os.Stat(filepath.Join(dir, DriverCtrReadyFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("stat %s: %w", DriverCtrReadyFile, err) + } + return info, true, nil +} + +// DaemonSetLister lists DaemonSets in a namespace by label selector. +type DaemonSetLister interface { + ListDaemonSets(ctx context.Context, namespace, labelSelector string) ([]appsv1.DaemonSet, error) +} + +// ManagedByOperator reports whether the operator manages the driver: a driver +// DaemonSet in namespace has a ClusterPolicy or NVIDIADriver as its +// controller. It makes the same check as nvidia-validator's driver step. +func ManagedByOperator(ctx context.Context, l DaemonSetLister, namespace string) (bool, error) { + dss, err := l.ListDaemonSets(ctx, namespace, DriverDaemonSetSelector) + if err != nil { + return false, fmt.Errorf("listing driver DaemonSets: %w", err) + } + for i := range dss { + owner := metav1.GetControllerOf(&dss[i]) + if owner == nil { + continue + } + if strings.HasPrefix(owner.APIVersion, "nvidia.com/") && + (owner.Kind == nvidiav1.ClusterPolicyCRDName || owner.Kind == nvidiav1alpha1.NVIDIADriverCRDName) { + return true, nil + } + } + return false, nil +} diff --git a/internal/gpuconfig/loadid/loadid.go b/internal/gpuconfig/loadid/loadid.go new file mode 100644 index 000000000..8d12205d3 --- /dev/null +++ b/internal/gpuconfig/loadid/loadid.go @@ -0,0 +1,50 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// Package loadid identifies the current load of the NVIDIA kernel module, so +// that a decision made for one driver load is never reused after a reload, +// and reports whether the driver is ready for NVML to be opened. +package loadid + +import ( + "errors" + "fmt" + "io/fs" + "os" + "syscall" +) + +// ModulePath is the sysfs directory of the nvidia kernel module. +const ModulePath = "/sys/module/nvidia" + +// Read returns the load ID of the current driver load: the inode number of +// path, normally ModulePath. The kernel gives the directory a new inode at +// every module load, so the load ID changes on every reload and never repeats +// within a boot. ok is false while the module isn't loaded. +func Read(path string) (l uint64, ok bool, err error) { + fi, err := os.Stat(path) + if errors.Is(err, fs.ErrNotExist) { + return 0, false, nil + } + if err != nil { + return 0, false, fmt.Errorf("stat %s: %w", path, err) + } + st, isStat := fi.Sys().(*syscall.Stat_t) + if !isStat { + return 0, false, fmt.Errorf("stat %s: no inode number", path) + } + return st.Ino, true, nil +} diff --git a/internal/gpuconfig/loadid/loadid_test.go b/internal/gpuconfig/loadid/loadid_test.go new file mode 100644 index 000000000..1339074d8 --- /dev/null +++ b/internal/gpuconfig/loadid/loadid_test.go @@ -0,0 +1,219 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package loadid + +import ( + "context" + "errors" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestRead(t *testing.T) { + dir := t.TempDir() + module := filepath.Join(dir, "nvidia") + + _, ok, err := Read(module) + require.NoError(t, err) + require.False(t, ok) + + require.NoError(t, os.Mkdir(module, 0o700)) + l, ok, err := Read(module) + require.NoError(t, err) + require.True(t, ok) + + fi, err := os.Stat(module) + require.NoError(t, err) + require.Equal(t, fi.Sys().(*syscall.Stat_t).Ino, l) + require.NotZero(t, l) +} + +// The format nvidia-validator's driver step writes. +const containerizedDriver = `IS_HOST_DRIVER=false +NVIDIA_DRIVER_ROOT=/run/nvidia/driver +DRIVER_ROOT_CTR_PATH=/driver-root +NVIDIA_DEV_ROOT=/run/nvidia/driver +DEV_ROOT_CTR_PATH=/driver-root +` + +const hostDriver = `IS_HOST_DRIVER=true +NVIDIA_DRIVER_ROOT=/ +DRIVER_ROOT_CTR_PATH=/host +NVIDIA_DEV_ROOT=/ +DEV_ROOT_CTR_PATH=/host +` + +func TestReadDriverInfo(t *testing.T) { + cases := map[string]struct { + content string + want *DriverInfo + wantErr bool + }{ + "containerized driver": { + content: containerizedDriver, + want: &DriverInfo{ + DriverRoot: "/run/nvidia/driver", DriverRootCtrPath: "/driver-root", + DevRoot: "/run/nvidia/driver", DevRootCtrPath: "/driver-root", + }, + }, + "host driver": { + content: hostDriver, + want: &DriverInfo{ + IsHostDriver: true, DriverRoot: "/", DriverRootCtrPath: "/host", + DevRoot: "/", DevRootCtrPath: "/host", + }, + }, + "unknown keys and blank lines are ignored": { + content: "IS_HOST_DRIVER=true\n\nNEW_KEY=x\n", + want: &DriverInfo{IsHostDriver: true}, + }, + "missing IS_HOST_DRIVER": { + content: "NVIDIA_DRIVER_ROOT=/\n", + wantErr: true, + }, + "invalid IS_HOST_DRIVER": { + content: "IS_HOST_DRIVER=maybe\n", + wantErr: true, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, DriverReadyFile), []byte(tc.content), 0o600)) + got, ok, err := readDriverInfo(dir) + if tc.wantErr { + require.Error(t, err) + require.False(t, ok) + return + } + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, tc.want, got) + }) + } +} + +func TestReadDriverInfoMissing(t *testing.T) { + info, ok, err := readDriverInfo(t.TempDir()) + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, info) +} + +func TestReady(t *testing.T) { + cases := map[string]struct { + driverReady string + ctrReady bool + operatorManaged bool + want bool + }{ + "no driver-ready": {want: false}, + "no driver-ready, container ready": {ctrReady: true, operatorManaged: true, want: false}, + "host driver": {driverReady: hostDriver, operatorManaged: true, want: true}, + "operator-managed container, not ready": {driverReady: containerizedDriver, operatorManaged: true, want: false}, + "operator-managed container, ready": {driverReady: containerizedDriver, ctrReady: true, operatorManaged: true, want: true}, + "container the operator doesn't manage": {driverReady: containerizedDriver, want: true}, + "unmanaged container with a leftover file": {driverReady: containerizedDriver, ctrReady: true, want: true}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + if tc.driverReady != "" { + require.NoError(t, os.WriteFile(filepath.Join(dir, DriverReadyFile), []byte(tc.driverReady), 0o600)) + } + if tc.ctrReady { + require.NoError(t, os.WriteFile(filepath.Join(dir, DriverCtrReadyFile), nil, 0o600)) + } + info, ok, err := Ready(dir, tc.operatorManaged) + require.NoError(t, err) + require.Equal(t, tc.want, ok) + require.Equal(t, tc.want, info != nil) + }) + } +} + +func TestReadyErrorOnUnparsableDriverReady(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, DriverReadyFile), []byte("NVIDIA_DRIVER_ROOT=/\n"), 0o600)) + _, ok, err := Ready(dir, true) + require.Error(t, err) + require.False(t, ok) +} + +func TestReadyErrorOnUnreadableDriverReady(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(dir, DriverReadyFile), 0o700)) + info, ok, err := Ready(dir, true) + require.Error(t, err) + require.False(t, ok) + require.Nil(t, info) +} + +type fakeLister struct { + dss []appsv1.DaemonSet + err error + namespace string + selector string +} + +func (f *fakeLister) ListDaemonSets(_ context.Context, namespace, labelSelector string) ([]appsv1.DaemonSet, error) { + f.namespace, f.selector = namespace, labelSelector + return f.dss, f.err +} + +func daemonSet(apiVersion, kind string, controller bool) appsv1.DaemonSet { + return appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{{APIVersion: apiVersion, Kind: kind, Name: "owner", Controller: &controller}}, + }} +} + +func TestManagedByOperator(t *testing.T) { + cases := map[string]struct { + dss []appsv1.DaemonSet + want bool + }{ + "no driver DaemonSet": {}, + "owned by ClusterPolicy": {dss: []appsv1.DaemonSet{daemonSet("nvidia.com/v1", "ClusterPolicy", true)}, want: true}, + "owned by NVIDIADriver": {dss: []appsv1.DaemonSet{daemonSet("nvidia.com/v1alpha1", "NVIDIADriver", true)}, want: true}, + "owner isn't the controller": {dss: []appsv1.DaemonSet{daemonSet("nvidia.com/v1", "ClusterPolicy", false)}}, + "owned by another kind": {dss: []appsv1.DaemonSet{daemonSet("nvidia.com/v1", "GPUCluster", true)}}, + "owned by another API group": {dss: []appsv1.DaemonSet{daemonSet("example.com/v1", "ClusterPolicy", true)}}, + "no owner": {dss: []appsv1.DaemonSet{{}}}, + "one of several is operator-owned": {dss: []appsv1.DaemonSet{{}, daemonSet("nvidia.com/v1", "ClusterPolicy", true)}, want: true}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + l := &fakeLister{dss: tc.dss} + got, err := ManagedByOperator(context.Background(), l, "gpu-operator") + require.NoError(t, err) + require.Equal(t, tc.want, got) + require.Equal(t, "gpu-operator", l.namespace) + require.Equal(t, "app.kubernetes.io/component=nvidia-driver", l.selector) + }) + } +} + +func TestManagedByOperatorListError(t *testing.T) { + _, err := ManagedByOperator(context.Background(), &fakeLister{err: errors.New("forbidden")}, "gpu-operator") + require.ErrorContains(t, err, "forbidden") +} diff --git a/internal/gpuconfig/statefile/file.go b/internal/gpuconfig/statefile/file.go new file mode 100644 index 000000000..63121cadb --- /dev/null +++ b/internal/gpuconfig/statefile/file.go @@ -0,0 +1,219 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package statefile + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "maps" + "os" + "path/filepath" + "syscall" + "time" +) + +const ( + // DefaultDir is the host tmpfs directory that holds the state, lock and temp files. + DefaultDir = "/run/nvidia/validations" + + // The names start with a dot so that the validators' preStop cleanup + // globs leave them in place. + StateFileName = ".gpu-config-state" + lockFileName = ".gpu-config.lock" + tempFileName = ".gpu-config-state.tmp" +) + +// ErrLockBusy is returned by TryLock when another process holds the lock. +var ErrLockBusy = errors.New("state file lock is busy") + +// Store gives access to the state, lock and temp files in one directory. +type Store struct { + dir string + now func() time.Time +} + +// New returns a Store for the state, lock and temp files in dir. +func New(dir string) *Store { + return &Store{dir: dir, now: time.Now} +} + +// Read reads the state file without the lock. A missing file reads as one with +// neither object; a file that doesn't parse returns an error wrapping ErrParse. +func (s *Store) Read() (*File, error) { + data, err := os.ReadFile(filepath.Join(s.dir, StateFileName)) + if errors.Is(err, fs.ErrNotExist) { + return &File{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading state file: %w", err) + } + return parse(data) +} + +// Lock takes the lock, blocking until it is free. +func (s *Store) Lock() (*Lock, error) { + return s.lock(syscall.LOCK_EX) +} + +// TryLock takes the lock without blocking, returning ErrLockBusy when it is +// held. +func (s *Store) TryLock() (*Lock, error) { + return s.lock(syscall.LOCK_EX | syscall.LOCK_NB) +} + +func (s *Store) lock(how int) (*Lock, error) { + f, err := os.OpenFile(filepath.Join(s.dir, lockFileName), os.O_RDONLY|os.O_CREATE, 0o600) + if err != nil { + return nil, fmt.Errorf("opening lock file: %w", err) + } + for { + err = syscall.Flock(int(f.Fd()), how) + if !errors.Is(err, syscall.EINTR) { + break + } + } + if err != nil { + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, ErrLockBusy + } + return nil, fmt.Errorf("locking %s: %w", lockFileName, err) + } + return &Lock{store: s, f: f}, nil +} + +// Lock is a held state file lock. Every write goes through it. +type Lock struct { + store *Store + f *os.File + // base is the file as read in this hold, the only base a write may use. + base *File +} + +// Unlock releases the lock and closes the lock file. Closing alone would +// release it too, but only once no other reference to the file remains. +func (l *Lock) Unlock() error { + if l.f == nil { + return nil + } + err := syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN) + if cerr := l.f.Close(); err == nil { + err = cerr + } + l.f = nil + l.base = nil + return err +} + +// Read reads the state file under the lock. Writes in this hold build on the +// result of the latest Read; after a failed Read they refuse to write. +func (l *Lock) Read() (*File, error) { + f, err := l.store.Read() + l.base = f + return f, err +} + +// WriteApply records an applied or skipped decision at module, keeping every +// other top-level key. +func (l *Lock) WriteApply(module uint64, value Decision, config string) error { + if value != Applied && value != Skipped { + return fmt.Errorf("apply.value must be %s or %s, got %q", Applied, Skipped, value) + } + return l.write("apply", func(now time.Time) any { + return Apply{Module: module, Value: value, Config: config, Time: now} + }) +} + +// WriteGatePass records a gate pass at module, which must be the load ID the +// gate read in this hold, or 0. +func (l *Lock) WriteGatePass(module uint64) error { + return l.write("gate", func(now time.Time) any { + return Gate{Module: module, Time: now} + }) +} + +func (l *Lock) write(key string, build func(now time.Time) any) error { + if l.f == nil { + return errors.New("state file write without the lock") + } + if l.base == nil { + return errors.New("state file write with no successful read in this lock hold") + } + now := l.store.timestamp() + content, err := encode(l.base, key, build(now), now) + if err != nil { + return err + } + // A base that parsed can still encode to content that doesn't, such as one + // with a duplicate or case-folded version key; never put that on disk. + written, err := parse(content) + if err != nil { + return fmt.Errorf("refusing to write an unparsable state file: %w", err) + } + + tmp := filepath.Join(l.store.dir, tempFileName) + // Only the lock holder writes the temp file, so truncating a leftover from + // a killed writer is safe. + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("opening temp state file: %w", err) + } + if _, err := f.Write(content); err != nil { + _ = f.Close() + return fmt.Errorf("writing temp state file: %w", err) + } + if err := f.Close(); err != nil { + return fmt.Errorf("closing temp state file: %w", err) + } + if err := os.Rename(tmp, filepath.Join(l.store.dir, StateFileName)); err != nil { + return fmt.Errorf("renaming temp state file: %w", err) + } + l.base = written + return nil +} + +// encode builds the new file content: base's top-level keys with key replaced +// by obj, or a new file when base doesn't exist. +func encode(base *File, key string, obj any, now time.Time) ([]byte, error) { + out := make(map[string]json.RawMessage, len(base.raw)+1) + if base.Exists() { + maps.Copy(out, base.raw) + } else { + out["version"] = json.RawMessage(fmt.Sprint(stateVersion)) + created, err := json.Marshal(now) + if err != nil { + return nil, fmt.Errorf("encoding created time: %w", err) + } + out["created"] = created + } + b, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("encoding %s object: %w", key, err) + } + out[key] = b + content, err := json.Marshal(out) + if err != nil { + return nil, fmt.Errorf("encoding state file: %w", err) + } + return content, nil +} + +func (s *Store) timestamp() time.Time { + return s.now().UTC().Truncate(time.Second) +} diff --git a/internal/gpuconfig/statefile/file_test.go b/internal/gpuconfig/statefile/file_test.go new file mode 100644 index 000000000..d3cc0a0a8 --- /dev/null +++ b/internal/gpuconfig/statefile/file_test.go @@ -0,0 +1,304 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package statefile + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +var testNow = time.Date(2026, 9, 26, 8, 15, 2, 500, time.UTC) + +func newTestStore(t *testing.T) *Store { + s := New(t.TempDir()) + s.now = func() time.Time { return testNow } + return s +} + +func writeState(t *testing.T, s *Store, content string) { + require.NoError(t, os.WriteFile(filepath.Join(s.dir, StateFileName), []byte(content), 0o600)) +} + +func readState(t *testing.T, s *Store) string { + b, err := os.ReadFile(filepath.Join(s.dir, StateFileName)) + require.NoError(t, err) + return string(b) +} + +func TestReadMissingFile(t *testing.T) { + f, err := newTestStore(t).Read() + require.NoError(t, err) + require.False(t, f.Exists()) + require.Nil(t, f.Apply) + require.Nil(t, f.Gate) +} + +func TestReadIOErrorIsNotParseError(t *testing.T) { + s := newTestStore(t) + require.NoError(t, os.Mkdir(filepath.Join(s.dir, StateFileName), 0o700)) + _, err := s.Read() + require.Error(t, err) + require.NotErrorIs(t, err, ErrParse) +} + +func TestWrites(t *testing.T) { + cases := map[string]struct { + existing string + write func(*Lock) error + want string + }{ + "gate creates the file": { + write: func(l *Lock) error { return l.WriteGatePass(7) }, + want: `{"created":"2026-09-26T08:15:02Z","gate":{"module":7,"time":"2026-09-26T08:15:02Z"},"version":1}`, + }, + "applier creates the file": { + write: func(l *Lock) error { return l.WriteApply(7, Skipped, "") }, + want: `{"apply":{"module":7,"value":"skipped","config":"","time":"2026-09-26T08:15:02Z"},"created":"2026-09-26T08:15:02Z","version":1}`, + }, + "gate writes module 0 explicitly": { + existing: `{"version":1,"created":"2026-01-01T00:00:00Z"}`, + write: func(l *Lock) error { return l.WriteGatePass(0) }, + want: `{"created":"2026-01-01T00:00:00Z","gate":{"module":0,"time":"2026-09-26T08:15:02Z"},"version":1}`, + }, + "applier carries gate and top-level keys, drops its own unknown fields": { + existing: `{"version":1,"created":"2026-01-01T00:00:00Z","future":{"module":3, "x":"<"},` + + `"apply":{"module":3,"value":"pending","config":"sha256:aa","time":"2026-01-01T00:00:00Z","extra":1},` + + `"gate":{"module":3,"time":"2026-01-01T00:00:01Z","extra":2}}`, + write: func(l *Lock) error { return l.WriteApply(3, Applied, "sha256:bb") }, + want: `{"apply":{"module":3,"value":"applied","config":"sha256:bb","time":"2026-09-26T08:15:02Z"},` + + // Carried keys keep their meaning, re-emitted compacted and HTML-escaped. + `"created":"2026-01-01T00:00:00Z","future":{"module":3,"x":"` + "\\u003c" + `"},` + + `"gate":{"module":3,"time":"2026-01-01T00:00:01Z","extra":2},"version":1}`, + }, + "gate carries apply and drops its own unknown fields": { + existing: `{"version":1,"created":"2026-01-01T00:00:00Z",` + + `"apply":{"module":3,"value":"applied","config":"","time":"2026-01-01T00:00:00Z","extra":1},` + + `"gate":{"module":2,"time":"2026-01-01T00:00:01Z","extra":2}}`, + write: func(l *Lock) error { return l.WriteGatePass(3) }, + want: `{"apply":{"module":3,"value":"applied","config":"","time":"2026-01-01T00:00:00Z","extra":1},` + + `"created":"2026-01-01T00:00:00Z","gate":{"module":3,"time":"2026-09-26T08:15:02Z"},"version":1}`, + }, + "null objects are replaced or carried": { + existing: `{"version":1,"apply":null,"gate":null}`, + write: func(l *Lock) error { return l.WriteApply(3, Skipped, "") }, + want: `{"apply":{"module":3,"value":"skipped","config":"","time":"2026-09-26T08:15:02Z"},"gate":null,"version":1}`, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + s := newTestStore(t) + if tc.existing != "" { + writeState(t, s, tc.existing) + } + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + + _, err = l.Read() + require.NoError(t, err) + require.NoError(t, tc.write(l)) + require.Equal(t, tc.want, readState(t, s)) + + _, err = s.Read() + require.NoError(t, err) + }) + } +} + +func TestWriteTruncatesLeftoverTempFile(t *testing.T) { + s := newTestStore(t) + require.NoError(t, os.WriteFile(filepath.Join(s.dir, tempFileName), make([]byte, 4096), 0o600)) + + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + _, err = l.Read() + require.NoError(t, err) + require.NoError(t, l.WriteGatePass(7)) + + got, err := s.Read() + require.NoError(t, err) + require.True(t, got.Passed(7)) + require.NoFileExists(t, filepath.Join(s.dir, tempFileName)) +} + +func TestWriteAfterUnlockFails(t *testing.T) { + s := newTestStore(t) + l, err := s.Lock() + require.NoError(t, err) + _, err = l.Read() + require.NoError(t, err) + require.NoError(t, l.Unlock()) + + require.Error(t, l.WriteGatePass(7)) + require.NoFileExists(t, filepath.Join(s.dir, StateFileName)) +} + +func TestTryLockBusy(t *testing.T) { + s := newTestStore(t) + held, err := s.Lock() + require.NoError(t, err) + + _, err = s.TryLock() + require.ErrorIs(t, err, ErrLockBusy) + + require.NoError(t, held.Unlock()) + l, err := s.TryLock() + require.NoError(t, err) + require.NoError(t, l.Unlock()) + require.FileExists(t, filepath.Join(s.dir, lockFileName)) +} + +func TestLockBlocksUntilUnlock(t *testing.T) { + s := newTestStore(t) + held, err := s.TryLock() + require.NoError(t, err) + + type lockResult struct { + l *Lock + err error + } + acquired := make(chan lockResult, 1) + go func() { + l, err := s.Lock() + acquired <- lockResult{l, err} + }() + + select { + case r := <-acquired: + require.NoError(t, r.err) + t.Fatal("Lock returned while the lock was held") + case <-time.After(100 * time.Millisecond): + } + + require.NoError(t, held.Unlock()) + select { + case r := <-acquired: + require.NoError(t, r.err) + require.NoError(t, r.l.Unlock()) + case <-time.After(5 * time.Second): + t.Fatal("Lock didn't return after Unlock") + } +} + +func TestWriteNeedsAReadInThisHold(t *testing.T) { + s := newTestStore(t) + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + + require.Error(t, l.WriteGatePass(7)) + require.NoFileExists(t, filepath.Join(s.dir, StateFileName)) +} + +func TestWriteAfterAFailedReadFails(t *testing.T) { + s := newTestStore(t) + writeState(t, s, `{"version":1}`) + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + + _, err = l.Read() + require.NoError(t, err) + writeState(t, s, `{"version":2}`) + _, err = l.Read() + require.ErrorIs(t, err, ErrParse) + require.Error(t, l.WriteGatePass(7)) + require.Equal(t, `{"version":2}`, readState(t, s)) +} + +func TestWriteNeverReplacesAFileWithAnUnparsableOne(t *testing.T) { + cases := map[string]string{ + "duplicate version key": `{"version":1,"version":null}`, + "case-folded version key": `{"version":2,"Version":1}`, + } + for name, existing := range cases { + t.Run(name, func(t *testing.T) { + s := newTestStore(t) + writeState(t, s, existing) + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + + _, err = l.Read() + require.NoError(t, err) + require.Error(t, l.WriteGatePass(7)) + require.Equal(t, existing, readState(t, s)) + }) + } +} + +func TestWritesInOneHoldBuildOnEachOther(t *testing.T) { + s := newTestStore(t) + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + _, err = l.Read() + require.NoError(t, err) + + require.NoError(t, l.WriteGatePass(3)) + require.NoError(t, l.WriteApply(3, Applied, "sha256:aa")) + + got, err := s.Read() + require.NoError(t, err) + require.True(t, got.Passed(3)) + require.True(t, got.AppliedAt(3)) +} + +func TestWriteApplyOnlyWritesDecisions(t *testing.T) { + cases := map[string]Decision{ + "pending": "pending", + "missing value": "", + "unknown value": "future", + } + for name, v := range cases { + t.Run(name, func(t *testing.T) { + s := newTestStore(t) + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + _, err = l.Read() + require.NoError(t, err) + + require.Error(t, l.WriteApply(3, v, "")) + require.NoFileExists(t, filepath.Join(s.dir, StateFileName)) + }) + } +} + +func TestCreationUsesOneTimestamp(t *testing.T) { + s := New(t.TempDir()) + tick := testNow + s.now = func() time.Time { + tick = tick.Add(time.Second) + return tick + } + l, err := s.Lock() + require.NoError(t, err) + defer func() { _ = l.Unlock() }() + _, err = l.Read() + require.NoError(t, err) + require.NoError(t, l.WriteGatePass(7)) + + got, err := s.Read() + require.NoError(t, err) + require.Equal(t, got.Created, got.Gate.Time) +} diff --git a/internal/gpuconfig/statefile/state.go b/internal/gpuconfig/statefile/state.go new file mode 100644 index 000000000..ac77afa4a --- /dev/null +++ b/internal/gpuconfig/statefile/state.go @@ -0,0 +1,156 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +// Package statefile implements the node-local GPU configuration state file: +// its encoding, the lock, atomic writes and the predicates read from it. The +// applier, which configures the GPUs, records its decision there; the gate, +// which holds GPU allocators until that decision exists, records its passes. +package statefile + +import ( + "encoding/json" + "errors" + "fmt" + "time" +) + +// stateVersion is the only state file version. parse rejects a file with any +// other version. +const stateVersion = 1 + +// Decision is the value the applier recorded for a driver load. Only Applied +// and Skipped count as a decision; a missing or unknown value reads as not +// decided. +type Decision string + +const ( + Applied Decision = "applied" + Skipped Decision = "skipped" +) + +// ErrParse marks a state file that exists but doesn't parse. Neither side +// writes over such a file. +var ErrParse = errors.New("state file parse error") + +// State is the decoded state file. The uint64 and time.Time field types do the +// schema check while decoding; fields have no omitempty because a writer +// writes every field. +type State struct { + Version uint64 `json:"version"` + Created time.Time `json:"created"` + // Apply and Gate are nil when absent or null. + Apply *Apply `json:"apply"` + Gate *Gate `json:"gate"` +} + +// Apply is the applier's object. +type Apply struct { + Module uint64 `json:"module"` + Value Decision `json:"value"` + Config string `json:"config"` + Time time.Time `json:"time"` +} + +// Gate is the gate's object. +type Gate struct { + Module uint64 `json:"module"` + Time time.Time `json:"time"` +} + +// File is one read of the state file: the decoded State, plus the raw +// top-level keys that a write carries through unchanged. +type File struct { + State + // raw is nil when the file doesn't exist. + raw map[string]json.RawMessage +} + +// Exists reports whether the state file existed at the read. +func (f *File) Exists() bool { + return f.raw != nil +} + +// parse decodes the contents of an existing state file. Every failure wraps +// ErrParse. Unknown keys are ignored, never an error, so that fields added +// later read cleanly in older binaries. +func parse(data []byte) (*File, error) { + f := &File{} + if err := json.Unmarshal(data, &f.State); err != nil { + return nil, fmt.Errorf("%w: %w", ErrParse, err) + } + // version is the only required field, so this also rejects a top-level + // {} or null, which leave it at 0. + if f.Version != stateVersion { + return nil, fmt.Errorf("%w: version %d, want %d", ErrParse, f.Version, stateVersion) + } + // raw keeps every top-level key, unknown ones included, so a field a + // newer version added survives a write by this one. + if err := json.Unmarshal(data, &f.raw); err != nil { + return nil, fmt.Errorf("%w: %w", ErrParse, err) + } + return f, nil +} + +// The predicates below take l, the load ID of the current driver load, which +// is at least 1. + +// ApplyAt reports whether the apply object was written for this driver load. +func (s *State) ApplyAt(l uint64) bool { + return s.Apply != nil && s.Apply.Module == l +} + +// Decided reports whether the applier recorded applied or skipped for this +// driver load. +func (s *State) Decided(l uint64) bool { + return s.ApplyAt(l) && (s.Apply.Value == Applied || s.Apply.Value == Skipped) +} + +// AppliedAt reports whether the applier recorded applied for this driver load. +func (s *State) AppliedAt(l uint64) bool { + return s.ApplyAt(l) && s.Apply.Value == Applied +} + +// Undecided reports whether the apply object is for this driver load but holds +// a missing or unknown value. +func (s *State) Undecided(l uint64) bool { + return s.ApplyAt(l) && !s.Decided(l) +} + +// Passed reports whether the gate last passed at this driver load. +func (s *State) Passed(l uint64) bool { + return s.Gate != nil && s.Gate.Module == l +} + +// GatePassedWithoutDecision reports whether the gate last passed with no +// applier decision to rely on; the gate records such a pass at load ID 0. +func (s *State) GatePassedWithoutDecision() bool { + return s.Gate != nil && s.Gate.Module == 0 +} + +// GatePassedThisBoot reports whether a gate has passed at any driver load since +// boot, so pods released at an earlier load may still be on the node. +func (s *State) GatePassedThisBoot() bool { + return s.Gate != nil +} + +// ConfigAt returns the stored config hash when the apply object is for this +// driver load, otherwise "". +func (s *State) ConfigAt(l uint64) string { + if !s.ApplyAt(l) { + return "" + } + return s.Apply.Config +} diff --git a/internal/gpuconfig/statefile/state_test.go b/internal/gpuconfig/statefile/state_test.go new file mode 100644 index 000000000..fcffffe1e --- /dev/null +++ b/internal/gpuconfig/statefile/state_test.go @@ -0,0 +1,195 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package statefile + +import ( + "encoding/json" + "maps" + "slices" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestParseErrors(t *testing.T) { + cases := map[string]string{ + "empty file": ``, + "invalid JSON": `{"version":1`, + "trailing data": `{"version":1} {}`, + "top-level null": `null`, + "top-level empty object": `{}`, + "top-level array": `[]`, + "version 0": `{"version":0}`, + "version 2": `{"version":2}`, + "version as string": `{"version":"1"}`, + "version negative": `{"version":-1}`, + "version fractional": `{"version":1.0}`, + "version exponent": `{"version":1e0}`, + "apply as number": `{"version":1,"apply":5}`, + "apply.module negative": `{"version":1,"apply":{"module":-1}}`, + "apply.module as string": `{"version":1,"apply":{"module":"7"}}`, + "apply.value as number": `{"version":1,"apply":{"module":7,"value":1}}`, + "apply.config as number": `{"version":1,"apply":{"module":7,"config":1}}`, + "gate as string": `{"version":1,"gate":"x"}`, + "gate.module fractional": `{"version":1,"gate":{"module":7.5}}`, + "created not RFC 3339": `{"version":1,"created":"2026-09-26 08:15:02"}`, + "created as number": `{"version":1,"created":0}`, + "apply.time not RFC 3339": `{"version":1,"apply":{"module":7,"time":"yesterday"}}`, + "gate.time missing seconds": `{"version":1,"gate":{"module":7,"time":"2026-09-26T08:15Z"}}`, + } + for name, data := range cases { + t.Run(name, func(t *testing.T) { + _, err := parse([]byte(data)) + require.ErrorIs(t, err, ErrParse) + }) + } +} + +func TestParse(t *testing.T) { + ts := time.Date(2026, 9, 26, 8, 15, 2, 0, time.UTC) + cases := map[string]struct { + data string + want State + }{ + "every field set": { + data: `{"version":1,"created":"2026-09-26T08:15:02Z",` + + `"apply":{"module":110711,"value":"applied","config":"sha256:50d8","time":"2026-09-26T08:15:02Z"},` + + `"gate":{"module":110711,"time":"2026-09-26T08:15:02Z"}}`, + want: State{ + Version: 1, Created: ts, + Apply: &Apply{Module: 110711, Value: Applied, Config: "sha256:50d8", Time: ts}, + Gate: &Gate{Module: 110711, Time: ts}, + }, + }, + "objects absent": { + data: `{"version":1}`, + want: State{Version: 1}, + }, + "objects null": { + data: `{"version":1,"apply":null,"gate":null}`, + want: State{Version: 1}, + }, + "objects empty are present": { + data: `{"version":1,"apply":{},"gate":{}}`, + want: State{Version: 1, Apply: &Apply{}, Gate: &Gate{}}, + }, + "null timestamps read as zero": { + data: `{"version":1,"created":null,"gate":{"module":3,"time":null}}`, + want: State{Version: 1, Gate: &Gate{Module: 3}}, + }, + "unknown value is not a parse error": { + data: `{"version":1,"apply":{"module":3,"value":"reticulating"}}`, + want: State{Version: 1, Apply: &Apply{Module: 3, Value: "reticulating"}}, + }, + "unknown keys are ignored": { + data: `{"version":1,"future":{"module":3},"apply":{"module":3,"extra":true},"gate":{"module":3,"extra":[1]}}`, + want: State{Version: 1, Apply: &Apply{Module: 3}, Gate: &Gate{Module: 3}}, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + f, err := parse([]byte(tc.data)) + require.NoError(t, err) + require.True(t, f.Exists()) + require.Equal(t, tc.want, f.State) + }) + } +} + +func TestPredicates(t *testing.T) { + const l = 110711 + type preds struct { + applyAt, decided, applied, undecided, passed, noDecision, passedBoot bool + config string + } + apply := func(module uint64, v Decision) *Apply { + return &Apply{Module: module, Value: v, Config: "sha256:aa"} + } + cases := map[string]struct { + state State + want preds + }{ + "no file": { + state: State{}, + want: preds{}, + }, + "applied at the current load, gate absent": { + state: State{Apply: apply(l, Applied)}, + want: preds{applyAt: true, decided: true, applied: true, config: "sha256:aa"}, + }, + "skipped and gate passed at the current load": { + state: State{Apply: apply(l, Skipped), Gate: &Gate{Module: l}}, + want: preds{applyAt: true, decided: true, passed: true, passedBoot: true, config: "sha256:aa"}, + }, + "pending at the current load": { + state: State{Apply: apply(l, "pending"), Gate: &Gate{Module: l}}, + want: preds{applyAt: true, undecided: true, passed: true, passedBoot: true, config: "sha256:aa"}, + }, + "missing value at the current load": { + state: State{Apply: apply(l, "")}, + want: preds{applyAt: true, undecided: true, config: "sha256:aa"}, + }, + "unknown value at the current load": { + state: State{Apply: apply(l, "future")}, + want: preds{applyAt: true, undecided: true, config: "sha256:aa"}, + }, + "applied at an earlier driver load, gate passed then": { + state: State{Apply: apply(l-1, Applied), Gate: &Gate{Module: l - 1}}, + want: preds{passedBoot: true}, + }, + "gate passed without a decision": { + state: State{Apply: apply(l-1, Applied), Gate: &Gate{Module: 0}}, + want: preds{noDecision: true, passedBoot: true}, + }, + "empty objects": { + state: State{Apply: &Apply{}, Gate: &Gate{}}, + want: preds{noDecision: true, passedBoot: true}, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + s := tc.state + got := preds{ + applyAt: s.ApplyAt(l), + decided: s.Decided(l), + applied: s.AppliedAt(l), + undecided: s.Undecided(l), + passed: s.Passed(l), + noDecision: s.GatePassedWithoutDecision(), + passedBoot: s.GatePassedThisBoot(), + config: s.ConfigAt(l), + } + require.Equal(t, tc.want, got) + }) + } +} + +// The encoder silently drops two fields that share a JSON name, so pin the +// exact key set of each object. +func TestObjectKeys(t *testing.T) { + keys := func(v any) []string { + b, err := json.Marshal(v) + require.NoError(t, err) + var m map[string]json.RawMessage + require.NoError(t, json.Unmarshal(b, &m)) + return slices.Sorted(maps.Keys(m)) + } + require.Equal(t, []string{"config", "module", "time", "value"}, keys(Apply{})) + require.Equal(t, []string{"module", "time"}, keys(Gate{})) + require.Equal(t, []string{"apply", "created", "gate", "version"}, keys(State{})) +}