Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ require (
github.com/Masterminds/semver/v3 v3.5.0
github.com/Masterminds/sprig/v3 v3.3.0
github.com/NVIDIA/go-nvlib v0.13.0
github.com/NVIDIA/go-nvml v0.13.4-1.0.20261102194955-f83235b8e2ea
github.com/NVIDIA/k8s-kata-manager v0.2.3
github.com/NVIDIA/k8s-operator-libs v0.0.0-20261020171813-42ee2fff224f
github.com/NVIDIA/nvidia-container-toolkit v1.20.2
Expand All @@ -27,6 +28,7 @@ require (
github.com/stretchr/testify v1.12.1
github.com/urfave/cli/v3 v3.13.0
go.uber.org/zap v1.28.0
go.yaml.in/yaml/v2 v2.4.4
k8s.io/api v0.37.1
k8s.io/apiextensions-apiserver v0.37.1
k8s.io/apimachinery v0.37.1
Expand Down Expand Up @@ -106,7 +108,6 @@ require (
github.com/x448/float16 v0.8.4 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.58.0 // indirect
golang.org/x/mod v0.41.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ github.com/Mellanox/maintenance-operator/api v0.3.0 h1:TX+QflSZSOzinodbFdDK+2avE
github.com/Mellanox/maintenance-operator/api v0.3.0/go.mod h1:hJ+3RfMK82wT9BstT81bWA+6ZUy/zG6UzO5VzmkQfXg=
github.com/NVIDIA/go-nvlib v0.13.0 h1:veqetT7p2m+u7yPVAtXQ/nLAqotTdZF1+RzpLd1Ycis=
github.com/NVIDIA/go-nvlib v0.13.0/go.mod h1:OmA7IhiBevXA4jUElFQjwmAZJLqrIAofyyn9MvRcz7s=
github.com/NVIDIA/go-nvml v0.13.4-1.0.20261102194955-f83235b8e2ea h1:1gmgDjmKYvWPAITUkMO4zFhmmnY+pYlQEOCTXIkvyKY=
github.com/NVIDIA/go-nvml v0.13.4-1.0.20261102194955-f83235b8e2ea/go.mod h1:id63qwpoDWpFXwnwM6psDCSqW4BmNu6mWpr4YeQtPGo=
github.com/NVIDIA/k8s-kata-manager v0.2.3 h1:d5+gRFqU5el/fKMXhHUaPY7haj+dbHL4nDsO/q05LBo=
github.com/NVIDIA/k8s-kata-manager v0.2.3/go.mod h1:xx5OUiMsHyKbyX0JjKHqAftvqS8vx00LFn/5EaMdtB4=
github.com/NVIDIA/k8s-operator-libs v0.0.0-20261020171813-42ee2fff224f h1:UeAm5zDax07lYEl4cpJMO29QQusgsztdCj27LTI1IVM=
Expand Down
298 changes: 298 additions & 0 deletions internal/gpuconfig/apply/apply.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,298 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/

// Package apply writes resolved GPU settings through NVML: range checks, the
// ECC step and the setters.
package apply

import (
"errors"
"fmt"
"slices"
"strings"

"github.com/NVIDIA/gpu-operator/internal/gpuconfig/config"
)

// Every error Apply returns wraps one of these.
var (
// ErrRange marks a value outside the range the GPU reports, or a value
// NVML rejects at write time with NVML_ERROR_INVALID_ARGUMENT.
ErrRange = errors.New("range error")
// ErrNVML marks an NVML read or set error other than not supported.
ErrNVML = errors.New("NVML error")
)

// Setting names, as the configuration spells them.
const (
PowerLimit = "powerLimit"
GPUClocks = "gpuClocks"
MemoryClocks = "memoryClocks"
ECCMode = "eccMode"
)

// Target is one GPU and the settings it takes.
type Target struct {
Device Device
config.Assignment
}

// Skip is a managed setting that wasn't written because NVML reported it, or
// the query of its range, not supported on that GPU. It isn't an error; the
// caller logs it.
type Skip struct {
UUID string
Setting string
}

// Apply checks every value against its GPU's range before any write, then
// stages ECC modes, then writes the other settings one device at a time. A
// setting whose range can't be queried is skipped, so no unchecked value is
// written. Apply stops at the first error; settings already written stay
// written, and the error lists them.
func Apply(targets []Target) ([]Skip, error) {
var skips []Skip
plans := make([]plan, 0, len(targets))
for _, t := range targets {
p, err := checkRanges(t)
if err != nil {
return skips, err
}
for _, setting := range p.skip {
skips = append(skips, Skip{t.GPU.UUID, setting})
}
plans = append(plans, p)
}

w := &written{}
for _, t := range targets {
if t.Settings.ECCMode == nil {
continue
}
supported, wrote, err := stageECC(t.Device, *t.Settings.ECCMode == config.ECCEnabled)
if err != nil {
return skips, w.wrap(nvmlError(t, ECCMode, err))
}
if !supported {
skips = append(skips, Skip{t.GPU.UUID, ECCMode})
continue
}
if wrote {
w.add(t.GPU.UUID, ECCMode+" staged")
}
}

for _, p := range plans {
s, err := p.write(w)
skips = append(skips, s...)
if err != nil {
return skips, w.wrap(err)
}
}
return skips, nil
}

// written records the settings Apply has written, per GPU in write order.
type written struct {
uuids []string
settings map[string][]string
}

func (w *written) add(uuid, setting string) {
if w.settings == nil {
w.settings = map[string][]string{}
}
if _, ok := w.settings[uuid]; !ok {
w.uuids = append(w.uuids, uuid)
}
w.settings[uuid] = append(w.settings[uuid], setting)
}

func (w *written) wrap(err error) error {
if len(w.uuids) == 0 {
return err
}
parts := make([]string, 0, len(w.uuids))
for _, uuid := range w.uuids {
parts = append(parts, uuid+": "+strings.Join(w.settings[uuid], ", "))
}
return fmt.Errorf("%w; written before the failure: %s", err, strings.Join(parts, "; "))
}

type plan struct {
Target
// skip lists settings whose range query isn't supported; they aren't written.
skip []string
}

func checkRanges(t Target) (plan, error) {
p := plan{Target: t}
s := t.Settings

if s.PowerLimit != nil {
lo, hi, err := t.Device.PowerLimitConstraints()
switch {
case errors.Is(err, ErrNotSupported):
p.skip = append(p.skip, PowerLimit)
case err != nil:
return p, nvmlError(t, PowerLimit, err)
case milliwatts(*s.PowerLimit) < lo || milliwatts(*s.PowerLimit) > hi:
return p, fmt.Errorf("%w: GPU %s: %s %d W (%d mW) is outside %d-%d mW",
ErrRange, t.GPU.UUID, PowerLimit, *s.PowerLimit, milliwatts(*s.PowerLimit), lo, hi)
}
}

if s.MemoryClocks == nil && s.GPUClocks == nil {
return p, nil
}
// Not supported skips both clock settings, because the graphics-clock
// query takes a memory clock.
mem, err := t.Device.SupportedMemoryClocks()
if errors.Is(err, ErrNotSupported) {
if s.GPUClocks != nil {
p.skip = append(p.skip, GPUClocks)
}
if s.MemoryClocks != nil {
p.skip = append(p.skip, MemoryClocks)
}
return p, nil
}
if err != nil {
setting := MemoryClocks
if s.MemoryClocks == nil {
setting = GPUClocks
}
return p, nvmlError(t, setting, err)
}

if s.MemoryClocks != nil {
if err := checkClocks(t, MemoryClocks, *s.MemoryClocks, mem); err != nil {
return p, err
}
}

if s.GPUClocks != nil {
clocks, err := graphicsClocks(t.Device, mem)
switch {
case errors.Is(err, ErrNotSupported):
p.skip = append(p.skip, GPUClocks)
case err != nil:
return p, nvmlError(t, GPUClocks, err)
default:
if err := checkClocks(t, GPUClocks, *s.GPUClocks, clocks); err != nil {
return p, err
}
}
}
return p, nil
}

func graphicsClocks(d Device, mem []uint32) ([]uint32, error) {
var out []uint32
for _, m := range mem {
g, err := d.SupportedGraphicsClocks(m)
if err != nil {
return nil, err
}
out = append(out, g...)
}
return out, nil
}

func checkClocks(t Target, setting string, r config.ClockRange, supported []uint32) error {
if len(supported) == 0 {
return fmt.Errorf("%w: GPU %s: %s: the GPU reports no supported clocks", ErrRange, t.GPU.UUID, setting)
}
lo, hi := slices.Min(supported), slices.Max(supported)
if r.Min < lo || r.Max > hi {
return fmt.Errorf("%w: GPU %s: %s %d-%d MHz is outside %d-%d MHz",
ErrRange, t.GPU.UUID, setting, r.Min, r.Max, lo, hi)
}
return nil
}

// stageECC writes the pending ECC mode only where it differs, to avoid an
// InfoROM write on every apply. A GPU that doesn't support ECC counts as
// nothing staged.
func stageECC(d Device, enabled bool) (supported, wrote bool, err error) {
_, pending, err := d.ECCModes()
if errors.Is(err, ErrNotSupported) {
return false, false, nil
}
if err != nil {
return false, false, err
}
if pending == enabled {
return true, false, nil
}
err = d.SetECCMode(enabled)
if errors.Is(err, ErrNotSupported) {
return false, false, nil
}
if err != nil {
return false, false, err
}
return true, true, nil
}

// write sets power limit and clocks on one device. Unlike ECC, these are
// written unconditionally, because a value read back can reflect what a
// workload did.
func (p plan) write(w *written) ([]Skip, error) {
var skips []Skip
s := p.Settings
set := func(setting string, err error) error {
switch {
case err == nil:
w.add(p.GPU.UUID, setting)
return nil
case errors.Is(err, ErrNotSupported):
skips = append(skips, Skip{p.GPU.UUID, setting})
return nil
case errors.Is(err, ErrInvalidArgument):
return fmt.Errorf("%w: GPU %s: %s rejected by NVML: %w", ErrRange, p.GPU.UUID, setting, err)
default:
return nvmlError(p.Target, setting, err)
}
}

if s.PowerLimit != nil && !slices.Contains(p.skip, PowerLimit) {
if err := set(PowerLimit, p.Device.SetPowerLimit(milliwatts(*s.PowerLimit))); err != nil {
return skips, err
}
}
if s.GPUClocks != nil && !slices.Contains(p.skip, GPUClocks) {
if err := set(GPUClocks, p.Device.SetGPULockedClocks(s.GPUClocks.Min, s.GPUClocks.Max)); err != nil {
return skips, err
}
}
if s.MemoryClocks != nil && !slices.Contains(p.skip, MemoryClocks) {
if err := set(MemoryClocks, p.Device.SetMemoryLockedClocks(s.MemoryClocks.Min, s.MemoryClocks.Max)); err != nil {
return skips, err
}
}
return skips, nil
}

// milliwatts converts a configured power limit; config.Parse guarantees the
// product fits a uint32.
func milliwatts(watts uint32) uint32 {
return watts * 1000
}

func nvmlError(t Target, setting string, err error) error {
return fmt.Errorf("%w: GPU %s: %s: %w", ErrNVML, t.GPU.UUID, setting, err)
}
Loading
Loading