From 731047f25348e6710f230e13bcbd6b95302eee04 Mon Sep 17 00:00:00 2001 From: Rajath Agasthya Date: Tue, 10 Nov 2026 11:35:47 -0600 Subject: [PATCH] Read driver upgrade state from the manager cache The upgrade state manager reads the driver DaemonSets, the driver Pods, and every node with a driver pod from the API server on every pass, also when no upgrade is in progress. Pass the manager's client to NewClusterUpgradeStateManager so that it reads these objects from the cache. The Node cache transform now keeps spec.unschedulable and the Ready condition, which the upgrade library reads to count unavailable nodes and to uncordon. The vendored k8s-operator-libs is an unreleased commit and needs a release before this merges. Signed-off-by: Rajath Agasthya --- cmd/gpu-operator/main.go | 1 + controllers/cache/cache.go | 14 +++-- controllers/cache/cache_test.go | 20 ++++-- go.mod | 2 +- go.sum | 4 +- .../pkg/upgrade/common_manager.go | 63 +++++++++++-------- .../k8s-operator-libs/pkg/upgrade/consts.go | 4 ++ .../pkg/upgrade/drain_manager.go | 5 -- .../upgrade/node_upgrade_state_provider.go | 24 +++---- .../pkg/upgrade/upgrade_state.go | 20 +++--- .../k8s-operator-libs/pkg/upgrade/util.go | 2 +- vendor/modules.txt | 2 +- 12 files changed, 96 insertions(+), 65 deletions(-) diff --git a/cmd/gpu-operator/main.go b/cmd/gpu-operator/main.go index 4578266f79..d26b64464f 100644 --- a/cmd/gpu-operator/main.go +++ b/cmd/gpu-operator/main.go @@ -173,6 +173,7 @@ func main() { clusterUpgradeStateManager, err := upgrade.NewClusterUpgradeStateManager( upgradeLogger, mgr.GetConfig(), + mgr.GetClient(), // nolint:staticcheck // TODO: update k8s-operator-libs to leverage events.EventRecorder instead mgr.GetEventRecorderFor("nvidia-gpu-operator"), diff --git a/controllers/cache/cache.go b/controllers/cache/cache.go index 2d51f103f6..3c503b72a9 100644 --- a/controllers/cache/cache.go +++ b/controllers/cache/cache.go @@ -58,6 +58,7 @@ func New(config *rest.Config, options ctrlcache.Options) (ctrlcache.Cache, error return ctrlcache.New(config, options) } +// stripUnusedNodeFields keeps only the Node fields that the operator reads. func stripUnusedNodeFields(in any) (any, error) { node, ok := in.(*corev1.Node) if !ok { @@ -65,10 +66,17 @@ func stripUnusedNodeFields(in any) (any, error) { } runtimeVersion := node.Status.NodeInfo.ContainerRuntimeVersion + var conditions []corev1.NodeCondition + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + conditions = append(conditions, corev1.NodeCondition{Type: condition.Type, Status: condition.Status}) + } + } node.ManagedFields = nil - node.Spec = corev1.NodeSpec{} + node.Spec = corev1.NodeSpec{Unschedulable: node.Spec.Unschedulable} node.Status = corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: runtimeVersion}, + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: runtimeVersion}, + Conditions: conditions, } return node, nil } @@ -85,8 +93,6 @@ func Options(operatorNamespace string) ctrlcache.Options { DefaultTransform: stripManagedFields, ByObject: map[client.Object]ctrlcache.ByObject{ &corev1.Node{}: { - // The operator reads only the container runtime version from node - // status. Drop all other status data from the cached object. Transform: stripUnusedNodeFields, }, &corev1.Pod{}: {Namespaces: cachedNamespaces}, diff --git a/controllers/cache/cache_test.go b/controllers/cache/cache_test.go index 616f572387..83ade85443 100644 --- a/controllers/cache/cache_test.go +++ b/controllers/cache/cache_test.go @@ -150,10 +150,16 @@ func TestCache(t *testing.T) { }, Spec: corev1.NodeSpec{Unschedulable: true}, Status: corev1.NodeStatus{ - Capacity: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, - Images: []corev1.ContainerImage{{Names: []string{"example.com/large-image:v1"}, SizeBytes: 1000000}}, - NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: "containerd://2.0.0", KernelVersion: "6.0.0"}, - Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + Capacity: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, + Images: []corev1.ContainerImage{{Names: []string{"example.com/large-image:v1"}, SizeBytes: 1000000}}, + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: "containerd://2.0.0", KernelVersion: "6.0.0"}, + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse}, + { + Type: corev1.NodeReady, Status: corev1.ConditionTrue, Reason: "KubeletReady", + LastHeartbeatTime: metav1.NewTime(time.Unix(1700000000, 0)), + }, + }, }, } configMap := &corev1.ConfigMap{ @@ -313,9 +319,11 @@ func TestCache(t *testing.T) { require.NoError(t, cachedClient.Get(ctx, client.ObjectKey{Name: node.Name}, gotNode)) expectedNode := node.DeepCopy() expectedNode.ManagedFields = nil - expectedNode.Spec = corev1.NodeSpec{} + // The upgrade library needs Spec.Unschedulable and the Ready condition. + expectedNode.Spec = corev1.NodeSpec{Unschedulable: true} expectedNode.Status = corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: node.Status.NodeInfo.ContainerRuntimeVersion}, + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: node.Status.NodeInfo.ContainerRuntimeVersion}, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, } require.Equal(t, expectedNode.ObjectMeta, gotNode.ObjectMeta) require.Equal(t, expectedNode.Spec, gotNode.Spec) diff --git a/go.mod b/go.mod index b96ab121d3..beea97544d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/NVIDIA/go-nvlib v0.13.0 github.com/NVIDIA/k8s-kata-manager v0.2.3 - github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494 + github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a github.com/NVIDIA/nvidia-container-toolkit v1.20.2 github.com/cyphar/filepath-securejoin v0.7.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc diff --git a/go.sum b/go.sum index 2f38bc4ac5..54ae3e11ce 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/NVIDIA/go-nvlib v0.13.0 h1:veqetT7p2m+u7yPVAtXQ/nLAqotTdZF1+RzpLd1Yci github.com/NVIDIA/go-nvlib v0.13.0/go.mod h1:OmA7IhiBevXA4jUElFQjwmAZJLqrIAofyyn9MvRcz7s= github.com/NVIDIA/k8s-kata-manager v0.2.3 h1:d5+gRFqU5el/fKMXhHUaPY7haj+dbHL4nDsO/q05LBo= github.com/NVIDIA/k8s-kata-manager v0.2.3/go.mod h1:xx5OUiMsHyKbyX0JjKHqAftvqS8vx00LFn/5EaMdtB4= -github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494 h1:j+tWK79l9AouBulQps7rxILLhy2fWYcEhH4zgYjth/o= -github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494/go.mod h1:L+aiCiTKN63AX9SWz/F8pv9Jw9FIfI+dAEr7VA+KowE= +github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a h1:PJTguzSwWLxQYBw/QNlqaRECYaK+CuDmsWpO67SZkbQ= +github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a/go.mod h1:TfactzFEcVt03/Qi40SkwOawkVVTXfNZJvhkS7hzTaM= github.com/NVIDIA/nvidia-container-toolkit v1.20.2 h1:ndvx+2v4c8HmRf8A5aONf1+sE32jNDZ7qnIaBEAN5i8= github.com/NVIDIA/nvidia-container-toolkit v1.20.2/go.mod h1:q/O2jOSbK8JeHYhHMWwhuwof0yXzs7VE5uVjfSZriDw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go index 2a16d979e2..0a6d41c41d 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go @@ -8,7 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/runtime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -116,13 +116,8 @@ type CommonUpgradeManagerImpl struct { func NewCommonUpgradeStateManager( log logr.Logger, k8sConfig *rest.Config, - scheme *runtime.Scheme, + k8sClient client.Client, eventRecorder record.EventRecorder) (*CommonUpgradeManagerImpl, error) { - k8sClient, err := client.New(k8sConfig, client.Options{Scheme: scheme}) - if err != nil { - return &CommonUpgradeManagerImpl{}, fmt.Errorf("error creating k8s client: %v", err) - } - k8sInterface, err := kubernetes.NewForConfig(k8sConfig) if err != nil { return &CommonUpgradeManagerImpl{}, fmt.Errorf("error creating k8s interface: %v", err) @@ -201,23 +196,11 @@ func (m *CommonUpgradeManagerImpl) GetDriverDaemonSets(ctx context.Context, name // GetPodsOwnedbyDs returns a list of the pods owned by the specified DaemonSet func (m *CommonUpgradeManagerImpl) GetPodsOwnedbyDs(ds *appsv1.DaemonSet, pods []corev1.Pod) []corev1.Pod { - dsPodList := []corev1.Pod{} - for i := range pods { - pod := &pods[i] - if IsOrphanedPod(pod) { - m.Log.V(consts.LogLevelInfo).Info("Driver Pod has no owner DaemonSet", "pod", pod.Name) - continue - } - m.Log.V(consts.LogLevelInfo).Info("Pod", "pod", pod.Name, "owner", pod.OwnerReferences[0].Name) - - if ds.UID != pod.OwnerReferences[0].UID { - m.Log.V(consts.LogLevelInfo).Info("Driver Pod is not owned by an Driver DaemonSet", - "pod", pod, "actual owner", pod.OwnerReferences[0]) - continue - } - dsPodList = append(dsPodList, *pod) - } - return dsPodList + podsByOwner := m.GetPodsByDSOwner( + map[types.UID]*appsv1.DaemonSet{ds.UID: ds}, + pods, + ) + return podsByOwner[ds.UID] } // GetOrphanedPods returns a list of the pods not owned by any DaemonSet @@ -233,8 +216,38 @@ func (m *CommonUpgradeManagerImpl) GetOrphanedPods(pods []corev1.Pod) []corev1.P return podList } +// GetPodsByDSOwner indexes pods controlled by the supplied DaemonSets. +// Pods without a controller are indexed under NoOwner. +func (m *CommonUpgradeManagerImpl) GetPodsByDSOwner( + daemonSets map[types.UID]*appsv1.DaemonSet, + pods []corev1.Pod) map[types.UID][]corev1.Pod { + podsByOwner := make(map[types.UID][]corev1.Pod) + + for _, pod := range pods { + owner := metav1.GetControllerOfNoCopy(&pod) + if owner == nil { + m.Log.V(consts.LogLevelInfo).Info("Driver Pod has no owner DaemonSet", "pod", pod.Name) + podsByOwner[NoOwner] = append(podsByOwner[NoOwner], pod) + continue + } + + daemonSet, found := daemonSets[owner.UID] + if !found || daemonSet == nil || !metav1.IsControlledBy(&pod, daemonSet) { + m.Log.V(consts.LogLevelInfo).Info("Driver Pod is not owned by a Driver DaemonSet", + "pod", pod, "actual owner", owner) + continue + } + + m.Log.V(consts.LogLevelInfo).Info("Pod", "pod", pod.Name, "owner", owner.Name) + podsByOwner[daemonSet.UID] = append(podsByOwner[daemonSet.UID], pod) + } + + m.Log.V(consts.LogLevelInfo).Info("Total orphaned Pods found:", "count", len(podsByOwner[NoOwner])) + return podsByOwner +} + func IsOrphanedPod(pod *corev1.Pod) bool { - return len(pod.OwnerReferences) < 1 + return metav1.GetControllerOfNoCopy(pod) == nil } // ProcessDoneOrUnknownNodes iterates over UpgradeStateDone or UpgradeStateUnknown nodes and determines diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go index 064ef8978d..9bab1ba6ab 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go @@ -16,6 +16,8 @@ limitations under the License. package upgrade +import "k8s.io/apimachinery/pkg/types" + const ( // UpgradeStateLabelKeyFmt is the format of the node label key indicating driver upgrade states UpgradeStateLabelKeyFmt = "nvidia.com/%s-driver-upgrade-state" @@ -83,6 +85,8 @@ const ( ) const ( + // NoOwner identifies pods that do not have a controller. + NoOwner types.UID = "" // nodeNameFieldSelectorFmt is the format of a field selector that can be used in metav1.ListOptions to filter by // node nodeNameFieldSelectorFmt = "spec.nodeName=%s" diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go index 4e0a70cebe..5910cc68bb 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go @@ -96,11 +96,6 @@ func (m *DrainManagerImpl) ScheduleNodesDrain(ctx context.Context, drainConfig * } for _, node := range drainConfig.Nodes { - // We need to shadow the loop variable or initialize some other one with its value - // to avoid concurrency issues when launching goroutines. - // If a loop variable is used as it is, all/most goroutines, spawned inside this loop, - // will use the 'node' value of the last item in drainConfig.Nodes - node := node if !m.drainingNodes.Has(node.Name) { m.log.V(consts.LogLevelInfo).Info("Schedule drain for node", "node", node.Name) logEvent(m.eventRecorder, node, corev1.EventTypeNormal, GetEventReason(), "Scheduling drain of the node") diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go index e6e1760baa..9ef41bd443 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go @@ -28,6 +28,12 @@ import ( "github.com/NVIDIA/k8s-operator-libs/pkg/consts" ) +// Interval and timeout for reading a Node until a patch shows in the cache. +const ( + pollInterval = 10 * time.Millisecond + pollTimeout = 10 * time.Second +) + // NodeUpgradeStateProvider allows for synchronized operations on node objects and ensures that the node, // got from the provider, always has the up-to-date upgrade state type NodeUpgradeStateProvider interface { @@ -97,13 +103,10 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeState( // so we wait until it's synced. // That way, since only one call to reconcile at a time is allowed for upgrade controller, each new update // will have the updated node object in the cache. - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*10) - defer cancel() - //nolint:staticcheck - err = wait.PollImmediateUntil(time.Second, func() (bool, error) { + err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) { p.Log.V(consts.LogLevelDebug).Info("Requesting node object to see if operator cache has updated", "node", node.Name) - err := p.K8sClient.Get(timeoutCtx, types.NamespacedName{Name: node.Name}, node) + err := p.K8sClient.Get(ctx, types.NamespacedName{Name: node.Name}, node) if err != nil { return false, err } @@ -114,7 +117,7 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeState( return false, nil } return true, nil - }, timeoutCtx.Done()) + }) if err != nil { p.Log.V(consts.LogLevelError).Error(err, "Error while waiting on node label update", @@ -168,13 +171,10 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeAnnotation( // so we wait until it's synced. // That way, since only one call to reconcile at a time is allowed for upgrade controller, each new update // will have the updated node object in the cache. - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*10) - defer cancel() - //nolint:staticcheck - err = wait.PollImmediateUntil(time.Second, func() (bool, error) { + err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) { p.Log.V(consts.LogLevelDebug).Info("Requesting node object to see if operator cache has updated", "node", node.Name) - err := p.K8sClient.Get(timeoutCtx, types.NamespacedName{Name: node.Name}, node) + err := p.K8sClient.Get(ctx, types.NamespacedName{Name: node.Name}, node) if err != nil { return false, err } @@ -194,7 +194,7 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeAnnotation( return false, nil } return true, nil - }, timeoutCtx.Done()) + }) if err != nil { p.Log.V(consts.LogLevelError).Error(err, "Error while waiting on node annotation update", diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go index bb94207488..dda154a7fb 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go @@ -23,6 +23,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -64,13 +65,16 @@ type ClusterUpgradeStateManagerImpl struct { opts StateOptions } -// NewClusterUpgradeStateManager creates a new instance of ClusterUpgradeStateManagerImpl +// NewClusterUpgradeStateManager creates a new instance of ClusterUpgradeStateManagerImpl. +// k8sClient is used for all controller-runtime reads and writes, and can be a cached client. +// Cache transforms need to preserve Node Spec.Unschedulable and the Ready condition. func NewClusterUpgradeStateManager( log logr.Logger, k8sConfig *rest.Config, + k8sClient client.Client, eventRecorder record.EventRecorder, opts StateOptions) (ClusterUpgradeStateManager, error) { - commonmanager, err := NewCommonUpgradeStateManager(log, k8sConfig, Scheme, eventRecorder) + commonmanager, err := NewCommonUpgradeStateManager(log, k8sConfig, k8sClient, eventRecorder) if err != nil { return nil, fmt.Errorf("failed to create commonmanager upgrade state manager. %v", err) } @@ -125,9 +129,10 @@ func (m *ClusterUpgradeStateManagerImpl) BuildState(ctx context.Context, namespa return nil, err } + podsByOwner := m.GetPodsByDSOwner(daemonSets, podList.Items) filteredPodList := []corev1.Pod{} for _, ds := range daemonSets { - dsPods := m.GetPodsOwnedbyDs(ds, podList.Items) + dsPods := podsByOwner[ds.UID] if int(ds.Status.DesiredNumberScheduled) != len(dsPods) { m.Log.V(consts.LogLevelInfo).Info("Driver DaemonSet has Unscheduled pods", "name", ds.Name) return nil, fmt.Errorf("driver DaemonSet should not have Unscheduled pods") @@ -136,17 +141,16 @@ func (m *ClusterUpgradeStateManagerImpl) BuildState(ctx context.Context, namespa } // Collect also orphaned driver pods - filteredPodList = append(filteredPodList, m.GetOrphanedPods(podList.Items)...) + filteredPodList = append(filteredPodList, podsByOwner[NoOwner]...) upgradeStateLabel := GetUpgradeStateLabelKey() for i := range filteredPodList { pod := &filteredPodList[i] var ownerDaemonSet *appsv1.DaemonSet - if IsOrphanedPod(pod) { - ownerDaemonSet = nil - } else { - ownerDaemonSet = daemonSets[pod.OwnerReferences[0].UID] + owner := metav1.GetControllerOfNoCopy(pod) + if owner != nil { + ownerDaemonSet = daemonSets[owner.UID] } // Check if pod is already scheduled to a Node if pod.Spec.NodeName == "" && pod.Status.Phase == corev1.PodPending { diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go index 1e2d8b84ab..6659535d12 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go @@ -161,7 +161,7 @@ func GetEventReason() string { // logEventf logs a formatted event for a given kubernetes object func logEventf(recorder record.EventRecorder, object runtime.Object, eventType string, reason string, messageFmt string, - args ...interface{}) { + args ...any) { if recorder != nil { recorder.Eventf(object, eventType, reason, messageFmt, args...) } diff --git a/vendor/modules.txt b/vendor/modules.txt index bf6bb60b46..f4e06a6c4a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/NVIDIA/go-nvlib/pkg/pciids # github.com/NVIDIA/k8s-kata-manager v0.2.3 ## explicit; go 1.23.0 github.com/NVIDIA/k8s-kata-manager/api/v1alpha1/config -# github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494 +# github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a ## explicit; go 1.26.0 github.com/NVIDIA/k8s-operator-libs/api/upgrade/v1alpha1 github.com/NVIDIA/k8s-operator-libs/pkg/consts