diff --git a/cmd/gpu-operator/main.go b/cmd/gpu-operator/main.go index 4578266f79..d26b64464f 100644 --- a/cmd/gpu-operator/main.go +++ b/cmd/gpu-operator/main.go @@ -173,6 +173,7 @@ func main() { clusterUpgradeStateManager, err := upgrade.NewClusterUpgradeStateManager( upgradeLogger, mgr.GetConfig(), + mgr.GetClient(), // nolint:staticcheck // TODO: update k8s-operator-libs to leverage events.EventRecorder instead mgr.GetEventRecorderFor("nvidia-gpu-operator"), diff --git a/controllers/cache/cache.go b/controllers/cache/cache.go index 2d51f103f6..3c503b72a9 100644 --- a/controllers/cache/cache.go +++ b/controllers/cache/cache.go @@ -58,6 +58,7 @@ func New(config *rest.Config, options ctrlcache.Options) (ctrlcache.Cache, error return ctrlcache.New(config, options) } +// stripUnusedNodeFields keeps only the Node fields that the operator reads. func stripUnusedNodeFields(in any) (any, error) { node, ok := in.(*corev1.Node) if !ok { @@ -65,10 +66,17 @@ func stripUnusedNodeFields(in any) (any, error) { } runtimeVersion := node.Status.NodeInfo.ContainerRuntimeVersion + var conditions []corev1.NodeCondition + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + conditions = append(conditions, corev1.NodeCondition{Type: condition.Type, Status: condition.Status}) + } + } node.ManagedFields = nil - node.Spec = corev1.NodeSpec{} + node.Spec = corev1.NodeSpec{Unschedulable: node.Spec.Unschedulable} node.Status = corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: runtimeVersion}, + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: runtimeVersion}, + Conditions: conditions, } return node, nil } @@ -85,8 +93,6 @@ func Options(operatorNamespace string) ctrlcache.Options { DefaultTransform: stripManagedFields, ByObject: map[client.Object]ctrlcache.ByObject{ &corev1.Node{}: { - // The operator reads only the container runtime version from node - // status. Drop all other status data from the cached object. Transform: stripUnusedNodeFields, }, &corev1.Pod{}: {Namespaces: cachedNamespaces}, diff --git a/controllers/cache/cache_test.go b/controllers/cache/cache_test.go index 616f572387..83ade85443 100644 --- a/controllers/cache/cache_test.go +++ b/controllers/cache/cache_test.go @@ -150,10 +150,16 @@ func TestCache(t *testing.T) { }, Spec: corev1.NodeSpec{Unschedulable: true}, Status: corev1.NodeStatus{ - Capacity: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, - Images: []corev1.ContainerImage{{Names: []string{"example.com/large-image:v1"}, SizeBytes: 1000000}}, - NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: "containerd://2.0.0", KernelVersion: "6.0.0"}, - Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, + Capacity: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("8")}, + Images: []corev1.ContainerImage{{Names: []string{"example.com/large-image:v1"}, SizeBytes: 1000000}}, + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: "containerd://2.0.0", KernelVersion: "6.0.0"}, + Conditions: []corev1.NodeCondition{ + {Type: corev1.NodeMemoryPressure, Status: corev1.ConditionFalse}, + { + Type: corev1.NodeReady, Status: corev1.ConditionTrue, Reason: "KubeletReady", + LastHeartbeatTime: metav1.NewTime(time.Unix(1700000000, 0)), + }, + }, }, } configMap := &corev1.ConfigMap{ @@ -313,9 +319,11 @@ func TestCache(t *testing.T) { require.NoError(t, cachedClient.Get(ctx, client.ObjectKey{Name: node.Name}, gotNode)) expectedNode := node.DeepCopy() expectedNode.ManagedFields = nil - expectedNode.Spec = corev1.NodeSpec{} + // The upgrade library needs Spec.Unschedulable and the Ready condition. + expectedNode.Spec = corev1.NodeSpec{Unschedulable: true} expectedNode.Status = corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: node.Status.NodeInfo.ContainerRuntimeVersion}, + NodeInfo: corev1.NodeSystemInfo{ContainerRuntimeVersion: node.Status.NodeInfo.ContainerRuntimeVersion}, + Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, } require.Equal(t, expectedNode.ObjectMeta, gotNode.ObjectMeta) require.Equal(t, expectedNode.Spec, gotNode.Spec) diff --git a/go.mod b/go.mod index b96ab121d3..beea97544d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/NVIDIA/go-nvlib v0.13.0 github.com/NVIDIA/k8s-kata-manager v0.2.3 - github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494 + github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a github.com/NVIDIA/nvidia-container-toolkit v1.20.2 github.com/cyphar/filepath-securejoin v0.7.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc diff --git a/go.sum b/go.sum index 2f38bc4ac5..54ae3e11ce 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/NVIDIA/go-nvlib v0.13.0 h1:veqetT7p2m+u7yPVAtXQ/nLAqotTdZF1+RzpLd1Yci github.com/NVIDIA/go-nvlib v0.13.0/go.mod h1:OmA7IhiBevXA4jUElFQjwmAZJLqrIAofyyn9MvRcz7s= github.com/NVIDIA/k8s-kata-manager v0.2.3 h1:d5+gRFqU5el/fKMXhHUaPY7haj+dbHL4nDsO/q05LBo= github.com/NVIDIA/k8s-kata-manager v0.2.3/go.mod h1:xx5OUiMsHyKbyX0JjKHqAftvqS8vx00LFn/5EaMdtB4= -github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494 h1:j+tWK79l9AouBulQps7rxILLhy2fWYcEhH4zgYjth/o= -github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494/go.mod h1:L+aiCiTKN63AX9SWz/F8pv9Jw9FIfI+dAEr7VA+KowE= +github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a h1:PJTguzSwWLxQYBw/QNlqaRECYaK+CuDmsWpO67SZkbQ= +github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a/go.mod h1:TfactzFEcVt03/Qi40SkwOawkVVTXfNZJvhkS7hzTaM= github.com/NVIDIA/nvidia-container-toolkit v1.20.2 h1:ndvx+2v4c8HmRf8A5aONf1+sE32jNDZ7qnIaBEAN5i8= github.com/NVIDIA/nvidia-container-toolkit v1.20.2/go.mod h1:q/O2jOSbK8JeHYhHMWwhuwof0yXzs7VE5uVjfSZriDw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go index 2a16d979e2..0a6d41c41d 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/common_manager.go @@ -8,7 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/runtime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -116,13 +116,8 @@ type CommonUpgradeManagerImpl struct { func NewCommonUpgradeStateManager( log logr.Logger, k8sConfig *rest.Config, - scheme *runtime.Scheme, + k8sClient client.Client, eventRecorder record.EventRecorder) (*CommonUpgradeManagerImpl, error) { - k8sClient, err := client.New(k8sConfig, client.Options{Scheme: scheme}) - if err != nil { - return &CommonUpgradeManagerImpl{}, fmt.Errorf("error creating k8s client: %v", err) - } - k8sInterface, err := kubernetes.NewForConfig(k8sConfig) if err != nil { return &CommonUpgradeManagerImpl{}, fmt.Errorf("error creating k8s interface: %v", err) @@ -201,23 +196,11 @@ func (m *CommonUpgradeManagerImpl) GetDriverDaemonSets(ctx context.Context, name // GetPodsOwnedbyDs returns a list of the pods owned by the specified DaemonSet func (m *CommonUpgradeManagerImpl) GetPodsOwnedbyDs(ds *appsv1.DaemonSet, pods []corev1.Pod) []corev1.Pod { - dsPodList := []corev1.Pod{} - for i := range pods { - pod := &pods[i] - if IsOrphanedPod(pod) { - m.Log.V(consts.LogLevelInfo).Info("Driver Pod has no owner DaemonSet", "pod", pod.Name) - continue - } - m.Log.V(consts.LogLevelInfo).Info("Pod", "pod", pod.Name, "owner", pod.OwnerReferences[0].Name) - - if ds.UID != pod.OwnerReferences[0].UID { - m.Log.V(consts.LogLevelInfo).Info("Driver Pod is not owned by an Driver DaemonSet", - "pod", pod, "actual owner", pod.OwnerReferences[0]) - continue - } - dsPodList = append(dsPodList, *pod) - } - return dsPodList + podsByOwner := m.GetPodsByDSOwner( + map[types.UID]*appsv1.DaemonSet{ds.UID: ds}, + pods, + ) + return podsByOwner[ds.UID] } // GetOrphanedPods returns a list of the pods not owned by any DaemonSet @@ -233,8 +216,38 @@ func (m *CommonUpgradeManagerImpl) GetOrphanedPods(pods []corev1.Pod) []corev1.P return podList } +// GetPodsByDSOwner indexes pods controlled by the supplied DaemonSets. +// Pods without a controller are indexed under NoOwner. +func (m *CommonUpgradeManagerImpl) GetPodsByDSOwner( + daemonSets map[types.UID]*appsv1.DaemonSet, + pods []corev1.Pod) map[types.UID][]corev1.Pod { + podsByOwner := make(map[types.UID][]corev1.Pod) + + for _, pod := range pods { + owner := metav1.GetControllerOfNoCopy(&pod) + if owner == nil { + m.Log.V(consts.LogLevelInfo).Info("Driver Pod has no owner DaemonSet", "pod", pod.Name) + podsByOwner[NoOwner] = append(podsByOwner[NoOwner], pod) + continue + } + + daemonSet, found := daemonSets[owner.UID] + if !found || daemonSet == nil || !metav1.IsControlledBy(&pod, daemonSet) { + m.Log.V(consts.LogLevelInfo).Info("Driver Pod is not owned by a Driver DaemonSet", + "pod", pod, "actual owner", owner) + continue + } + + m.Log.V(consts.LogLevelInfo).Info("Pod", "pod", pod.Name, "owner", owner.Name) + podsByOwner[daemonSet.UID] = append(podsByOwner[daemonSet.UID], pod) + } + + m.Log.V(consts.LogLevelInfo).Info("Total orphaned Pods found:", "count", len(podsByOwner[NoOwner])) + return podsByOwner +} + func IsOrphanedPod(pod *corev1.Pod) bool { - return len(pod.OwnerReferences) < 1 + return metav1.GetControllerOfNoCopy(pod) == nil } // ProcessDoneOrUnknownNodes iterates over UpgradeStateDone or UpgradeStateUnknown nodes and determines diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go index 064ef8978d..9bab1ba6ab 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/consts.go @@ -16,6 +16,8 @@ limitations under the License. package upgrade +import "k8s.io/apimachinery/pkg/types" + const ( // UpgradeStateLabelKeyFmt is the format of the node label key indicating driver upgrade states UpgradeStateLabelKeyFmt = "nvidia.com/%s-driver-upgrade-state" @@ -83,6 +85,8 @@ const ( ) const ( + // NoOwner identifies pods that do not have a controller. + NoOwner types.UID = "" // nodeNameFieldSelectorFmt is the format of a field selector that can be used in metav1.ListOptions to filter by // node nodeNameFieldSelectorFmt = "spec.nodeName=%s" diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go index 4e0a70cebe..5910cc68bb 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/drain_manager.go @@ -96,11 +96,6 @@ func (m *DrainManagerImpl) ScheduleNodesDrain(ctx context.Context, drainConfig * } for _, node := range drainConfig.Nodes { - // We need to shadow the loop variable or initialize some other one with its value - // to avoid concurrency issues when launching goroutines. - // If a loop variable is used as it is, all/most goroutines, spawned inside this loop, - // will use the 'node' value of the last item in drainConfig.Nodes - node := node if !m.drainingNodes.Has(node.Name) { m.log.V(consts.LogLevelInfo).Info("Schedule drain for node", "node", node.Name) logEvent(m.eventRecorder, node, corev1.EventTypeNormal, GetEventReason(), "Scheduling drain of the node") diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go index e6e1760baa..9ef41bd443 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/node_upgrade_state_provider.go @@ -28,6 +28,12 @@ import ( "github.com/NVIDIA/k8s-operator-libs/pkg/consts" ) +// Interval and timeout for reading a Node until a patch shows in the cache. +const ( + pollInterval = 10 * time.Millisecond + pollTimeout = 10 * time.Second +) + // NodeUpgradeStateProvider allows for synchronized operations on node objects and ensures that the node, // got from the provider, always has the up-to-date upgrade state type NodeUpgradeStateProvider interface { @@ -97,13 +103,10 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeState( // so we wait until it's synced. // That way, since only one call to reconcile at a time is allowed for upgrade controller, each new update // will have the updated node object in the cache. - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*10) - defer cancel() - //nolint:staticcheck - err = wait.PollImmediateUntil(time.Second, func() (bool, error) { + err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) { p.Log.V(consts.LogLevelDebug).Info("Requesting node object to see if operator cache has updated", "node", node.Name) - err := p.K8sClient.Get(timeoutCtx, types.NamespacedName{Name: node.Name}, node) + err := p.K8sClient.Get(ctx, types.NamespacedName{Name: node.Name}, node) if err != nil { return false, err } @@ -114,7 +117,7 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeState( return false, nil } return true, nil - }, timeoutCtx.Done()) + }) if err != nil { p.Log.V(consts.LogLevelError).Error(err, "Error while waiting on node label update", @@ -168,13 +171,10 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeAnnotation( // so we wait until it's synced. // That way, since only one call to reconcile at a time is allowed for upgrade controller, each new update // will have the updated node object in the cache. - timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*10) - defer cancel() - //nolint:staticcheck - err = wait.PollImmediateUntil(time.Second, func() (bool, error) { + err = wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, true, func(ctx context.Context) (bool, error) { p.Log.V(consts.LogLevelDebug).Info("Requesting node object to see if operator cache has updated", "node", node.Name) - err := p.K8sClient.Get(timeoutCtx, types.NamespacedName{Name: node.Name}, node) + err := p.K8sClient.Get(ctx, types.NamespacedName{Name: node.Name}, node) if err != nil { return false, err } @@ -194,7 +194,7 @@ func (p *NodeUpgradeStateProviderImpl) ChangeNodeUpgradeAnnotation( return false, nil } return true, nil - }, timeoutCtx.Done()) + }) if err != nil { p.Log.V(consts.LogLevelError).Error(err, "Error while waiting on node annotation update", diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go index bb94207488..dda154a7fb 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/upgrade_state.go @@ -23,6 +23,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -64,13 +65,16 @@ type ClusterUpgradeStateManagerImpl struct { opts StateOptions } -// NewClusterUpgradeStateManager creates a new instance of ClusterUpgradeStateManagerImpl +// NewClusterUpgradeStateManager creates a new instance of ClusterUpgradeStateManagerImpl. +// k8sClient is used for all controller-runtime reads and writes, and can be a cached client. +// Cache transforms need to preserve Node Spec.Unschedulable and the Ready condition. func NewClusterUpgradeStateManager( log logr.Logger, k8sConfig *rest.Config, + k8sClient client.Client, eventRecorder record.EventRecorder, opts StateOptions) (ClusterUpgradeStateManager, error) { - commonmanager, err := NewCommonUpgradeStateManager(log, k8sConfig, Scheme, eventRecorder) + commonmanager, err := NewCommonUpgradeStateManager(log, k8sConfig, k8sClient, eventRecorder) if err != nil { return nil, fmt.Errorf("failed to create commonmanager upgrade state manager. %v", err) } @@ -125,9 +129,10 @@ func (m *ClusterUpgradeStateManagerImpl) BuildState(ctx context.Context, namespa return nil, err } + podsByOwner := m.GetPodsByDSOwner(daemonSets, podList.Items) filteredPodList := []corev1.Pod{} for _, ds := range daemonSets { - dsPods := m.GetPodsOwnedbyDs(ds, podList.Items) + dsPods := podsByOwner[ds.UID] if int(ds.Status.DesiredNumberScheduled) != len(dsPods) { m.Log.V(consts.LogLevelInfo).Info("Driver DaemonSet has Unscheduled pods", "name", ds.Name) return nil, fmt.Errorf("driver DaemonSet should not have Unscheduled pods") @@ -136,17 +141,16 @@ func (m *ClusterUpgradeStateManagerImpl) BuildState(ctx context.Context, namespa } // Collect also orphaned driver pods - filteredPodList = append(filteredPodList, m.GetOrphanedPods(podList.Items)...) + filteredPodList = append(filteredPodList, podsByOwner[NoOwner]...) upgradeStateLabel := GetUpgradeStateLabelKey() for i := range filteredPodList { pod := &filteredPodList[i] var ownerDaemonSet *appsv1.DaemonSet - if IsOrphanedPod(pod) { - ownerDaemonSet = nil - } else { - ownerDaemonSet = daemonSets[pod.OwnerReferences[0].UID] + owner := metav1.GetControllerOfNoCopy(pod) + if owner != nil { + ownerDaemonSet = daemonSets[owner.UID] } // Check if pod is already scheduled to a Node if pod.Spec.NodeName == "" && pod.Status.Phase == corev1.PodPending { diff --git a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go index 1e2d8b84ab..6659535d12 100644 --- a/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go +++ b/vendor/github.com/NVIDIA/k8s-operator-libs/pkg/upgrade/util.go @@ -161,7 +161,7 @@ func GetEventReason() string { // logEventf logs a formatted event for a given kubernetes object func logEventf(recorder record.EventRecorder, object runtime.Object, eventType string, reason string, messageFmt string, - args ...interface{}) { + args ...any) { if recorder != nil { recorder.Eventf(object, eventType, reason, messageFmt, args...) } diff --git a/vendor/modules.txt b/vendor/modules.txt index bf6bb60b46..f4e06a6c4a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -36,7 +36,7 @@ github.com/NVIDIA/go-nvlib/pkg/pciids # github.com/NVIDIA/k8s-kata-manager v0.2.3 ## explicit; go 1.23.0 github.com/NVIDIA/k8s-kata-manager/api/v1alpha1/config -# github.com/NVIDIA/k8s-operator-libs v0.0.0-20260629200812-d720f2557494 +# github.com/NVIDIA/k8s-operator-libs v0.0.0-20261113163306-991fa4a9a29a ## explicit; go 1.26.0 github.com/NVIDIA/k8s-operator-libs/api/upgrade/v1alpha1 github.com/NVIDIA/k8s-operator-libs/pkg/consts