DRA kubelet: adapt to v1alpha3 API

This adds the ability to select specific requests inside a claim for a
container.

NodePrepareResources is always called, even if the claim is not used by any
container. This could be useful for drivers where that call has some effect
other than injecting CDI device IDs into containers. It also ensures that
drivers can validate configs.

Because of a faulty E2E test, kubelet was told to contact the wrong driver for
a claim. This was not visible in the kubelet log output. Now changes to the
claim info cache are getting logged. While at it, naming of variables and some
existing log output gets harmonized.

Co-authored-by: Oksana Baranova <[email protected]>
Co-authored-by: Ed Bartosh <[email protected]>

Author: 3 people

pkg/kubelet/cm/container_manager_linux.go

@@ -661,10 +661,6 @@ func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Containe
 		if err != nil {
 			return nil, err
 		}
-		// NOTE: Passing CDI device names as annotations is a temporary solution
-		// It will be removed after all runtimes are updated
-		// to get CDI device names from the ContainerConfig.CDIDevices field
-		opts.Annotations = append(opts.Annotations, resOpts.Annotations...)
 		opts.CDIDevices = append(opts.CDIDevices, resOpts.CDIDevices...)
 	}
 	// Allocate should already be called during predicateAdmitHandler.Admit(),
@@ -969,15 +965,18 @@ func (cm *containerManagerImpl) GetDynamicResources(pod *v1.Pod, container *v1.C
 		// a set of CDIDevices from a different kubelet plugin. In the future we may want to
 		// include the name of the kubelet plugin and/or other types of resources that are
 		// not CDIDevices (assuming the DRAmanager supports this).
-		for _, klPluginCdiDevices := range containerClaimInfo.CDIDevices {
+		for /* driverName */ _, driverState := range containerClaimInfo.Drivers {
 			var cdiDevices []*podresourcesapi.CDIDevice
-			for _, cdiDevice := range klPluginCdiDevices {
-				cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDevice})
+			for /*requestName */ _, requestDevices := range driverState.CDIDevices {
+				for _, cdiDevice := range requestDevices {
+					cdiDevices = append(cdiDevices, &podresourcesapi.CDIDevice{Name: cdiDevice})
+				}
 			}
 			claimResources = append(claimResources, &podresourcesapi.ClaimResource{CDIDevices: cdiDevices})
 		}
 		containerDynamicResource := podresourcesapi.DynamicResource{
-			ClassName:      containerClaimInfo.ClassName,
+			// As of 1.31, ResourceClaims are not associated with a single class anymore.
+			// ClassName: ???
 			ClaimName:      containerClaimInfo.ClaimName,
 			ClaimNamespace: containerClaimInfo.Namespace,
 			ClaimResources: claimResources,

pkg/kubelet/cm/devicemanager/topology_hints_test.go

@@ -440,7 +440,6 @@ func TestTopologyAlignedAllocation(t *testing.T) {
 				opts: &pluginapi.DevicePluginOptions{GetPreferredAllocationAvailable: true},
 			}
 		}
-
 		allocated, err := m.devicesToAllocate("podUID", "containerName", tc.resource, tc.request, sets.New[string]())
 		if err != nil {
 			t.Errorf("Unexpected error: %v", err)

pkg/kubelet/cm/dra/claiminfo.go

@@ -17,26 +17,22 @@ limitations under the License.
 package dra
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 
 	resourceapi "k8s.io/api/resource/v1alpha3"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/kubelet/cm/dra/state"
-	"k8s.io/kubernetes/pkg/kubelet/cm/util/cdi"
-	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 )
 
 // ClaimInfo holds information required
 // to prepare and unprepare a resource claim.
 // +k8s:deepcopy-gen=true
 type ClaimInfo struct {
 	state.ClaimInfoState
-	// annotations is a mapping of container annotations per DRA plugin associated with
-	// a prepared resource
-	annotations map[string][]kubecontainer.Annotation
-	prepared    bool
+	prepared bool
 }
 
 // claimInfoCache is a cache of processed resource claims keyed by namespace/claimname.
@@ -47,89 +43,50 @@ type claimInfoCache struct {
 }
 
 // newClaimInfoFromClaim creates a new claim info from a resource claim.
-func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) *ClaimInfo {
-	// Grab the allocation.resourceHandles. If there are no
-	// allocation.resourceHandles, create a single resourceHandle with no
-	// content. This will trigger processing of this claim by a single
-	// kubelet plugin whose name matches resourceClaim.Status.DriverName.
-	resourceHandles := claim.Status.Allocation.ResourceHandles
-	if len(resourceHandles) == 0 {
-		resourceHandles = make([]resourceapi.ResourceHandle, 1)
-	}
+// It verifies that the kubelet can handle the claim.
+func newClaimInfoFromClaim(claim *resourceapi.ResourceClaim) (*ClaimInfo, error) {
 	claimInfoState := state.ClaimInfoState{
-		DriverName:      claim.Status.DriverName,
-		ClassName:       claim.Spec.ResourceClassName,
-		ClaimUID:        claim.UID,
-		ClaimName:       claim.Name,
-		Namespace:       claim.Namespace,
-		PodUIDs:         sets.New[string](),
-		ResourceHandles: resourceHandles,
-		CDIDevices:      make(map[string][]string),
+		ClaimUID:  claim.UID,
+		ClaimName: claim.Name,
+		Namespace: claim.Namespace,
+		PodUIDs:   sets.New[string](),
+		Drivers:   make(map[string]state.DriverState),
+	}
+	if claim.Status.Allocation == nil {
+		return nil, errors.New("not allocated")
+	}
+	for _, result := range claim.Status.Allocation.Devices.Results {
+		claimInfoState.Drivers[result.Driver] = state.DriverState{
+			CDIDevices: make(map[string][]string),
+		}
 	}
 	info := &ClaimInfo{
 		ClaimInfoState: claimInfoState,
-		annotations:    make(map[string][]kubecontainer.Annotation),
 		prepared:       false,
 	}
-	return info
+	return info, nil
 }
 
 // newClaimInfoFromClaim creates a new claim info from a checkpointed claim info state object.
 func newClaimInfoFromState(state *state.ClaimInfoState) *ClaimInfo {
 	info := &ClaimInfo{
 		ClaimInfoState: *state.DeepCopy(),
-		annotations:    make(map[string][]kubecontainer.Annotation),
 		prepared:       false,
 	}
-	for pluginName, devices := range info.CDIDevices {
-		annotations, _ := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, devices)
-		info.annotations[pluginName] = append(info.annotations[pluginName], annotations...)
-	}
 	return info
 }
 
 // setCDIDevices adds a set of CDI devices to the claim info.
-func (info *ClaimInfo) setCDIDevices(pluginName string, cdiDevices []string) error {
-	// NOTE: Passing CDI device names as annotations is a temporary solution
-	// It will be removed after all runtimes are updated
-	// to get CDI device names from the ContainerConfig.CDIDevices field
-	annotations, err := cdi.GenerateAnnotations(info.ClaimUID, info.DriverName, cdiDevices)
-	if err != nil {
-		return fmt.Errorf("failed to generate container annotations, err: %+v", err)
-	}
-
-	if info.CDIDevices == nil {
-		info.CDIDevices = make(map[string][]string)
-	}
-
-	if info.annotations == nil {
-		info.annotations = make(map[string][]kubecontainer.Annotation)
+func (info *ClaimInfo) setCDIDevices(driverName string, requestName string, cdiDevices []string) {
+	if info.Drivers == nil {
+		info.Drivers = make(map[string]state.DriverState)
 	}
-
-	info.CDIDevices[pluginName] = cdiDevices
-	info.annotations[pluginName] = annotations
-
-	return nil
-}
-
-// annotationsAsList returns container annotations as a single list.
-func (info *ClaimInfo) annotationsAsList() []kubecontainer.Annotation {
-	var lst []kubecontainer.Annotation
-	for _, v := range info.annotations {
-		lst = append(lst, v...)
-	}
-	return lst
-}
-
-// cdiDevicesAsList returns a list of CDIDevices from the provided claim info.
-func (info *ClaimInfo) cdiDevicesAsList() []kubecontainer.CDIDevice {
-	var cdiDevices []kubecontainer.CDIDevice
-	for _, devices := range info.CDIDevices {
-		for _, device := range devices {
-			cdiDevices = append(cdiDevices, kubecontainer.CDIDevice{Name: device})
-		}
+	driverState := info.Drivers[driverName]
+	if driverState.CDIDevices == nil {
+		driverState.CDIDevices = make(map[string][]string)
 	}
-	return cdiDevices
+	driverState.CDIDevices[requestName] = cdiDevices
+	info.Drivers[driverName] = driverState
 }
 
 // addPodReference adds a pod reference to the claim info.