fix pod bind wrong card when there are other gpus

fix pod bind wrong card when there are other gpus

Signed-off-by: joy717 <joy717.xmu@gmail.com>

Author: jiayu.chen

internal/pkg/dcu/server.go

@@ -445,6 +445,11 @@ func (p *Plugin) Allocate(ctx context.Context, reqs *kubeletdevicepluginv1beta1.
 		nodelock.ReleaseNodeLock(nodename, NodeLockDCU)
 		return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
 	}
+	drms, err := util.ListDcuDrmDevices()
+	if err != nil {
+		util.PodAllocationFailed(nodename, current, NodeLockDCU)
+		return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
+	}
 	for idx := range reqs.ContainerRequests {
 		currentCtr, devreq, err := util.GetNextDeviceRequest(util.HygonDCUDevice, *current)
 		klog.Infoln("deviceAllocateFromAnnotation=", devreq)
@@ -479,9 +484,24 @@ func (p *Plugin) Allocate(ctx context.Context, reqs *kubeletdevicepluginv1beta1.
 		car.Devices = append(car.Devices, dev)
 
 		for _, val := range devreq {
-			var id int
+			var devIdx = -1
 			klog.Infof("Allocating device ID: %s", val.UUID)
-			fmt.Sscanf(val.UUID, "DCU-%d", &id)
+			succeedCount, err := fmt.Sscanf(val.UUID, "DCU-%d", &devIdx)
+			if err != nil || succeedCount == 0 || devIdx == -1 {
+				klog.Errorf("Invalid request device uuid: %s", val.UUID)
+				util.PodAllocationFailed(nodename, current, NodeLockDCU)
+				return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("invalid request device uuid %s", val.UUID)
+			}
+
+			if devIdx > len(drms) {
+				klog.Errorf("Invalid device index: %d, all devices counts are: %d", devIdx, len(drms))
+				util.PodAllocationFailed(nodename, current, NodeLockDCU)
+				return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("cant find dcu dri for %d", devIdx)
+			}
+
+			drmName := drms[devIdx]
+			var id int
+			fmt.Sscanf(drmName, "card%d", &id)
 
 			devpath := fmt.Sprintf("/dev/dri/card%d", id)
 			dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
@@ -490,7 +510,7 @@ func (p *Plugin) Allocate(ctx context.Context, reqs *kubeletdevicepluginv1beta1.
 			dev.Permissions = "rw"
 			car.Devices = append(car.Devices, dev)
 
-			devpath = fmt.Sprintf("/dev/dri/renderD%d", (id + 128))
+			devpath = fmt.Sprintf("/dev/dri/renderD%d", id+128)
 			dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
 			dev.HostPath = devpath
 			dev.ContainerPath = devpath

internal/pkg/util/drm_slice.go

@@ -0,0 +1,17 @@
+package util
+
+import "fmt"
+
+// DrmSlice is a struct for sort to natual sorting, we assume that the element is a 'card%d' format
+type DrmSlice []string
+
+func (p DrmSlice) Len() int           { return len(p) }
+func (p DrmSlice) Less(i, j int) bool { return DrmSortLess(p[i], p[j]) }
+func (p DrmSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func DrmSortLess(a, b string) bool {
+	var aid, bid int
+	fmt.Sscanf(a, "card%d", &aid)
+	fmt.Sscanf(b, "card%d", &bid)
+	return aid <= bid
+}

internal/pkg/util/types.go

@@ -42,6 +42,12 @@ const (
 
 	// NodeNameEnvName define env var name for use get node name.
 	NodeNameEnvName = "NODE_NAME"
+
+	//HygonVendorID is the pci vendor id for hygon
+	HygonVendorID = "0x1d94"
+
+	// LocationDri is the location of the dri
+	LocationDri = "/dev/dri"
 )
 
 type DevicePluginConfigs struct {

internal/pkg/util/util.go

@@ -22,6 +22,8 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -422,3 +424,33 @@ func PodAllocationFailed(nodeName string, pod *corev1.Pod, lockname string) {
 		klog.Errorf("release lock failed:%v", err.Error())
 	}
 }
+
+// ListDcuDrmDevices list all drm devices, and filter it by dcu vendorID
+func ListDcuDrmDevices() ([]string, error) {
+	filenames := make([]string, 0)
+	dcuDrms := make([]string, 0)
+	files, err := os.ReadDir(LocationDri)
+	if err != nil {
+		klog.Errorf("ListDcuDrmDevices list dri directory error: %v", err.Error())
+		return nil, err
+	}
+	for _, f := range files {
+		if !f.IsDir() && strings.HasPrefix(f.Name(), "card") {
+			filenames = append(filenames, f.Name())
+		}
+	}
+	// we need the devices order correctly, sort it first
+	sort.Sort(DrmSlice(filenames))
+	for _, f := range filenames {
+		vendorID, err := os.ReadFile(fmt.Sprintf("/sys/class/drm/%s/device/vendor", f))
+		if err != nil {
+			klog.Errorf("ListDcuDrmDevices read vendor file error: %v", err.Error())
+			return nil, err
+		}
+		fixedVendorID := strings.TrimSpace(string(vendorID))
+		if fixedVendorID == HygonVendorID {
+			dcuDrms = append(dcuDrms, f)
+		}
+	}
+	return dcuDrms, nil
+}