fix pod bind wrong card when there are other gpus

Signed-off-by: joy717 <joy717.xmu@gmail.com>

Author: jiayu.chen

internal/pkg/dcu/server.go

@@ -378,7 +378,7 @@ func (p *Plugin) createvdevFiles(current *corev1.Pod, ctr *corev1.Container, req
 	}
 	dirName := string(current.UID) + "_" + ctr.Name + "_" + fmt.Sprint(devidx) + "_" + fmt.Sprint(pipeid) + "_" + fmt.Sprint(vdevidx) + "_" + fmt.Sprint(coremsk1) + "_" + fmt.Sprint(coremsk2)
 	cacheFileHostDirectory := fmt.Sprintf("/usr/local/vgpu/dcu/%s", dirName)
-	err = createvdevFile(pcibusId, coremsk1, coremsk2, reqcores, mem, 0, vdevidx, pipeid, cacheFileHostDirectory, "vdev0.conf")
+	err = createvdevFile(pcibusId, coremsk1, coremsk2, reqcores, mem, devidx, vdevidx, pipeid, cacheFileHostDirectory, fmt.Sprintf("vdev%d.conf", vdevidx))
 	if err != nil {
 		return "", err
 	}
@@ -445,6 +445,11 @@ func (p *Plugin) Allocate(ctx context.Context, reqs *kubeletdevicepluginv1beta1.
 		nodelock.ReleaseNodeLock(nodename, NodeLockDCU)
 		return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
 	}
+	drmCards, drmRenders, err := util.ListDcuDrmDevices()
+	if err != nil {
+		util.PodAllocationFailed(nodename, current, NodeLockDCU)
+		return &kubeletdevicepluginv1beta1.AllocateResponse{}, err
+	}
 	for idx := range reqs.ContainerRequests {
 		currentCtr, devreq, err := util.GetNextDeviceRequest(util.HygonDCUDevice, *current)
 		klog.Infoln("deviceAllocateFromAnnotation=", devreq)
@@ -479,18 +484,33 @@ func (p *Plugin) Allocate(ctx context.Context, reqs *kubeletdevicepluginv1beta1.
 		car.Devices = append(car.Devices, dev)
 
 		for _, val := range devreq {
-			var id int
+			var devIdx = -1
 			klog.Infof("Allocating device ID: %s", val.UUID)
-			fmt.Sscanf(val.UUID, "DCU-%d", &id)
+			succeedCount, err := fmt.Sscanf(val.UUID, "DCU-%d", &devIdx)
+			if err != nil || succeedCount == 0 || devIdx == -1 {
+				klog.Errorf("Invalid request device uuid: %s", val.UUID)
+				util.PodAllocationFailed(nodename, current, NodeLockDCU)
+				return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("invalid request device uuid %s", val.UUID)
+			}
+
+			if devIdx > len(drmCards) || devIdx > len(drmRenders) {
+				klog.Errorf("Invalid device index: %d, all devices counts is: %d, all renders count is: %d", devIdx, len(drmCards), len(drmRenders))
+				util.PodAllocationFailed(nodename, current, NodeLockDCU)
+				return &kubeletdevicepluginv1beta1.AllocateResponse{}, fmt.Errorf("can not match dcu dri request %s. cards %d, renders %d", val.UUID, len(drmCards), len(drmRenders))
+			}
 
-			devpath := fmt.Sprintf("/dev/dri/card%d", id)
+			drmCardName := drmCards[devIdx]
+			klog.Infof("All dcu dri card devs: %v, mapped dri: %s", drmCards, drmCardName)
+			devpath := fmt.Sprintf("/dev/dri/%s", drmCardName)
 			dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
 			dev.HostPath = devpath
 			dev.ContainerPath = devpath
 			dev.Permissions = "rw"
 			car.Devices = append(car.Devices, dev)
 
-			devpath = fmt.Sprintf("/dev/dri/renderD%d", (id + 128))
+			drmRenderName := drmRenders[devIdx]
+			klog.Infof("All dcu dri render devs: %v, mapped dri: %s", drmRenders, drmRenderName)
+			devpath = fmt.Sprintf("/dev/dri/%s", drmRenderName)
 			dev = new(kubeletdevicepluginv1beta1.DeviceSpec)
 			dev.HostPath = devpath
 			dev.ContainerPath = devpath

internal/pkg/util/drm_slice.go

@@ -0,0 +1,34 @@
+package util
+
+import (
+	"fmt"
+	"strings"
+)
+
+// DrmSlice is a struct for sort to natural sorting, we assume that the element is a 'card%d'/'renderD%d' format.
+// Note: if you want a common natural sorting, you should NOT use this one.
+type DrmSlice []string
+
+func (p DrmSlice) Len() int           { return len(p) }
+func (p DrmSlice) Less(i, j int) bool { return DrmSortLess(p[i], p[j]) }
+func (p DrmSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func DrmSortLess(a, b string) bool {
+	cardA := strings.HasPrefix(a, DriPrefixCard)
+	cardB := strings.HasPrefix(b, DriPrefixCard)
+	if cardA && cardB {
+		var aid, bid int
+		fmt.Sscanf(a, "card%d", &aid)
+		fmt.Sscanf(b, "card%d", &bid)
+		return aid <= bid
+	}
+	renderA := strings.HasPrefix(a, DriPrefixRender)
+	renderB := strings.HasPrefix(b, DriPrefixRender)
+	if renderA && renderB {
+		var aid, bid int
+		fmt.Sscanf(a, "renderD%d", &aid)
+		fmt.Sscanf(b, "renderD%d", &bid)
+		return aid <= bid
+	}
+	return a <= b
+}

internal/pkg/util/types.go

@@ -42,6 +42,17 @@ const (
 
 	// NodeNameEnvName define env var name for use get node name.
 	NodeNameEnvName = "NODE_NAME"
+
+	//HygonVendorID is the pci vendor id for hygon
+	HygonVendorID = "0x1d94"
+
+	// LocationDri is the location of the dri
+	LocationDri = "/dev/dri"
+
+	//DriPrefixCard is the "card" file name prefix in /dev/dri. eg: /dev/dri/card0
+	DriPrefixCard = "card"
+	//DriPrefixRender is the "render" file name prefix in /dev/dri. eg: /dev/dri/renderD128
+	DriPrefixRender = "renderD"
 )
 
 type DevicePluginConfigs struct {

internal/pkg/util/util.go

@@ -22,6 +22,8 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -422,3 +424,48 @@ func PodAllocationFailed(nodeName string, pod *corev1.Pod, lockname string) {
 		klog.Errorf("release lock failed:%v", err.Error())
 	}
 }
+
+// ListDcuDrmDevices list all drm devices, and filter it by dcu vendorID
+func ListDcuDrmDevices() ([]string, []string, error) {
+	filenames := make([]string, 0)
+	dcuDrms := make([]string, 0)
+	dcuRenders := make([]string, 0)
+	files, err := os.ReadDir(LocationDri)
+	if err != nil {
+		klog.Errorf("ListDcuDrmDevices list dri directory error: %v", err.Error())
+		return nil, nil, err
+	}
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+		if strings.HasPrefix(f.Name(), DriPrefixCard) || strings.HasPrefix(f.Name(), DriPrefixRender) {
+			filenames = append(filenames, f.Name())
+		}
+	}
+	// we need the devices order correctly, sort it first
+	sort.Sort(DrmSlice(filenames))
+	for _, f := range filenames {
+		vendorID, err := os.ReadFile(fmt.Sprintf("/sys/class/drm/%s/device/vendor", f))
+		if err != nil {
+			klog.Errorf("ListDcuDrmDevices read vendor file error: %v", err.Error())
+			return nil, nil, err
+		}
+		fixedVendorID := strings.TrimSpace(string(vendorID))
+		if fixedVendorID != HygonVendorID {
+			klog.Infof("ListDcuDrmDevices dri dev %s vendorID %s is not dcu, skip it", f, fixedVendorID)
+			continue
+		}
+		if strings.HasPrefix(f, DriPrefixCard) {
+			dcuDrms = append(dcuDrms, f)
+		}
+		if strings.HasPrefix(f, DriPrefixRender) {
+			dcuRenders = append(dcuRenders, f)
+		}
+	}
+	if len(dcuDrms) != len(dcuRenders) {
+		return nil, nil, fmt.Errorf("ListDcuDrmDevices dcuDrms %v and dcuRenders %v length not equal", dcuDrms, dcuRenders)
+	}
+
+	return dcuDrms, dcuRenders, nil
+}