Support IB network type

README.md

@@ -53,9 +53,10 @@ Then, edit **cluster.yml**:
 name: "open3fs"
 workDir: "/opt/3fs"
 # networkType configure the network type of the cluster, can be one of the following:
-# - RDMA: use RDMA network protocol
+# -    IB: use InfiniBand network protocol
+# -  RDMA: use RDMA network protocol
 # - ERDMA: use aliyun ERDMA as RDMA network protocol
-# - RXE: use linux rxe kernel module to mock RDMA network protocol
+# -   RXE: use Linux rxe kernel module to mock RDMA network protocol
 networkType: "ERDMA"
 nodes:
   - name: node1

cluster.yml.sample

@@ -7,9 +7,10 @@ name: "open3fs"
 # workDir is the work dir for all 3fs services
 workDir: "/opt/3fs"
 # networkType configure the network type of the cluster, can be one of the following:
-# - RDMA: use RDMA network protocol
+# -    IB: use InfiniBand network protocol
+# -  RDMA: use RDMA network protocol
 # - ERDMA: use aliyun ERDMA as RDMA network protocol
-# - RXE: use linux rxe kernel module to mock RDMA network protocol
+# -   RXE: use Linux rxe kernel module to mock RDMA network protocol
 networkType: "RDMA"
 # Set the log level of 3fs services.
 # Supported values: "DEBUG", "INFO", "WARN", "ERROR", "FATAL"

pkg/config/config.go

@@ -29,12 +29,13 @@ type NetworkType string
 
 // defines network types
 const (
+	NetworkTypeIB    NetworkType = "IB"
 	NetworkTypeRDMA  NetworkType = "RDMA"
 	NetworkTypeRXE   NetworkType = "RXE"
 	NetworkTypeERDMA NetworkType = "ERDMA"
 )
 
-var networkTypes = utils.NewSet(NetworkTypeRDMA, NetworkTypeRXE, NetworkTypeERDMA)
+var networkTypes = utils.NewSet(NetworkTypeIB, NetworkTypeRDMA, NetworkTypeRXE, NetworkTypeERDMA)
 
 // DiskType is the type of disk definition
 type DiskType string

pkg/mgmtd/steps.go

@@ -88,7 +88,8 @@ func (s *genAdminCliConfigStep) Execute(ctx context.Context) error {
 	port := strconv.Itoa(s.Runtime.Services.Mgmtd.RDMAListenPort)
 	for i, nodeName := range s.Runtime.Services.Mgmtd.Nodes {
 		node := s.Runtime.Nodes[nodeName]
-		mgmtdServerAddresses[i] = fmt.Sprintf(`"RDMA://%s"`, net.JoinHostPort(node.Host, port))
+		mgmtdServerAddresses[i] = fmt.Sprintf(`"%s://%s"`,
+			s.Runtime.MgmtdProtocol, net.JoinHostPort(node.Host, port))
 	}
 	mgmtdServerAddressesStr := fmt.Sprintf("[%s]", strings.Join(mgmtdServerAddresses, ","))
 	s.Runtime.Store(task.RuntimeMgmtdServerAddressesKey, mgmtdServerAddressesStr)

pkg/mgmtd/templates/mgmtd_main.toml.tmpl

@@ -80,7 +80,7 @@ proc_thread_pool_stratetry = 'SHARED_QUEUE'
 [[server.base.groups]]
 check_connections_interval = '1min'
 connection_expiration_time = '1day'
-network_type = 'RDMA'
+network_type = '{{ .MgmtdProtocol }}'
 services = [ 'Mgmtd' ]
 use_independent_thread_pool = false
 

pkg/task/runner.go

@@ -52,6 +52,13 @@ type Runtime struct {
 	WorkDir   string
 	LocalEm   *external.Manager
 	LocalNode *config.Node
+
+	// MgmtdProtocol is used to set the protocol of mgmtd address.
+	// It maps RDMA types to RDMA://
+	// It maps IB types to IPoIB://
+	// Currently, only mgmtd address uses IPoIB protocol, all other services still use RDMA protocol.
+	// TODO: Find the reason from 3FS code base.
+	MgmtdProtocol string
 }
 
 // LoadString load string value form sync map
@@ -96,6 +103,10 @@ type Runner struct {
 // Init initializes all tasks.
 func (r *Runner) Init() {
 	r.Runtime = &Runtime{Cfg: r.cfg, WorkDir: r.cfg.WorkDir, LocalNode: r.localNode}
+	r.Runtime.MgmtdProtocol = "RDMA"
+	if r.cfg.NetworkType == config.NetworkTypeIB {
+		r.Runtime.MgmtdProtocol = "IPoIB"
+	}
 	r.Runtime.Nodes = make(map[string]config.Node, len(r.cfg.Nodes))
 	for _, node := range r.cfg.Nodes {
 		r.Runtime.Nodes[node.Name] = node

pkg/task/runner_test.go

@@ -50,6 +50,13 @@ func (s *runnerSuite) TestInit() {
 	s.mockTask.AssertExpectations(s.T())
 }
 
+func (s *runnerSuite) TestInitWithIB() {
+	s.runner.cfg.NetworkType = config.NetworkTypeIB
+	s.TestInit()
+
+	s.Equal(s.runner.Runtime.MgmtdProtocol, "IPoIB")
+}
+
 func (s *runnerSuite) TestRegisterAfterInit() {
 	s.TestInit()
 	s.mockTask.On("Name").Return("mockTask")

pkg/task/steps/3fs_steps.go

@@ -206,6 +206,7 @@ func (s *prepare3FSConfigStep) genConfigs(tmpDir string) error {
 		"RDMAListenPort":       s.rdmaListenPort,
 		"TCPListenPort":        s.tcpListenPort,
 		"MgmtdServerAddresses": mgmtdServerAddresses,
+		"MgmtdProtocol":        s.Runtime.MgmtdProtocol,
 	}
 	for k, v := range s.extraMainTomlData {
 		mainTmplData[k] = v

pkg/task/task.go

@@ -211,6 +211,10 @@ func (s *BaseStep) GetErdmaSoPath(ctx context.Context) error {
 // GetRdmaVolumes returns the volumes need mapped to container for the rdma network.
 func (s *BaseStep) GetRdmaVolumes() []*external.VolumeArgs {
 	volumes := []*external.VolumeArgs{}
+	if s.Runtime.Cfg.NetworkType == config.NetworkTypeIB {
+		return volumes
+	}
+
 	if s.Runtime.Cfg.NetworkType != config.NetworkTypeRDMA {
 		ibdev2netdevScriptPath := path.Join(s.Runtime.Cfg.WorkDir, "bin", "ibdev2netdev")
 		volumes = append(volumes, &external.VolumeArgs{