operator: Refactor and simplify volume mounts for config

Signed-off-by: Aaron Wilson <aawilson@nvidia.com>
NVIDIA · Aug 1, 2024 · 93e19ee · 93e19ee
1 parent 565ae10
commit 93e19ee
Show file tree

Hide file tree

Showing 12 changed files with 326 additions and 338 deletions.
diff --git a/operator/cmd/aisinit/main.go b/operator/cmd/aisinit/main.go
@@ -1,3 +1,7 @@
+// Package main contains logic for the aisinit container
+/*
+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+ */
 package main
 
 import (
@@ -88,6 +92,9 @@ func getMappedHostname(original, mapPath string) string {
 	return original
 }
 
+// AIS init is intended to be deployed inside the same pod as the aisnode container
+// The purpose of AIS init is to take a template config and output a result config based on the environment of the
+// deployed pod, with optional additional flags
 func main() {
 	var (
 		role                   string

diff --git a/operator/cmd/main.go b/operator/cmd/main.go
@@ -1,6 +1,6 @@
 // Package main contains logic for managing AIS operator manager
 /*
- * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved.
  */
 package main
 

diff --git a/operator/pkg/resources/cmn/configmap.go b/operator/pkg/resources/cmn/configmap.go
@@ -55,7 +55,7 @@ func NewGlobalCM(ais *aisv1.AIStore, toUpdate *aisv1.ConfigToUpdate) (*corev1.Co
 			Namespace: ais.Namespace,
 		},
 		Data: map[string]string{
-			aisGlobalConfigFileName: conf,
+			AISGlobalConfigName: conf,
 		},
 	}
 	if ais.Spec.HostnameMap != nil {
@@ -67,3 +67,7 @@ func NewGlobalCM(ais *aisv1.AIStore, toUpdate *aisv1.ConfigToUpdate) (*corev1.Co
 	}
 	return cm, nil
 }
+
+func AISConfigMapName(ais *aisv1.AIStore, daeType string) string {
+	return ais.Name + "-" + daeType
+}
diff --git a/operator/pkg/resources/cmn/res.go b/operator/pkg/resources/cmn/res.go
@@ -6,7 +6,6 @@ package cmn
 
 import (
 	"fmt"
-	"path"
 
 	aisapc "github.com/NVIDIA/aistore/api/apc"
 	aisv1 "github.com/ais-operator/api/v1beta1"
@@ -17,26 +16,6 @@ import (
 )
 
 const (
-	LogsDir = "/var/log/ais"
-
-	configVolume         = "config-mount"
-	configGlobalVolume   = "config-global"
-	configTemplateVolume = "config-template"
-	envVolume            = "env-mount"
-	stateVolume          = "state-mount"
-	awsSecretVolume      = "aws-creds"
-	gcpSecretVolume      = "gcp-creds" //nolint:gosec // This is not really credential.
-	tlsSecretVolume      = "tls-certs"
-	logsVolume           = "logs-dir"
-
-	configTemplateDir = "/var/ais_config_template"
-	globalConfigDir   = "/var/global_config"
-	aisConfigDir      = "/var/ais_config"
-
-	aisGlobalConfigFileName = "ais.json"
-	aisLocalConfigName      = "ais_local.json"
-	hostnameMapFileName     = "hostname_map.json"
-
 	// probe constants
 	// TODO: obtain probe specs from AIStore custom resource spec.
 	defaultProbePeriodSeconds        = 5
@@ -53,129 +32,6 @@ const (
 	probeReadinessEndpoint = probeLivenessEndpoint + "?readiness=true"
 )
 
-func NewAISVolumes(ais *aisv1.AIStore, daeType string) []corev1.Volume {
-	volumes := []corev1.Volume{
-		{
-			Name: configVolume,
-			VolumeSource: corev1.VolumeSource{
-				EmptyDir: &corev1.EmptyDirVolumeSource{},
-			},
-		},
-		{
-			Name: configTemplateVolume,
-			VolumeSource: corev1.VolumeSource{
-				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: ais.Name + "-" + daeType,
-					},
-				},
-			},
-		},
-		{
-			Name: configGlobalVolume,
-			VolumeSource: corev1.VolumeSource{
-				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: globalConfigMapName(ais),
-					},
-				},
-			},
-		},
-
-		{
-			Name: "statsd-config",
-			VolumeSource: corev1.VolumeSource{
-				ConfigMap: &corev1.ConfigMapVolumeSource{
-					LocalObjectReference: corev1.LocalObjectReference{
-						Name: ais.Name + "-statsd",
-					},
-				},
-			},
-		},
-		newLogsVolume(ais, daeType),
-	}
-
-	// Only create hostpath volumes if no storage class is provided for state
-	if ais.Spec.StateStorageClass == nil {
-		hostpathVolumes := []corev1.Volume{
-			{
-				Name: envVolume,
-				VolumeSource: corev1.VolumeSource{
-					HostPath: &corev1.HostPathVolumeSource{
-						//nolint:all
-						Path: path.Join(*ais.Spec.HostpathPrefix, ais.Namespace, ais.Name, daeType+"_env"),
-						Type: aisapc.Ptr(corev1.HostPathDirectoryOrCreate),
-					},
-				},
-			},
-			{
-				Name: stateVolume,
-				VolumeSource: corev1.VolumeSource{
-					HostPath: &corev1.HostPathVolumeSource{
-						//nolint:all
-						Path: path.Join(*ais.Spec.HostpathPrefix, ais.Namespace, ais.Name, daeType),
-						Type: aisapc.Ptr(corev1.HostPathDirectoryOrCreate),
-					},
-				},
-			},
-		}
-		volumes = append(volumes, hostpathVolumes...)
-	}
-
-	if ais.Spec.AWSSecretName != nil {
-		volumes = append(volumes, corev1.Volume{
-			Name: awsSecretVolume,
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: *ais.Spec.AWSSecretName,
-				},
-			},
-		})
-	}
-	if ais.Spec.GCPSecretName != nil {
-		volumes = append(volumes, corev1.Volume{
-			Name: gcpSecretVolume,
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: *ais.Spec.GCPSecretName,
-				},
-			},
-		})
-	}
-
-	if ais.Spec.TLSSecretName != nil {
-		volumes = append(volumes, corev1.Volume{
-			Name: tlsSecretVolume,
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: *ais.Spec.TLSSecretName,
-				},
-			},
-		})
-	}
-	return volumes
-}
-
-func newLogsVolume(ais *aisv1.AIStore, daeType string) corev1.Volume {
-	if ais.Spec.LogsDirectory != "" {
-		return corev1.Volume{
-			Name: logsVolume,
-			VolumeSource: corev1.VolumeSource{
-				HostPath: &corev1.HostPathVolumeSource{
-					Path: path.Join(ais.Spec.LogsDirectory, ais.Namespace, ais.Name, daeType),
-					Type: aisapc.Ptr(corev1.HostPathDirectoryOrCreate),
-				},
-			},
-		}
-	}
-	return corev1.Volume{
-		Name: logsVolume,
-		VolumeSource: corev1.VolumeSource{
-			EmptyDir: &corev1.EmptyDirVolumeSource{},
-		},
-	}
-}
-
 func newHTTPProbeHandle(ais *aisv1.AIStore, daemonRole, probeEndpoint string) corev1.ProbeHandler {
 	var (
 		httpPort  intstr.IntOrString
@@ -236,143 +92,6 @@ func NewStartupProbe(ais *aisv1.AIStore, daemonRole string) *corev1.Probe {
 	}
 }
 
-func NewAISVolumeMounts(ais *aisv1.AIStore, daeType string) []corev1.VolumeMount {
-	spec := ais.Spec
-	volumeMounts := []corev1.VolumeMount{
-		{
-			Name:      configVolume,
-			MountPath: aisConfigDir,
-		},
-		{
-			Name:      configGlobalVolume,
-			MountPath: path.Join(aisConfigDir, aisGlobalConfigFileName),
-			SubPath:   aisGlobalConfigFileName,
-		},
-		{
-			Name:      "statsd-config",
-			MountPath: "/var/statsd_config",
-		},
-		newLogsVolumeMount(daeType),
-	}
-
-	hostMountSubPath := getHostMountSubPath(daeType)
-	if spec.StateStorageClass != nil {
-		volumeName := getPVCPrefix(ais) + "state"
-		dynamicMounts := []corev1.VolumeMount{
-			{
-				Name:      volumeName,
-				MountPath: "/var/ais_env",
-				SubPath:   "env/",
-			},
-			{
-				Name:      volumeName,
-				MountPath: "/etc/ais",
-				SubPath:   "state/",
-			},
-		}
-		volumeMounts = append(volumeMounts, dynamicMounts...)
-	} else {
-		hostMounts := []corev1.VolumeMount{
-			{
-				Name:        envVolume,
-				MountPath:   "/var/ais_env",
-				SubPathExpr: hostMountSubPath,
-			},
-			{
-				Name:        stateVolume,
-				MountPath:   "/etc/ais",
-				SubPathExpr: hostMountSubPath,
-			},
-		}
-		volumeMounts = append(volumeMounts, hostMounts...)
-	}
-
-	if spec.AWSSecretName != nil {
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      awsSecretVolume,
-			ReadOnly:  true,
-			MountPath: "/root/.aws",
-		})
-	}
-	if spec.GCPSecretName != nil {
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      gcpSecretVolume,
-			ReadOnly:  true,
-			MountPath: "/var/gcp",
-		})
-	}
-	if spec.TLSSecretName != nil {
-		volumeMounts = append(volumeMounts, corev1.VolumeMount{
-			Name:      tlsSecretVolume,
-			ReadOnly:  true,
-			MountPath: "/var/certs",
-		})
-	}
-
-	return volumeMounts
-}
-
-func newLogsVolumeMount(daeType string) corev1.VolumeMount {
-	return corev1.VolumeMount{
-		Name:        logsVolume,
-		MountPath:   LogsDir,
-		SubPathExpr: getHostMountSubPath(daeType),
-	}
-}
-
-func NewInitContainerArgs(daeType string, hostnameMap map[string]string) []string {
-	args := []string{
-		"-role=" + daeType,
-		"-local_config_template=" + path.Join(configTemplateDir, aisLocalConfigName),
-		"-output_local_config=" + path.Join(aisConfigDir, aisLocalConfigName),
-	}
-	if len(hostnameMap) != 0 {
-		args = append(args, "-hostname_map_file="+path.Join(globalConfigDir, hostnameMapFileName))
-	}
-	return args
-}
-
-func NewInitVolumeMounts(ais *aisv1.AIStore, daeType string) []corev1.VolumeMount {
-	hostMountSubPath := getHostMountSubPath(daeType)
-
-	volumeMounts := []corev1.VolumeMount{
-		{
-			Name:      configVolume,
-			MountPath: aisConfigDir,
-		},
-		{
-			Name:      configTemplateVolume,
-			MountPath: configTemplateDir,
-		},
-		{
-			Name:      configGlobalVolume,
-			MountPath: globalConfigDir,
-		},
-	}
-
-	if ais.Spec.StateStorageClass != nil {
-		dynamicMounts := []corev1.VolumeMount{
-			{
-				Name:      getPVCPrefix(ais) + "state",
-				MountPath: "/var/ais_env",
-				SubPath:   "env/",
-			},
-		}
-		volumeMounts = append(volumeMounts, dynamicMounts...)
-	} else {
-		hostMounts := []corev1.VolumeMount{
-			{
-				Name:        envVolume,
-				MountPath:   "/var/ais_env",
-				SubPathExpr: hostMountSubPath,
-			},
-		}
-		volumeMounts = append(volumeMounts, hostMounts...)
-	}
-
-	return volumeMounts
-}
-
 func NewDaemonPorts(spec *aisv1.DaemonSpec) []corev1.ContainerPort {
 	var hostPort int32
 	if spec.HostPort != nil {
@@ -418,24 +137,16 @@ func createPodAntiAffinity(podLabels map[string]string) *corev1.PodAntiAffinity
 	}
 }
 
-func getHostMountSubPath(daeType string) string {
-	// Always use the pod name as sub path for targets, since target pods are bound to specific nodes
-	if daeType == aisapc.Target {
-		return "$(MY_POD)"
-	}
-	return ""
-}
-
 // Generate PVC claim ref for a specific namespace and cluster
-func getPVCPrefix(ais *aisv1.AIStore) string {
-	return fmt.Sprintf("%s-%s-", ais.Namespace, ais.Name)
+func getStatePVCName(ais *aisv1.AIStore) string {
+	return fmt.Sprintf("%s-%s-%s", ais.Namespace, ais.Name, "state")
 }
 
 // DefineStatePVC Define a PVC to use for pod state using dynamically configured volumes
 func DefineStatePVC(ais *aisv1.AIStore, storageClass *string) *corev1.PersistentVolumeClaim {
 	return &corev1.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: getPVCPrefix(ais) + "state",
+			Name: getStatePVCName(ais),
 		},
 		Spec: corev1.PersistentVolumeClaimSpec{
 			AccessModes: []corev1.PersistentVolumeAccessMode{