From 93e19eebfbf2d35d69ea06bbf7abab3904128488 Mon Sep 17 00:00:00 2001 From: Aaron Wilson Date: Wed, 31 Jul 2024 17:04:44 -0500 Subject: [PATCH] operator: Refactor and simplify volume mounts for config Signed-off-by: Aaron Wilson --- operator/cmd/aisinit/main.go | 7 + operator/cmd/main.go | 2 +- operator/pkg/resources/cmn/configmap.go | 6 +- operator/pkg/resources/cmn/res.go | 295 +------------------ operator/pkg/resources/cmn/statefulset.go | 26 ++ operator/pkg/resources/cmn/volumes.go | 251 ++++++++++++++++ operator/pkg/resources/proxy/configmap.go | 12 +- operator/pkg/resources/proxy/statefulset.go | 19 +- operator/pkg/resources/statsd/configmap.go | 10 +- operator/pkg/resources/target/configmap.go | 14 +- operator/pkg/resources/target/statefulset.go | 18 +- operator/tests/tutils/ais_cluster.go | 4 +- 12 files changed, 326 insertions(+), 338 deletions(-) create mode 100644 operator/pkg/resources/cmn/volumes.go diff --git a/operator/cmd/aisinit/main.go b/operator/cmd/aisinit/main.go index b65aeb8d..7f5daecc 100644 --- a/operator/cmd/aisinit/main.go +++ b/operator/cmd/aisinit/main.go @@ -1,3 +1,7 @@ +// Package main contains logic for the aisinit container +/* + * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + */ package main import ( @@ -88,6 +92,9 @@ func getMappedHostname(original, mapPath string) string { return original } +// AIS init is intended to be deployed inside the same pod as the aisnode container +// The purpose of AIS init is to take a template config and output a result config based on the environment of the +// deployed pod, with optional additional flags func main() { var ( role string diff --git a/operator/cmd/main.go b/operator/cmd/main.go index da1ff683..c35ba4f5 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -1,6 +1,6 @@ // Package main contains logic for managing AIS operator manager /* - * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. */ package main diff --git a/operator/pkg/resources/cmn/configmap.go b/operator/pkg/resources/cmn/configmap.go index 4eca8a7f..fdd2d304 100644 --- a/operator/pkg/resources/cmn/configmap.go +++ b/operator/pkg/resources/cmn/configmap.go @@ -55,7 +55,7 @@ func NewGlobalCM(ais *aisv1.AIStore, toUpdate *aisv1.ConfigToUpdate) (*corev1.Co Namespace: ais.Namespace, }, Data: map[string]string{ - aisGlobalConfigFileName: conf, + AISGlobalConfigName: conf, }, } if ais.Spec.HostnameMap != nil { @@ -67,3 +67,7 @@ func NewGlobalCM(ais *aisv1.AIStore, toUpdate *aisv1.ConfigToUpdate) (*corev1.Co } return cm, nil } + +func AISConfigMapName(ais *aisv1.AIStore, daeType string) string { + return ais.Name + "-" + daeType +} diff --git a/operator/pkg/resources/cmn/res.go b/operator/pkg/resources/cmn/res.go index 6cf243d8..177ceb10 100644 --- a/operator/pkg/resources/cmn/res.go +++ b/operator/pkg/resources/cmn/res.go @@ -6,7 +6,6 @@ package cmn import ( "fmt" - "path" aisapc "github.com/NVIDIA/aistore/api/apc" aisv1 "github.com/ais-operator/api/v1beta1" @@ -17,26 +16,6 @@ import ( ) const ( - LogsDir = "/var/log/ais" - - configVolume = "config-mount" - configGlobalVolume = "config-global" - configTemplateVolume = "config-template" - envVolume = "env-mount" - stateVolume = "state-mount" - awsSecretVolume = "aws-creds" - gcpSecretVolume = "gcp-creds" //nolint:gosec // This is not really credential. - tlsSecretVolume = "tls-certs" - logsVolume = "logs-dir" - - configTemplateDir = "/var/ais_config_template" - globalConfigDir = "/var/global_config" - aisConfigDir = "/var/ais_config" - - aisGlobalConfigFileName = "ais.json" - aisLocalConfigName = "ais_local.json" - hostnameMapFileName = "hostname_map.json" - // probe constants // TODO: obtain probe specs from AIStore custom resource spec. defaultProbePeriodSeconds = 5 @@ -53,129 +32,6 @@ const ( probeReadinessEndpoint = probeLivenessEndpoint + "?readiness=true" ) -func NewAISVolumes(ais *aisv1.AIStore, daeType string) []corev1.Volume { - volumes := []corev1.Volume{ - { - Name: configVolume, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: configTemplateVolume, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: ais.Name + "-" + daeType, - }, - }, - }, - }, - { - Name: configGlobalVolume, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: globalConfigMapName(ais), - }, - }, - }, - }, - - { - Name: "statsd-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: ais.Name + "-statsd", - }, - }, - }, - }, - newLogsVolume(ais, daeType), - } - - // Only create hostpath volumes if no storage class is provided for state - if ais.Spec.StateStorageClass == nil { - hostpathVolumes := []corev1.Volume{ - { - Name: envVolume, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - //nolint:all - Path: path.Join(*ais.Spec.HostpathPrefix, ais.Namespace, ais.Name, daeType+"_env"), - Type: aisapc.Ptr(corev1.HostPathDirectoryOrCreate), - }, - }, - }, - { - Name: stateVolume, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - //nolint:all - Path: path.Join(*ais.Spec.HostpathPrefix, ais.Namespace, ais.Name, daeType), - Type: aisapc.Ptr(corev1.HostPathDirectoryOrCreate), - }, - }, - }, - } - volumes = append(volumes, hostpathVolumes...) - } - - if ais.Spec.AWSSecretName != nil { - volumes = append(volumes, corev1.Volume{ - Name: awsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: *ais.Spec.AWSSecretName, - }, - }, - }) - } - if ais.Spec.GCPSecretName != nil { - volumes = append(volumes, corev1.Volume{ - Name: gcpSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: *ais.Spec.GCPSecretName, - }, - }, - }) - } - - if ais.Spec.TLSSecretName != nil { - volumes = append(volumes, corev1.Volume{ - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: *ais.Spec.TLSSecretName, - }, - }, - }) - } - return volumes -} - -func newLogsVolume(ais *aisv1.AIStore, daeType string) corev1.Volume { - if ais.Spec.LogsDirectory != "" { - return corev1.Volume{ - Name: logsVolume, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: path.Join(ais.Spec.LogsDirectory, ais.Namespace, ais.Name, daeType), - Type: aisapc.Ptr(corev1.HostPathDirectoryOrCreate), - }, - }, - } - } - return corev1.Volume{ - Name: logsVolume, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - } -} - func newHTTPProbeHandle(ais *aisv1.AIStore, daemonRole, probeEndpoint string) corev1.ProbeHandler { var ( httpPort intstr.IntOrString @@ -236,143 +92,6 @@ func NewStartupProbe(ais *aisv1.AIStore, daemonRole string) *corev1.Probe { } } -func NewAISVolumeMounts(ais *aisv1.AIStore, daeType string) []corev1.VolumeMount { - spec := ais.Spec - volumeMounts := []corev1.VolumeMount{ - { - Name: configVolume, - MountPath: aisConfigDir, - }, - { - Name: configGlobalVolume, - MountPath: path.Join(aisConfigDir, aisGlobalConfigFileName), - SubPath: aisGlobalConfigFileName, - }, - { - Name: "statsd-config", - MountPath: "/var/statsd_config", - }, - newLogsVolumeMount(daeType), - } - - hostMountSubPath := getHostMountSubPath(daeType) - if spec.StateStorageClass != nil { - volumeName := getPVCPrefix(ais) + "state" - dynamicMounts := []corev1.VolumeMount{ - { - Name: volumeName, - MountPath: "/var/ais_env", - SubPath: "env/", - }, - { - Name: volumeName, - MountPath: "/etc/ais", - SubPath: "state/", - }, - } - volumeMounts = append(volumeMounts, dynamicMounts...) - } else { - hostMounts := []corev1.VolumeMount{ - { - Name: envVolume, - MountPath: "/var/ais_env", - SubPathExpr: hostMountSubPath, - }, - { - Name: stateVolume, - MountPath: "/etc/ais", - SubPathExpr: hostMountSubPath, - }, - } - volumeMounts = append(volumeMounts, hostMounts...) - } - - if spec.AWSSecretName != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: awsSecretVolume, - ReadOnly: true, - MountPath: "/root/.aws", - }) - } - if spec.GCPSecretName != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: gcpSecretVolume, - ReadOnly: true, - MountPath: "/var/gcp", - }) - } - if spec.TLSSecretName != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: "/var/certs", - }) - } - - return volumeMounts -} - -func newLogsVolumeMount(daeType string) corev1.VolumeMount { - return corev1.VolumeMount{ - Name: logsVolume, - MountPath: LogsDir, - SubPathExpr: getHostMountSubPath(daeType), - } -} - -func NewInitContainerArgs(daeType string, hostnameMap map[string]string) []string { - args := []string{ - "-role=" + daeType, - "-local_config_template=" + path.Join(configTemplateDir, aisLocalConfigName), - "-output_local_config=" + path.Join(aisConfigDir, aisLocalConfigName), - } - if len(hostnameMap) != 0 { - args = append(args, "-hostname_map_file="+path.Join(globalConfigDir, hostnameMapFileName)) - } - return args -} - -func NewInitVolumeMounts(ais *aisv1.AIStore, daeType string) []corev1.VolumeMount { - hostMountSubPath := getHostMountSubPath(daeType) - - volumeMounts := []corev1.VolumeMount{ - { - Name: configVolume, - MountPath: aisConfigDir, - }, - { - Name: configTemplateVolume, - MountPath: configTemplateDir, - }, - { - Name: configGlobalVolume, - MountPath: globalConfigDir, - }, - } - - if ais.Spec.StateStorageClass != nil { - dynamicMounts := []corev1.VolumeMount{ - { - Name: getPVCPrefix(ais) + "state", - MountPath: "/var/ais_env", - SubPath: "env/", - }, - } - volumeMounts = append(volumeMounts, dynamicMounts...) - } else { - hostMounts := []corev1.VolumeMount{ - { - Name: envVolume, - MountPath: "/var/ais_env", - SubPathExpr: hostMountSubPath, - }, - } - volumeMounts = append(volumeMounts, hostMounts...) - } - - return volumeMounts -} - func NewDaemonPorts(spec *aisv1.DaemonSpec) []corev1.ContainerPort { var hostPort int32 if spec.HostPort != nil { @@ -418,24 +137,16 @@ func createPodAntiAffinity(podLabels map[string]string) *corev1.PodAntiAffinity } } -func getHostMountSubPath(daeType string) string { - // Always use the pod name as sub path for targets, since target pods are bound to specific nodes - if daeType == aisapc.Target { - return "$(MY_POD)" - } - return "" -} - // Generate PVC claim ref for a specific namespace and cluster -func getPVCPrefix(ais *aisv1.AIStore) string { - return fmt.Sprintf("%s-%s-", ais.Namespace, ais.Name) +func getStatePVCName(ais *aisv1.AIStore) string { + return fmt.Sprintf("%s-%s-%s", ais.Namespace, ais.Name, "state") } // DefineStatePVC Define a PVC to use for pod state using dynamically configured volumes func DefineStatePVC(ais *aisv1.AIStore, storageClass *string) *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: getPVCPrefix(ais) + "state", + Name: getStatePVCName(ais), }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{ diff --git a/operator/pkg/resources/cmn/statefulset.go b/operator/pkg/resources/cmn/statefulset.go index 7b98cc22..31fd10ab 100644 --- a/operator/pkg/resources/cmn/statefulset.go +++ b/operator/pkg/resources/cmn/statefulset.go @@ -6,8 +6,10 @@ package cmn import ( "fmt" + "path" "path/filepath" + aisapc "github.com/NVIDIA/aistore/api/apc" aisv1 "github.com/ais-operator/api/v1beta1" nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" corev1 "k8s.io/api/core/v1" @@ -34,3 +36,27 @@ func NewLogSidecar(daeType string) corev1.Container { Env: []corev1.EnvVar{EnvFromFieldPath(EnvPodName, "metadata.name")}, } } + +func NewInitContainerArgs(daeType string, hostnameMap map[string]string) []string { + args := []string{ + "-role=" + daeType, + "-local_config_template=" + path.Join(InitConfTemplateDir, AISLocalConfigName), + "-output_local_config=" + path.Join(AisConfigDir, AISLocalConfigName), + } + if len(hostnameMap) != 0 { + args = append(args, "-hostname_map_file="+path.Join(initGlobalConfDir, hostnameMapFileName)) + } + return args +} + +func NewAISContainerArgs(ais *aisv1.AIStore, daeType string) []string { + args := []string{ + "-config=" + path.Join(AisConfigDir, AISGlobalConfigName), + "-local_config=" + path.Join(AisConfigDir, AISLocalConfigName), + "-role=" + daeType, + } + if daeType == aisapc.Proxy { + args = append(args, fmt.Sprintf("-ntargets=%d", ais.GetTargetSize())) + } + return args +} diff --git a/operator/pkg/resources/cmn/volumes.go b/operator/pkg/resources/cmn/volumes.go new file mode 100644 index 00000000..c9fab43e --- /dev/null +++ b/operator/pkg/resources/cmn/volumes.go @@ -0,0 +1,251 @@ +// Package cmn provides utilities for common AIS cluster resources +/* + * Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. + */ +package cmn + +import ( + "path" + + aisapc "github.com/NVIDIA/aistore/api/apc" + "github.com/ais-operator/api/v1beta1" + "github.com/ais-operator/pkg/resources/statsd" + v1 "k8s.io/api/core/v1" +) + +const ( + // StateDir Container-internal location of configs and current state of the aisnode + StateDir = "/etc/ais" + // InitConfTemplateDir Container-internal location of config template, mounted from the config map + InitConfTemplateDir = "/var/ais_config_template" + // AisConfigDir Container-internal location of initial config, written by init container and used at aisnode start + AisConfigDir = "/var/ais_config" + LogsDir = "/var/log/ais" + StatsDDir = "/var/statsd_config" + initGlobalConfDir = "/var/global_config" + + hostnameMapFileName = "hostname_map.json" + AISGlobalConfigName = "ais.json" + AISLocalConfigName = "ais_local.json" + + StatsDVolume = "statsd-config" + configTemplateVolume = "config-template" + configVolume = "config-mount" + configGlobalVolume = "config-global" + stateVolume = "state-mount" + awsSecretVolume = "aws-creds" + gcpSecretVolume = "gcp-creds" //nolint:gosec // This is not really credential. + tlsSecretVolume = "tls-certs" + logsVolume = "logs-dir" +) + +func NewAISVolumes(ais *v1beta1.AIStore, daeType string) []v1.Volume { + volumes := []v1.Volume{ + { + Name: configTemplateVolume, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: AISConfigMapName(ais, daeType), + }, + }, + }, + }, + { + Name: configVolume, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + { + Name: configGlobalVolume, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: globalConfigMapName(ais), + }, + }, + }, + }, + { + Name: StatsDVolume, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: statsd.ConfigMapName(ais), + }, + }, + }, + }, + newLogsVolume(ais, daeType), + } + + // Only create hostpath volumes if no storage class is provided for state + if ais.Spec.StateStorageClass == nil { + hostpathVolumes := []v1.Volume{ + { + Name: stateVolume, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + //nolint:all + Path: path.Join(*ais.Spec.HostpathPrefix, ais.Namespace, ais.Name, daeType), + Type: aisapc.Ptr(v1.HostPathDirectoryOrCreate), + }, + }, + }, + } + volumes = append(volumes, hostpathVolumes...) + } + + if ais.Spec.AWSSecretName != nil { + volumes = append(volumes, v1.Volume{ + Name: awsSecretVolume, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: *ais.Spec.AWSSecretName, + }, + }, + }) + } + if ais.Spec.GCPSecretName != nil { + volumes = append(volumes, v1.Volume{ + Name: gcpSecretVolume, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: *ais.Spec.GCPSecretName, + }, + }, + }) + } + + if ais.Spec.TLSSecretName != nil { + volumes = append(volumes, v1.Volume{ + Name: tlsSecretVolume, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: *ais.Spec.TLSSecretName, + }, + }, + }) + } + return volumes +} + +func newLogsVolume(ais *v1beta1.AIStore, daeType string) v1.Volume { + if ais.Spec.LogsDirectory != "" { + return v1.Volume{ + Name: logsVolume, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: path.Join(ais.Spec.LogsDirectory, ais.Namespace, ais.Name, daeType), + Type: aisapc.Ptr(v1.HostPathDirectoryOrCreate), + }, + }, + } + } + return v1.Volume{ + Name: logsVolume, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + } +} + +func NewAISVolumeMounts(ais *v1beta1.AIStore, daeType string) []v1.VolumeMount { + spec := ais.Spec + volumeMounts := []v1.VolumeMount{ + { + Name: configVolume, + MountPath: AisConfigDir, + }, + { + Name: configGlobalVolume, + MountPath: path.Join(AisConfigDir, AISGlobalConfigName), + SubPath: AISGlobalConfigName, + }, + { + Name: StatsDVolume, + MountPath: StatsDDir, + }, + newLogsVolumeMount(daeType), + } + + if spec.StateStorageClass != nil { + volumeName := getStatePVCName(ais) + dynamicMounts := []v1.VolumeMount{ + { + Name: volumeName, + MountPath: StateDir, + }, + } + volumeMounts = append(volumeMounts, dynamicMounts...) + } else { + hostMountSubPath := getHostMountSubPath(daeType) + hostMounts := []v1.VolumeMount{ + { + Name: stateVolume, + MountPath: StateDir, + SubPathExpr: hostMountSubPath, + }, + } + volumeMounts = append(volumeMounts, hostMounts...) + } + + if spec.AWSSecretName != nil { + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: awsSecretVolume, + ReadOnly: true, + MountPath: "/root/.aws", + }) + } + if spec.GCPSecretName != nil { + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: gcpSecretVolume, + ReadOnly: true, + MountPath: "/var/gcp", + }) + } + if spec.TLSSecretName != nil { + volumeMounts = append(volumeMounts, v1.VolumeMount{ + Name: tlsSecretVolume, + ReadOnly: true, + MountPath: "/var/certs", + }) + } + + return volumeMounts +} + +func newLogsVolumeMount(daeType string) v1.VolumeMount { + return v1.VolumeMount{ + Name: logsVolume, + MountPath: LogsDir, + SubPathExpr: getHostMountSubPath(daeType), + } +} + +func NewInitVolumeMounts() []v1.VolumeMount { + volumeMounts := []v1.VolumeMount{ + { + Name: configTemplateVolume, + MountPath: InitConfTemplateDir, + }, + { + Name: configVolume, + MountPath: AisConfigDir, + }, + { + Name: configGlobalVolume, + MountPath: initGlobalConfDir, + }, + } + return volumeMounts +} + +func getHostMountSubPath(daeType string) string { + // Always use the pod name as sub path for targets, since target pods are bound to specific nodes + if daeType == aisapc.Target { + return "$(MY_POD)" + } + return "" +} diff --git a/operator/pkg/resources/proxy/configmap.go b/operator/pkg/resources/proxy/configmap.go index 78f79c76..293c58b5 100644 --- a/operator/pkg/resources/proxy/configmap.go +++ b/operator/pkg/resources/proxy/configmap.go @@ -15,13 +15,9 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func configMapName(ais *aisv1.AIStore) string { - return ais.Name + "-" + aisapc.Proxy -} - func ConfigMapNSName(ais *aisv1.AIStore) types.NamespacedName { return types.NamespacedName{ - Name: configMapName(ais), + Name: cmn.AISConfigMapName(ais, aisapc.Proxy), Namespace: ais.Namespace, } } @@ -34,18 +30,18 @@ func NewProxyCM(ais *aisv1.AIStore) (*corev1.ConfigMap, error) { } return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(ais), + Name: cmn.AISConfigMapName(ais, aisapc.Proxy), Namespace: ais.Namespace, }, Data: map[string]string{ - "ais_local.json": confLocal, + cmn.AISLocalConfigName: confLocal, }, }, nil } func localConfTemplate(spec *aisv1.ServiceSpec) aiscmn.LocalConfig { return aiscmn.LocalConfig{ - ConfigDir: "/etc/ais", + ConfigDir: cmn.StateDir, LogDir: cmn.LogsDir, HostNet: aiscmn.LocalNetConfig{ Hostname: "${AIS_PUBLIC_HOSTNAME}", diff --git a/operator/pkg/resources/proxy/statefulset.go b/operator/pkg/resources/proxy/statefulset.go index 092f7087..7ae9276c 100644 --- a/operator/pkg/resources/proxy/statefulset.go +++ b/operator/pkg/resources/proxy/statefulset.go @@ -6,12 +6,14 @@ package proxy import ( "fmt" + "path" "strconv" aisapc "github.com/NVIDIA/aistore/api/apc" "github.com/NVIDIA/aistore/api/env" aisv1 "github.com/ais-operator/api/v1beta1" "github.com/ais-operator/pkg/resources/cmn" + "github.com/ais-operator/pkg/resources/statsd" apiv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -104,7 +106,7 @@ func proxyPodSpec(ais *aisv1.AIStore) corev1.PodSpec { cmn.EnvFromValue(cmn.EnvDefaultPrimaryPod, ais.DefaultPrimaryName()), }, optionals...), Args: cmn.NewInitContainerArgs(aisapc.Proxy, ais.Spec.HostnameMap), - VolumeMounts: cmn.NewInitVolumeMounts(ais, aisapc.Proxy), + VolumeMounts: cmn.NewInitVolumeMounts(), }, }, Containers: []corev1.Container{ @@ -113,21 +115,16 @@ func proxyPodSpec(ais *aisv1.AIStore) corev1.PodSpec { Image: ais.Spec.NodeImage, ImagePullPolicy: corev1.PullAlways, Command: []string{"aisnode"}, - Args: []string{ - "-config=/var/ais_config/ais.json", - "-local_config=/var/ais_config/ais_local.json", - fmt.Sprintf("-ntargets=%d", ais.GetTargetSize()), - "-role=" + aisapc.Proxy, - }, + Args: cmn.NewAISContainerArgs(ais, aisapc.Proxy), Env: append([]corev1.EnvVar{ cmn.EnvFromFieldPath(cmn.EnvPodName, "metadata.name"), cmn.EnvFromValue(cmn.EnvNS, ais.Namespace), cmn.EnvFromValue(cmn.EnvClusterDomain, ais.GetClusterDomain()), - cmn.EnvFromValue(cmn.EnvShutdownMarkerPath, "/var/ais_config"), + cmn.EnvFromValue(cmn.EnvShutdownMarkerPath, cmn.AisConfigDir), cmn.EnvFromValue(cmn.EnvCIDR, ""), // TODO: Should take from specs - cmn.EnvFromValue(cmn.EnvConfigFilePath, "/var/ais_config/ais.json"), - cmn.EnvFromValue(cmn.EnvLocalConfigFilePath, "/var/ais_config/ais_local.json"), - cmn.EnvFromValue(cmn.EnvStatsDConfig, "/var/statsd_config/statsd.json"), + cmn.EnvFromValue(cmn.EnvConfigFilePath, path.Join(cmn.AisConfigDir, cmn.AISGlobalConfigName)), + cmn.EnvFromValue(cmn.EnvLocalConfigFilePath, path.Join(cmn.AisConfigDir, cmn.AISLocalConfigName)), + cmn.EnvFromValue(cmn.EnvStatsDConfig, path.Join(cmn.StatsDDir, statsd.ConfigFile)), cmn.EnvFromValue(cmn.EnvEnablePrometheus, strconv.FormatBool(ais.Spec.EnablePromExporter != nil && *ais.Spec.EnablePromExporter)), cmn.EnvFromValue(cmn.EnvDaemonRole, aisapc.Proxy), diff --git a/operator/pkg/resources/statsd/configmap.go b/operator/pkg/resources/statsd/configmap.go index b7eb4fe6..b91c9c1d 100644 --- a/operator/pkg/resources/statsd/configmap.go +++ b/operator/pkg/resources/statsd/configmap.go @@ -11,13 +11,15 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func configMapName(ais *aisv1.AIStore) string { +const ConfigFile = "statsd.json" + +func ConfigMapName(ais *aisv1.AIStore) string { return ais.Name + "-statsd" } func ConfigMapNSName(ais *aisv1.AIStore) types.NamespacedName { return types.NamespacedName{ - Name: configMapName(ais), + Name: ConfigMapName(ais), Namespace: ais.Namespace, } } @@ -25,11 +27,11 @@ func ConfigMapNSName(ais *aisv1.AIStore) types.NamespacedName { func NewStatsDCM(ais *aisv1.AIStore) *corev1.ConfigMap { return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: ais.Name + "-statsd", + Name: ConfigMapName(ais), Namespace: ais.Namespace, }, Data: map[string]string{ - "statsd.json": `{ + ConfigFile: `{ "graphiteHost": "", "graphitePort": 2003 }`, diff --git a/operator/pkg/resources/target/configmap.go b/operator/pkg/resources/target/configmap.go index 70d0161c..56fa2499 100644 --- a/operator/pkg/resources/target/configmap.go +++ b/operator/pkg/resources/target/configmap.go @@ -32,13 +32,9 @@ type ( } ) -func configMapName(ais *aisv1.AIStore) string { - return ais.Name + "-" + aisapc.Target -} - func ConfigMapNSName(ais *aisv1.AIStore) types.NamespacedName { return types.NamespacedName{ - Name: configMapName(ais), + Name: cmn.AISConfigMapName(ais, aisapc.Target), Namespace: ais.Namespace, } } @@ -50,11 +46,11 @@ func NewTargetCM(ctx context.Context, ais *aisv1.AIStore) (*corev1.ConfigMap, er } return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: configMapName(ais), + Name: cmn.AISConfigMapName(ais, aisapc.Target), Namespace: ais.Namespace, }, Data: map[string]string{ - "ais_local.json": localConfStr, + cmn.AISLocalConfigName: localConfStr, }, }, nil } @@ -78,7 +74,7 @@ func buildLocalConf(ctx context.Context, spec *aisv1.AIStoreSpec) (string, error func templateLocalConf(ctx context.Context, spec *aisv1.AIStoreSpec, netConfig *aiscmn.LocalNetConfig) aiscmn.LocalConfig { localConf := aiscmn.LocalConfig{ - ConfigDir: "/etc/ais", + ConfigDir: cmn.StateDir, LogDir: cmn.LogsDir, HostNet: *netConfig, } @@ -90,7 +86,7 @@ func templateLocalConf(ctx context.Context, spec *aisv1.AIStoreSpec, netConfig * func templateOldLocalConf(spec *aisv1.AIStoreSpec, netConfig *aiscmn.LocalNetConfig) v322LocalConfig { localConf := v322LocalConfig{ - ConfigDir: "/etc/ais", + ConfigDir: cmn.StateDir, LogDir: cmn.LogsDir, HostNet: *netConfig, } diff --git a/operator/pkg/resources/target/statefulset.go b/operator/pkg/resources/target/statefulset.go index 5182948f..52133fae 100644 --- a/operator/pkg/resources/target/statefulset.go +++ b/operator/pkg/resources/target/statefulset.go @@ -6,6 +6,7 @@ package target import ( "log" + "path" "strconv" "strings" @@ -13,6 +14,7 @@ import ( aisv1 "github.com/ais-operator/api/v1beta1" "github.com/ais-operator/pkg/resources/cmn" "github.com/ais-operator/pkg/resources/proxy" + "github.com/ais-operator/pkg/resources/statsd" "gopkg.in/inf.v0" apiv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -108,7 +110,7 @@ func NewTargetSS(ais *aisv1.AIStore) *apiv1.StatefulSet { cmn.EnvFromValue(cmn.EnvProxyServicePort, ais.Spec.ProxySpec.ServicePort.String()), }, optionals...), Args: cmn.NewInitContainerArgs(aisapc.Target, ais.Spec.HostnameMap), - VolumeMounts: cmn.NewInitVolumeMounts(ais, aisapc.Target), + VolumeMounts: cmn.NewInitVolumeMounts(), }, }, Containers: []corev1.Container{ @@ -117,20 +119,16 @@ func NewTargetSS(ais *aisv1.AIStore) *apiv1.StatefulSet { Image: ais.Spec.NodeImage, ImagePullPolicy: corev1.PullAlways, Command: []string{"aisnode"}, - Args: []string{ - "-config=/var/ais_config/ais.json", - "-local_config=/var/ais_config/ais_local.json", - "-role=" + aisapc.Target, - }, + Args: cmn.NewAISContainerArgs(ais, aisapc.Target), Env: append([]corev1.EnvVar{ cmn.EnvFromFieldPath(cmn.EnvPodName, "metadata.name"), cmn.EnvFromValue(cmn.EnvClusterDomain, ais.GetClusterDomain()), cmn.EnvFromValue(cmn.EnvNS, ais.Namespace), cmn.EnvFromValue(cmn.EnvCIDR, ""), // TODO: add - cmn.EnvFromValue(cmn.EnvConfigFilePath, "/var/ais_config/ais.json"), - cmn.EnvFromValue(cmn.EnvShutdownMarkerPath, "/var/ais_config"), - cmn.EnvFromValue(cmn.EnvLocalConfigFilePath, "/var/ais_config/ais_local.json"), - cmn.EnvFromValue(cmn.EnvStatsDConfig, "/var/statsd_config/statsd.json"), + cmn.EnvFromValue(cmn.EnvConfigFilePath, path.Join(cmn.AisConfigDir, cmn.AISGlobalConfigName)), + cmn.EnvFromValue(cmn.EnvShutdownMarkerPath, cmn.AisConfigDir), + cmn.EnvFromValue(cmn.EnvLocalConfigFilePath, path.Join(cmn.AisConfigDir, cmn.AISLocalConfigName)), + cmn.EnvFromValue(cmn.EnvStatsDConfig, path.Join(cmn.StatsDDir, statsd.ConfigFile)), cmn.EnvFromValue(cmn.EnvDaemonRole, aisapc.Target), cmn.EnvFromValue(cmn.EnvEnablePrometheus, strconv.FormatBool(ais.Spec.EnablePromExporter != nil && *ais.Spec.EnablePromExporter)), diff --git a/operator/tests/tutils/ais_cluster.go b/operator/tests/tutils/ais_cluster.go index cbc34f48..2b88f698 100644 --- a/operator/tests/tutils/ais_cluster.go +++ b/operator/tests/tutils/ais_cluster.go @@ -21,8 +21,8 @@ import ( // TODO: Should be provided from test config. const ( - aisNodeImage = "aistorage/aisnode:v3.24-abffbe3" - aisInitImage = "aistorage/ais-init:v1.1.2-rc-test-init" + aisNodeImage = "aistorage/aisnode:v3.24-rc2" + aisInitImage = "aistorage/ais-init:v1.2.0" ) type (