From 1b671c306e61bebd2c61207347cec76604bc37f2 Mon Sep 17 00:00:00 2001 From: Prashanth Dintyala Date: Mon, 13 Dec 2021 17:46:23 -0800 Subject: [PATCH] use camel case in field name; document new allowSharedNoDisks option Signed-off-by: Prashanth Dintyala --- operator/README.md | 21 +++- operator/api/v1beta1/aistore_types.go | 2 +- .../crd/bases/ais.nvidia.com_aistores.yaml | 110 ++++++++++++------ .../config/samples/ais_v1beta1_aistore.yaml | 11 +- 4 files changed, 104 insertions(+), 40 deletions(-) diff --git a/operator/README.md b/operator/README.md index 9d455234..ad38be83 100644 --- a/operator/README.md +++ b/operator/README.md @@ -16,7 +16,7 @@ To deploy AIS operator on an existing K8s cluster, execute the following command AIS operator employs [admission webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/) to enforce the validity of the managed AIS cluster. AIS operator runs a webhook server with `tls` enabled, responsible for validating each AIS cluster resource being created or updated. [Operator-SDK](https://sdk.operatorframework.io/) recommends using [cert-manager](https://github.com/jetstack/cert-manager) for provisioning the certificates required by the webhook server, however, any solution which can provide certificates to the AIS operator pod at location `/tmp/k8s-webhook-server/serving-certs/tls.(crt/key)`, should work. - + For quick deployment, the `deploy` command provides an option to deploy a basic version of `cert-manager`. However, for more advanced deployments it's recommended to follow [cert-manager documentation](https://cert-manager.io/docs/installation/kubernetes/). ```console @@ -75,6 +75,25 @@ $ minikube tunnel For more information and details on *minikube tunneling*, please see [this link](https://minikube.sigs.K8s.io/docs/commands/tunnel/). +### Deploying cluster with shared or no disks + +For development/testing K8s setup where the `mountpaths` attached to the storage targets pods are not block devices, i.e. have no disks, or share the disk, will result in the target pods to fail with `has no disks` or `filesystem sharing is not allowed` error. +To deploy AIStore cluster on such K8s environments is possible by setting the `allowSharedNoDisks` property to `true`, as follows: + +```yaml +# config/samples/ais_v1beta1_sample.yaml +apiVersion: ais.nvidia.com/v1beta1 +kind: AIStore +metadata: + name: aistore-sample +spec: + size: 4 # > number of K8s nodes + allowSharedNoDisks: true +... +``` + +> **WARNING:** It is NOT recommended to set the `allowSharedNoDisks` property to `true` for production deployments. + ### Locally testing multi-node AIS cluster diff --git a/operator/api/v1beta1/aistore_types.go b/operator/api/v1beta1/aistore_types.go index 86aa0833..3fa18a5a 100644 --- a/operator/api/v1beta1/aistore_types.go +++ b/operator/api/v1beta1/aistore_types.go @@ -147,7 +147,7 @@ type TargetSpec struct { Mounts []Mount `json:"mounts"` // AllowSharedOrNoDisks - disables FsID and mountpath disks validation on target nodes. NOT recommended for production deployments // +optional - AllowSharedOrNoDisks *bool `json:"allow_shared_no_disks,omitempty"` + AllowSharedOrNoDisks *bool `json:"allowSharedNoDisks,omitempty"` } type Mount struct { diff --git a/operator/config/crd/bases/ais.nvidia.com_aistores.yaml b/operator/config/crd/bases/ais.nvidia.com_aistores.yaml index c594ca9a..0ac6c80e 100644 --- a/operator/config/crd/bases/ais.nvidia.com_aistores.yaml +++ b/operator/config/crd/bases/ais.nvidia.com_aistores.yaml @@ -1235,12 +1235,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the - container runtime. + container runtime. Note that this field cannot be set when + spec.os.name is windows. properties: add: description: Added capabilities @@ -1260,25 +1262,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root - on the host. Defaults to false. + on the host. Defaults to false. Note that this field cannot + be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag - to be enabled. + to be enabled. Note that this field cannot be set when spec.os.name + is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. - Default is false. + Default is false. Note that this field cannot be set when + spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -1295,7 +1301,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -1304,7 +1311,8 @@ spec: SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -1326,7 +1334,8 @@ spec: seccompProfile: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container - level, the container options override the pod options. + level, the container options override the pod options. Note + that this field cannot be set when spec.os.name is windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -1350,6 +1359,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -1421,7 +1432,8 @@ spec: bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and - permissions of any volume." + permissions of any volume. Note that this field cannot be + set when spec.os.name is windows." format: int64 type: integer fsGroupChangePolicy: @@ -1431,14 +1443,16 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' + and "Always". If not specified, "Always" is used. Note that + this field cannot be set when spec.os.name is windows.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. + takes precedence for that container. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -1456,6 +1470,8 @@ spec: unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is + windows. format: int64 type: integer seLinuxOptions: @@ -1464,7 +1480,8 @@ spec: SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. + for that container. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -1485,7 +1502,8 @@ spec: type: object seccompProfile: description: The seccomp options to use by the containers - in this pod. + in this pod. Note that this field cannot be set when spec.os.name + is windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -1508,6 +1526,8 @@ spec: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container. + Note that this field cannot be set when spec.os.name is + windows. items: format: int64 type: integer @@ -1515,7 +1535,8 @@ spec: sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container - runtime) might fail to launch. + runtime) might fail to launch. Note that this field cannot + be set when spec.os.name is windows. items: description: Sysctl defines a kernel parameter to be set properties: @@ -1535,7 +1556,8 @@ spec: containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -2492,7 +2514,7 @@ spec: type: array type: object type: object - allow_shared_no_disks: + allowSharedNoDisks: description: AllowSharedOrNoDisks - disables FsID and mountpath disks validation on target nodes. NOT recommended for production deployments @@ -2507,12 +2529,14 @@ spec: This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged - 2) has CAP_SYS_ADMIN' + 2) has CAP_SYS_ADMIN Note that this field cannot be set + when spec.os.name is windows.' type: boolean capabilities: description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the - container runtime. + container runtime. Note that this field cannot be set when + spec.os.name is windows. properties: add: description: Added capabilities @@ -2532,25 +2556,29 @@ spec: privileged: description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root - on the host. Defaults to false. + on the host. Defaults to false. Note that this field cannot + be set when spec.os.name is windows. type: boolean procMount: description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag - to be enabled. + to be enabled. Note that this field cannot be set when spec.os.name + is windows. type: string readOnlyRootFilesystem: description: Whether this container has a read-only root filesystem. - Default is false. + Default is false. Note that this field cannot be set when + spec.os.name is windows. type: boolean runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -2567,7 +2595,8 @@ spec: process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the - value specified in SecurityContext takes precedence. + value specified in SecurityContext takes precedence. Note + that this field cannot be set when spec.os.name is windows. format: int64 type: integer seLinuxOptions: @@ -2576,7 +2605,8 @@ spec: SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -2598,7 +2628,8 @@ spec: seccompProfile: description: The seccomp options to use by this container. If seccomp options are provided at both the pod & container - level, the container options override the pod options. + level, the container options override the pod options. Note + that this field cannot be set when spec.os.name is windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -2622,6 +2653,8 @@ spec: containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is + linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission @@ -2759,7 +2792,8 @@ spec: bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and - permissions of any volume." + permissions of any volume. Note that this field cannot be + set when spec.os.name is windows." format: int64 type: integer fsGroupChangePolicy: @@ -2769,14 +2803,16 @@ spec: support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" - and "Always". If not specified, "Always" is used.' + and "Always". If not specified, "Always" is used. Note that + this field cannot be set when spec.os.name is windows.' type: string runAsGroup: description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. + takes precedence for that container. Note that this field + cannot be set when spec.os.name is windows. format: int64 type: integer runAsNonRoot: @@ -2794,6 +2830,8 @@ spec: unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + Note that this field cannot be set when spec.os.name is + windows. format: int64 type: integer seLinuxOptions: @@ -2802,7 +2840,8 @@ spec: SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. + for that container. Note that this field cannot be set when + spec.os.name is windows. properties: level: description: Level is SELinux level label that applies @@ -2823,7 +2862,8 @@ spec: type: object seccompProfile: description: The seccomp options to use by the containers - in this pod. + in this pod. Note that this field cannot be set when spec.os.name + is windows. properties: localhostProfile: description: localhostProfile indicates a profile defined @@ -2846,6 +2886,8 @@ spec: description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container. + Note that this field cannot be set when spec.os.name is + windows. items: format: int64 type: integer @@ -2853,7 +2895,8 @@ spec: sysctls: description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container - runtime) might fail to launch. + runtime) might fail to launch. Note that this field cannot + be set when spec.os.name is windows. items: description: Sysctl defines a kernel parameter to be set properties: @@ -2873,7 +2916,8 @@ spec: containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence. + takes precedence. Note that this field cannot be set when + spec.os.name is linux. properties: gmsaCredentialSpec: description: GMSACredentialSpec is where the GMSA admission diff --git a/operator/config/samples/ais_v1beta1_aistore.yaml b/operator/config/samples/ais_v1beta1_aistore.yaml index 161b8059..8842b4f7 100644 --- a/operator/config/samples/ais_v1beta1_aistore.yaml +++ b/operator/config/samples/ais_v1beta1_aistore.yaml @@ -21,14 +21,15 @@ spec: - path: "/ais1" size: 10Gi - nodiskio: - enabled: false - dryobjsize: "8M" - - nodeImage: "aistore/aisnode:3.7" + nodeImage: "aistore/aisnode:3.8" initImage: "aistore/ais-init:latest" hostpathPrefix: "/etc/ais" + # In certain environments (e.g. minikube), storage volumes attached to AIS targets may not have associated block devices. + # Alternatively, AIS targets may "see" multiple mountpath directories sharing a single given block device. + # In both of those cases, set allowSharedNoDisks = true (but note that this setting is **not recommended** for production). + allowSharedNoDisks: false + # To be able to access the AIS deployment outside kubernetes cluster, set: # enableExternalLB: true # NOTE: For external access to work, the kubernetes cluster should have the capabilitly to create LoadBalancer services with valid external IP.