feat: add imageRegistryCredentials handler (#174)

Co-authored-by: Jimmi Dyson <jimmidyson@gmail.com>

Author: dkoshkin

api/v1alpha1/clusterconfig_types.go

@@ -4,6 +4,7 @@
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/capi/clustertopology/variables"
@@ -28,6 +29,9 @@ type GenericClusterConfig struct {
 	// +optional
 	ExtraAPIServerCertSANs ExtraAPIServerCertSANs `json:"extraAPIServerCertSANs,omitempty"`
 
+	// +optional
+	ImageRegistries ImageRegistries `json:"imageRegistries,omitempty"`
+
 	// +optional
 	Addons *Addons `json:"addons,omitempty"`
 }
@@ -46,6 +50,7 @@ func (GenericClusterConfig) VariableSchema() clusterv1.VariableSchema {
 					"",
 				).VariableSchema().
 					OpenAPIV3Schema,
+				"imageRegistries": ImageRegistries{}.VariableSchema().OpenAPIV3Schema,
 			},
 		},
 	}
@@ -175,6 +180,84 @@ func (ExtraAPIServerCertSANs) VariableSchema() clusterv1.VariableSchema {
 	}
 }
 
+type ImageRegistries struct {
+	// +optional
+	ImageRegistryCredentials ImageRegistryCredentials `json:"credentials,omitempty"`
+}
+
+func (ImageRegistries) VariableSchema() clusterv1.VariableSchema {
+	return clusterv1.VariableSchema{
+		OpenAPIV3Schema: clusterv1.JSONSchemaProps{
+			Description: "Configuration for image registries.",
+			Type:        "object",
+			Properties: map[string]clusterv1.JSONSchemaProps{
+				"credentials": ImageRegistryCredentials{}.VariableSchema().OpenAPIV3Schema,
+			},
+		},
+	}
+}
+
+type ImageRegistryCredentials []ImageRegistryCredentialsResource
+
+func (ImageRegistryCredentials) VariableSchema() clusterv1.VariableSchema {
+	resourceSchema := ImageRegistryCredentialsResource{}.VariableSchema().OpenAPIV3Schema
+
+	return clusterv1.VariableSchema{
+		OpenAPIV3Schema: clusterv1.JSONSchemaProps{
+			Description: "Image registry credentials to set up on all Nodes in the cluster. " +
+				"Enabling this will configure the Kubelets with " +
+				"https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/.",
+			Type:  "array",
+			Items: &resourceSchema,
+		},
+	}
+}
+
+// ImageRegistryCredentialsResource required for providing credentials for an image registry URL.
+type ImageRegistryCredentialsResource struct {
+	// Registry URL.
+	URL string `json:"url"`
+
+	// The Secret containing the registry credentials.
+	// The Secret should have keys 'username' and 'password'.
+	// This credentials Secret is not required for some registries, e.g. ECR.
+	// +optional
+	Secret *corev1.ObjectReference `json:"secretRef,omitempty"`
+}
+
+func (ImageRegistryCredentialsResource) VariableSchema() clusterv1.VariableSchema {
+	return clusterv1.VariableSchema{
+		OpenAPIV3Schema: clusterv1.JSONSchemaProps{
+			Type: "object",
+			Properties: map[string]clusterv1.JSONSchemaProps{
+				"url": {
+					Description: "Registry URL.",
+					Type:        "string",
+				},
+				"secretRef": {
+					Description: "The Secret containing the registry credentials. " +
+						"The Secret should have keys 'username' and 'password'. " +
+						"This credentials Secret is not required for some registries, e.g. ECR.",
+					Type: "object",
+					Properties: map[string]clusterv1.JSONSchemaProps{
+						"name": {
+							Description: "The name of the Secret containing the registry credentials.",
+							Type:        "string",
+						},
+						"namespace": {
+							Description: "The namespace of the Secret containing the registry credentials. " +
+								"Defaults to the namespace of the KubeadmControlPlaneTemplate and KubeadmConfigTemplate" +
+								" that reference this variable.",
+							Type: "string",
+						},
+					},
+				},
+			},
+			Required: []string{"url"},
+		},
+	}
+}
+
 type Addons struct {
 	// +optional
 	CNI *CNI `json:"cni,omitempty"`

api/v1alpha1/doc.go

@@ -4,8 +4,4 @@
 // Package v1alpha1 contains API Schema definitions for the CAPI extensions v1alpha1 API group
 // +kubebuilder:object:generate=true
 // +groupName=capiext.labs.d2iq.io
-//
-//go:generate -command CTRLGEN controller-gen  paths="./..."
-//go:generate CTRLGEN rbac:headerFile="../../hack/license-header.yaml.txt",roleName=capi-runtime-extensions-manager-role output:rbac:artifacts:config=../../charts/capi-runtime-extensions/templates
-//go:generate CTRLGEN object:headerFile="../../hack/license-header.go.txt" output:object:artifacts:config=/dev/null
 package v1alpha1

api/v1alpha1/zz_generated.deepcopy.go

@@ -4,7 +4,7 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  name: capi-runtime-extensions-manager-role
+  name: {{ include "chart.name" . }}-manager-role
 rules:
 - apiGroups:
   - ""
@@ -23,8 +23,11 @@ rules:
   resources:
   - secrets
   verbs:
+  - create
   - get
   - list
+  - patch
+  - update
   - watch
 - apiGroups:
   - addons.cluster.x-k8s.io

charts/capi-runtime-extensions/templates/role.yaml

@@ -6,6 +6,7 @@ package capitest
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/onsi/gomega"
@@ -24,6 +25,7 @@ type PatchTestDef struct {
 	Vars                  []runtimehooksv1.Variable
 	RequestItem           runtimehooksv1.GeneratePatchesRequestItem
 	ExpectedPatchMatchers []JSONPatchMatcher
+	ExpectedFailure       bool
 }
 
 type JSONPatchMatcher struct {
@@ -53,7 +55,12 @@ func ValidateGeneratePatches[T mutation.GeneratePatches](
 			}
 			resp := &runtimehooksv1.GeneratePatchesResponse{}
 			h.GeneratePatches(context.Background(), req, resp)
-			g.Expect(resp.Status).To(gomega.Equal(runtimehooksv1.ResponseStatusSuccess))
+			expectedStatus := runtimehooksv1.ResponseStatusSuccess
+			if tt.ExpectedFailure {
+				expectedStatus = runtimehooksv1.ResponseStatusFailure
+			}
+			g.Expect(resp.Status).
+				To(gomega.Equal(expectedStatus), fmt.Sprintf("Message: %s", resp.Message))
 
 			if len(tt.ExpectedPatchMatchers) == 0 {
 				g.Expect(resp.Items).To(gomega.BeEmpty())
@@ -101,6 +108,7 @@ func VariableWithValue(
 
 		for _, p := range variablePath[:len(variablePath)-1] {
 			nestedValue[p] = make(map[string]any, 1)
+			nestedValue = nestedValue[p].(map[string]any)
 		}
 
 		nestedValue[variablePath[len(variablePath)-1]] = value

common/pkg/testutils/capitest/patches.go

@@ -4,6 +4,7 @@
 package request
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -16,6 +17,13 @@ import (
 	"github.com/d2iq-labs/capi-runtime-extensions/common/pkg/testutils/capitest/serializer"
 )
 
+const (
+	ClusterName                                  = "test-cluster"
+	KubeadmConfigTemplateRequestObjectName       = "test-kubeadmconfigtemplate"
+	KubeadmControlPlaneTemplateRequestObjectName = "test-kubeadmcontrolplanetemplate"
+	Namespace                                    = corev1.NamespaceDefault
+)
+
 // NewRequestItem returns a GeneratePatchesRequestItem with the given variables and object.
 func NewRequestItem(
 	object runtime.Object,
@@ -42,10 +50,17 @@ func NewKubeadmConfigTemplateRequestItem(uid types.UID) runtimehooksv1.GenerateP
 				APIVersion: bootstrapv1.GroupVersion.String(),
 				Kind:       "KubeadmConfigTemplate",
 			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      KubeadmConfigTemplateRequestObjectName,
+				Namespace: Namespace,
+			},
 			Spec: bootstrapv1.KubeadmConfigTemplateSpec{
 				Template: bootstrapv1.KubeadmConfigTemplateResource{
 					Spec: bootstrapv1.KubeadmConfigSpec{
 						PostKubeadmCommands: []string{"initial-post-kubeadm"},
+						JoinConfiguration: &bootstrapv1.JoinConfiguration{
+							NodeRegistration: bootstrapv1.NodeRegistrationOptions{},
+						},
 					},
 				},
 			},
@@ -67,10 +82,30 @@ func NewKubeadmControlPlaneTemplateRequestItem(
 				APIVersion: controlplanev1.GroupVersion.String(),
 				Kind:       "KubeadmControlPlaneTemplate",
 			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      KubeadmControlPlaneTemplateRequestObjectName,
+				Namespace: Namespace,
+			},
+			Spec: controlplanev1.KubeadmControlPlaneTemplateSpec{
+				Template: controlplanev1.KubeadmControlPlaneTemplateResource{
+					Spec: controlplanev1.KubeadmControlPlaneTemplateResourceSpec{
+						KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{
+							InitConfiguration: &bootstrapv1.InitConfiguration{
+								NodeRegistration: bootstrapv1.NodeRegistrationOptions{},
+							},
+							JoinConfiguration: &bootstrapv1.JoinConfiguration{
+								NodeRegistration: bootstrapv1.NodeRegistrationOptions{},
+							},
+						},
+					},
+				},
+			},
 		},
 		&runtimehooksv1.HolderReference{
 			Kind:      "Cluster",
 			FieldPath: "spec.controlPlaneRef",
+			Name:      ClusterName,
+			Namespace: Namespace,
 		},
 		uid,
 	)

common/pkg/testutils/capitest/request/items.go

@@ -38,6 +38,6 @@ configure defaults specific for their environment rather than compiling the defa
 The Helm chart comes with default configurations for the Calico Installation CRS per supported provider, but overriding
 is possible. To do so, specify:
 
-```bash
+```shell
 --set-file handlers.CalicoCNI.defaultInstallationConfigMaps.DockerCluster.configMap.content=<file>
 ```

docs/content/addons/calico-cni.md

@@ -39,6 +39,11 @@ spec:
             additionalNo:
               - no-proxy-1.example.com
               - no-proxy-2.example.com
+          imageRegistries:
+            credentials:
+              - url: https://my-registry.io
+                secretRef:
+                  name: my-registry-credentials
           cni:
             provider: calico
 ```

docs/content/customization/_index.md

@@ -0,0 +1,43 @@
++++
+title = "Image registries"
++++
+
+Add image registry configuration to all Nodes in the cluster.
+
+When the `credentials` variable is set, `files` and `preKubeadmnCommands` with configurations for
+[Kubelet image credential provider](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-credential-provider/)
+and [dynamic credential provider](https://github.com/mesosphere/dynamic-credential-provider) will be added.
+
+This customization will be available when the
+[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`.
+
+## Example
+
+If your registry requires static credentials, create a Kubernetes Secret with keys for `username` and `password`:
+
+```shell
+kubectl create secret generic my-registry-credentials \
+  --from-literal username=${REGISTRY_USERNAME} --from-literal password=${REGISTRY_PASSWORD}
+```
+
+To add image registry credentials, specify the following configuration:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          imageRegistries:
+            credentials:
+              - url: https://my-registry.io
+                secretRef:
+                  name: my-registry-credentials
+```
+
+Applying this configuration will result in new files and preKubeadmCommands
+on the `KubeadmControlPlaneTemplate` and `KubeadmConfigTemplate`.