diff --git a/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml b/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml index cb13ba698..d3ce9a646 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml @@ -1,12 +1,13 @@ -replicaCount: 2 +replicaCount: {{ .Replicas }} persistence: enabled: true size: 50Gi service: type: ClusterIP clusterIP: {{ .ServiceIP }} - port: 80 + port: 443 statefulSet: enabled: true syncer: interval: 2m +tlsSecretName: {{ .TLSSecretName }} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml index 83519b707..94a2b8e62 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml @@ -32,3 +32,20 @@ spec: kind: {{ .Values.certificates.issuer.kind }} name: {{ template "chart.issuerName" . }} secretName: {{ template "chart.name" . }}-admission-tls +--- +# CA used to sign certificates for the clusters' registry addons +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: registry-addon-root-ca + namespace: {{ .Release.Namespace }} + labels: + {{- include "chart.labels" . | nindent 4 }} +spec: + isCA: true + commonName: registry-addon + secretName: registry-addon-root-ca + issuerRef: + kind: {{ .Values.certificates.issuer.kind }} + name: {{ template "chart.issuerName" . }} + duration: 87600h # 10 years diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml index 77d78c82f..d5292ef2c 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml @@ -25,7 +25,7 @@ data: RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}' cncf-distribution-registry: | ChartName: docker-registry - ChartVersion: 2.3.1 + ChartVersion: 2.3.2 RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/staging/{{ end }}' cosi-controller: | ChartName: cosi diff --git a/docs/content/addons/registry-certificate.png b/docs/content/addons/registry-certificate.png new file mode 100644 index 000000000..dd3049832 Binary files /dev/null and b/docs/content/addons/registry-certificate.png differ diff --git a/docs/content/addons/registry.md b/docs/content/addons/registry.md index 0174a8ac8..0b6e2e8b3 100644 --- a/docs/content/addons/registry.md +++ b/docs/content/addons/registry.md @@ -31,6 +31,24 @@ spec: registry: {} ``` +## Registry Certificate + +1. A root CA Certificate is deployed in the provider's namespace. +2. cert-manager generates a 10-year self-signed root Certificate + and creates a Secret `registry-addon-root-ca` in the provider's namespace. +3. BCC handler copies `ca.crt` from the `registry-addon-root-ca` Secret + to a new cluster Secret `-registry-addon-ca`. + A client pushing to the registry can use either the root CA Secret or the cluster Secret to trust the registry. +4. The cluster CA Secret contents (`ca.crt`) is written out as files on the Nodes + and used by Containerd to trust the registry addon. +5. During the initial cluster creation, the ACPI handler uses the root CA to create a new 2-year server certificate + for the registry and creates a Secret `registry-tls` on the remote cluster. +6. During cluster upgrades, the BCU handler renews the server certificate + and updates the Secret `registry-tls` on the remote cluster with the new certificate. + It is expected that clusters will be upgraded at least once every 2 years to avoid certificate expiration. + +![registry-certificate.png](registry-certificate.png) + [Distribution]: https://github.com/distribution/distribution [Cluster API Add-on Provider for Helm]: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm [Regsync]: https://regclient.org/usage/regsync/ diff --git a/hack/addons/helm-chart-bundler/repos.yaml b/hack/addons/helm-chart-bundler/repos.yaml index b3cd73091..ef8d9efb3 100644 --- a/hack/addons/helm-chart-bundler/repos.yaml +++ b/hack/addons/helm-chart-bundler/repos.yaml @@ -35,7 +35,7 @@ repositories: repoURL: https://mesosphere.github.io/charts/staging/ charts: docker-registry: - - 2.3.1 + - 2.3.2 local-path-provisioner: repoURL: https://charts.containeroo.ch charts: diff --git a/hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl b/hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl index 87ea194ce..9c7767a28 100644 --- a/hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl +++ b/hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl @@ -18,7 +18,7 @@ helmCharts: - name: docker-registry repo: https://mesosphere.github.io/charts/staging/ releaseName: cncf-distribution-registry - version: 2.3.1 + version: 2.3.2 valuesFile: helm-values.yaml includeCRDs: true skipTests: true diff --git a/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go b/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go index a4080956f..066f4c71b 100644 --- a/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go +++ b/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go @@ -24,6 +24,11 @@ import ( const ( DefaultHelmReleaseName = "cncf-distribution-registry" DefaultHelmReleaseNamespace = "registry-system" + + stsName = "cncf-distribution-registry-docker-registry" + stsHeadlessServiceName = "cncf-distribution-registry-docker-registry-headless" + stsReplicas = 2 + tlsSecretName = "registry-tls" ) type Config struct { @@ -59,14 +64,64 @@ func New( } } +// Setup ensures any pre-requisites for the CNCF Distribution registry addon are met. +// It is expected to be called before the cluster is created. +// Specifically, it ensures that the CA secret for the registry is created in the cluster's namespace. +func (n *CNCFDistribution) Setup( + ctx context.Context, + _ v1alpha1.RegistryAddon, + cluster *clusterv1.Cluster, + log logr.Logger, +) error { + log.Info("Setting up CA for CNCF Distribution registry") + err := utils.EnsureCASecretForCluster( + ctx, + n.client, + cluster, + ) + if err != nil { + return fmt.Errorf("failed to ensure CA secret for CNCF Distribution registry addon: %w", err) + } + return nil +} + +// Apply applies the CNCF Distribution registry addon to the cluster. func (n *CNCFDistribution) Apply( ctx context.Context, _ v1alpha1.RegistryAddon, cluster *clusterv1.Cluster, log logr.Logger, ) error { - log.Info("Applying CNCF Distribution registry installation") + // Copy the TLS secret to the remote cluster. + serviceIP, err := utils.ServiceIPForCluster(cluster) + if err != nil { + return fmt.Errorf("error getting service IP for the CNCF distribution registry: %w", err) + } + opts := &utils.EnsureCertificateOpts{ + RemoteSecretKey: ctrlclient.ObjectKey{ + Name: tlsSecretName, + Namespace: DefaultHelmReleaseNamespace, + }, + Spec: utils.CertificateSpec{ + CommonName: stsName, + DNSNames: certificateDNSNames(), + IPAddresses: certificateIPAddresses(serviceIP), + }, + } + err = utils.EnsureRegistryServerCertificateSecretOnRemoteCluster( + ctx, + n.client, + cluster, + opts, + ) + if err != nil { + return fmt.Errorf( + "failed to copy certificate secret for CNCF Distribution registry addon to remote cluster: %w", + err, + ) + } + log.Info("Applying CNCF Distribution registry installation") helmChartInfo, err := n.helmChartInfoGetter.For(ctx, log, config.CNCFDistributionRegistry) if err != nil { return fmt.Errorf("failed to get CNCF Distribution registry helm chart: %w", err) @@ -101,11 +156,15 @@ func templateValues(cluster *clusterv1.Cluster, text string) (string, error) { } type input struct { - ServiceIP string + ServiceIP string + Replicas int32 + TLSSecretName string } templateInput := input{ - ServiceIP: serviceIP, + Replicas: stsReplicas, + ServiceIP: serviceIP, + TLSSecretName: tlsSecretName, } var b bytes.Buffer @@ -119,3 +178,34 @@ func templateValues(cluster *clusterv1.Cluster, text string) (string, error) { return b.String(), nil } + +func certificateDNSNames() []string { + names := []string{ + stsName, + fmt.Sprintf("%s.%s", stsName, DefaultHelmReleaseNamespace), + fmt.Sprintf("%s.%s.svc", stsName, DefaultHelmReleaseNamespace), + fmt.Sprintf("%s.%s.svc.cluster.local", stsName, DefaultHelmReleaseNamespace), + } + for i := 0; i < stsReplicas; i++ { + names = append(names, + []string{ + fmt.Sprintf("%s-%d", stsName, i), + fmt.Sprintf("%s-%d.%s.%s", stsName, i, stsHeadlessServiceName, DefaultHelmReleaseNamespace), + fmt.Sprintf("%s-%d.%s.%s.svc", stsName, i, stsHeadlessServiceName, DefaultHelmReleaseNamespace), + fmt.Sprintf( + "%s-%d.%s.%s.svc.cluster.local", + stsName, i, stsHeadlessServiceName, DefaultHelmReleaseNamespace, + ), + }..., + ) + } + + return names +} + +func certificateIPAddresses(serviceIP string) []string { + return []string{ + serviceIP, + "127.0.0.1", + } +} diff --git a/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler_test.go b/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler_test.go new file mode 100644 index 000000000..fbe67c790 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler_test.go @@ -0,0 +1,29 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cncfdistribution + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_certificateDNSNames(t *testing.T) { + //nolint:lll // Keep long lines for readability. + expected := []string{ + "cncf-distribution-registry-docker-registry", + "cncf-distribution-registry-docker-registry.registry-system", + "cncf-distribution-registry-docker-registry.registry-system.svc", + "cncf-distribution-registry-docker-registry.registry-system.svc.cluster.local", + "cncf-distribution-registry-docker-registry-0", + "cncf-distribution-registry-docker-registry-0.cncf-distribution-registry-docker-registry-headless.registry-system", + "cncf-distribution-registry-docker-registry-0.cncf-distribution-registry-docker-registry-headless.registry-system.svc", + "cncf-distribution-registry-docker-registry-0.cncf-distribution-registry-docker-registry-headless.registry-system.svc.cluster.local", + "cncf-distribution-registry-docker-registry-1", + "cncf-distribution-registry-docker-registry-1.cncf-distribution-registry-docker-registry-headless.registry-system", + "cncf-distribution-registry-docker-registry-1.cncf-distribution-registry-docker-registry-headless.registry-system.svc", + "cncf-distribution-registry-docker-registry-1.cncf-distribution-registry-docker-registry-headless.registry-system.svc.cluster.local", + } + assert.Equal(t, expected, certificateDNSNames()) +} diff --git a/pkg/handlers/generic/lifecycle/registry/handler.go b/pkg/handlers/generic/lifecycle/registry/handler.go index 891e747d6..876f11940 100644 --- a/pkg/handlers/generic/lifecycle/registry/handler.go +++ b/pkg/handlers/generic/lifecycle/registry/handler.go @@ -20,6 +20,12 @@ import ( ) type RegistryProvider interface { + Setup( + ctx context.Context, + registryVar v1alpha1.RegistryAddon, + cluster *clusterv1.Cluster, + log logr.Logger, + ) error Apply( ctx context.Context, registryVar v1alpha1.RegistryAddon, @@ -37,6 +43,7 @@ type RegistryHandler struct { var ( _ commonhandlers.Named = &RegistryHandler{} + _ lifecycle.BeforeClusterCreate = &RegistryHandler{} _ lifecycle.AfterControlPlaneInitialized = &RegistryHandler{} _ lifecycle.BeforeClusterUpgrade = &RegistryHandler{} ) @@ -57,6 +64,17 @@ func (r *RegistryHandler) Name() string { return "RegistryHandler" } +func (r *RegistryHandler) BeforeClusterCreate( + ctx context.Context, + req *runtimehooksv1.BeforeClusterCreateRequest, + resp *runtimehooksv1.BeforeClusterCreateResponse, +) { + commonResponse := &runtimehooksv1.CommonResponse{} + r.setup(ctx, &req.Cluster, commonResponse) + resp.Status = commonResponse.GetStatus() + resp.Message = commonResponse.GetMessage() +} + func (r *RegistryHandler) AfterControlPlaneInitialized( ctx context.Context, req *runtimehooksv1.AfterControlPlaneInitializedRequest, @@ -79,6 +97,91 @@ func (r *RegistryHandler) BeforeClusterUpgrade( resp.Message = commonResponse.GetMessage() } +func (r *RegistryHandler) setup( + ctx context.Context, + cluster *clusterv1.Cluster, + resp *runtimehooksv1.CommonResponse, +) { + clusterKey := ctrlclient.ObjectKeyFromObject(cluster) + + log := ctrl.LoggerFrom(ctx).WithValues( + "cluster", + clusterKey, + ) + + varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables) + registryVar, err := variables.Get[v1alpha1.RegistryAddon]( + varMap, + r.variableName, + r.variablePath...) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5). + Info( + "Skipping RegistryAddon, field is not specified", + "error", + err, + ) + return + } + log.Error( + err, + "failed to read RegistryAddon provider from cluster definition", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to read RegistryAddon provider from cluster definition: %v", + err, + ), + ) + return + } + + handler, ok := r.ProviderHandler[registryVar.Provider] + if !ok { + err = fmt.Errorf("unknown RegistryAddon Provider") + log.Error(err, "provider", registryVar.Provider) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("%s %s", err, registryVar.Provider), + ) + return + } + + log.Info(fmt.Sprintf("Setting up RegistryAddon provider prerequisites %s", registryVar.Provider)) + err = handler.Setup( + ctx, + registryVar, + cluster, + log, + ) + if err != nil { + log.Error( + err, + fmt.Sprintf( + "failed to set up RegistryAddon provider prerequisites %s", + registryVar.Provider, + ), + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf( + "failed to set up RegistryAddon provider prerequisites: %v", + err, + ), + ) + return + } + + resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) + resp.SetMessage( + fmt.Sprintf( + "Set up RegistryAddon provider prerequisites %s", + registryVar.Provider, + ), + ) +} + func (r *RegistryHandler) apply( ctx context.Context, cluster *clusterv1.Cluster, diff --git a/pkg/handlers/generic/lifecycle/registry/utils/tls.go b/pkg/handlers/generic/lifecycle/registry/utils/tls.go new file mode 100644 index 000000000..15928e610 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/utils/tls.go @@ -0,0 +1,271 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "cmp" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + _ "embed" + "encoding/pem" + "fmt" + "math/big" + "net" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils" +) + +const ( + caCrtKey = "ca.crt" +) + +var ( + // Similar to CAPI, set the NotBefore to a few minutes in the past to account for clock skew. + // This cert is being generated on the management cluster, but used by a workload cluster. + defaultCertificateNotBeforeSkew = 5 * time.Minute + // Valid for 2 years to avoid expiring before the cluster is upgraded. + defaultCertificateDuration = 2 * 365 * 24 * time.Hour +) + +type EnsureCertificateOpts struct { + // RemoteSecretKey is the name and namespace of the TLS secret to be created on the remote cluster. + RemoteSecretKey ctrlclient.ObjectKey + + Spec CertificateSpec +} + +type CertificateSpec struct { + // CommonName is the common name to be included in the certificate. + CommonName string + // DNSNames is a list of DNS names to be included in the certificate. + DNSNames []string + // IPAddresses is a list of IP addresses to be included in the certificate. + IPAddresses []string + // Duration is the duration for which the certificate is valid. + Duration time.Duration +} + +// EnsureCASecretForCluster ensures that the registry addon CA secret exists for the given cluster. +// It copies the ca.crt value from the global CA secret to a unique secret in the cluster's namespace. +func EnsureCASecretForCluster( + ctx context.Context, + c ctrlclient.Client, + cluster *clusterv1.Cluster, +) error { + globalTLSCertificateSecret, err := handlersutils.SecretForRegistryAddonRootCA(ctx, c) + if err != nil { + return err + } + + clusterCASecret := buildClusterCASecret(globalTLSCertificateSecret, cluster) + + // Copy the global CA certificate to a cluster CA secret. + err = handlersutils.EnsureSecretForLocalCluster(ctx, c, clusterCASecret, cluster) + if err != nil { + return fmt.Errorf("failed to ensure cluster CA secret for cluster: %w", err) + } + + return nil +} + +// EnsureRegistryServerCertificateSecretOnRemoteCluster ensures that a registry TLS certificate is signed +// by the global CA and is created as secret on the remote cluster. +// +// The high level flow is as follows: +// 1. Create a new TLS certificate and sign it with the global CA. +// 2. Copy the TLS certificate secret to the remote cluster to be used by the registry Pods. +// +// Intentionally not using cert-manager to create the certificate, +// as we want to avoid automatic renewal and instead recreate the certificate each time with a new expiration date. +func EnsureRegistryServerCertificateSecretOnRemoteCluster( + ctx context.Context, + c ctrlclient.Client, + cluster *clusterv1.Cluster, + opts *EnsureCertificateOpts, +) error { + globalTLSCertificateSecret, err := handlersutils.SecretForRegistryAddonRootCA(ctx, c) + if err != nil { + return fmt.Errorf("failed to get TLS secret used to sign the certificate: %w", err) + } + + // Always recreate the TLS certificate using the global CA to sign it. + certPEM, keyPEM, caPEM, err := generateCertificateData(globalTLSCertificateSecret, opts) + if err != nil { + return fmt.Errorf("failed to generate new certificate: %w", err) + } + err = copyTLSCertificateSecretToRemoteCluster( + ctx, + c, + cluster, + opts.RemoteSecretKey, + certPEM, keyPEM, caPEM, + ) + if err != nil { + return err + } + + return nil +} + +func buildClusterCASecret( + globalTLSCertificateSecret *corev1.Secret, + cluster *clusterv1.Cluster, +) *corev1.Secret { + // The root CA will have, tls.crt, tls.key, and ca.crt. + // We only copy the ca.crt from the global TLS secret to the cluster CA secret. + data := map[string][]byte{ + caCrtKey: globalTLSCertificateSecret.Data[caCrtKey], + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: handlersutils.SecretNameForRegistryAddonCA(cluster), + Namespace: cluster.Namespace, + }, + Data: data, + } +} + +func generateCertificateData( + globalCASecret *corev1.Secret, + opts *EnsureCertificateOpts, +) (serverCertPEM, serverKeyPEM, caCertPEM []byte, err error) { + // 1. load CA PEMs from Secret + caCertPEM, ok := globalCASecret.Data[corev1.TLSCertKey] + if !ok { + return nil, nil, nil, + fmt.Errorf("%s not found in Secret", corev1.TLSCertKey) + } + caKeyPEM, ok := globalCASecret.Data[corev1.TLSPrivateKeyKey] + if !ok { + return nil, nil, nil, + fmt.Errorf("%s not found in Secret", corev1.TLSPrivateKeyKey) + } + + // 2. parse CA cert + caBlock, _ := pem.Decode(caCertPEM) + if caBlock == nil || caBlock.Type != "CERTIFICATE" { + return nil, nil, nil, fmt.Errorf("failed to decode CA certificate PEM") + } + caCert, err := x509.ParseCertificate(caBlock.Bytes) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to parse CA cert: %w", err) + } + + // 3. parse CA private key (PKCS#1 or PKCS#8) + keyBlock, _ := pem.Decode(caKeyPEM) + if keyBlock == nil { + return nil, nil, nil, fmt.Errorf("failed to decode CA private key PEM") + } + var caPriv interface{} + switch keyBlock.Type { + case "RSA PRIVATE KEY": + caPriv, err = x509.ParsePKCS1PrivateKey(keyBlock.Bytes) + case "PRIVATE KEY": + caPriv, err = x509.ParsePKCS8PrivateKey(keyBlock.Bytes) + default: + err = fmt.Errorf("unsupported private key encoding type %q", keyBlock.Type) + } + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to parse CA private key: %w", err) + } + + // 4. generate server key + serverKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to generate server key: %w", err) + } + + // 5. build server cert template + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to generate serial: %w", err) + } + tmpl := &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: opts.Spec.CommonName, + }, + NotBefore: time.Now().Add(-1 * defaultCertificateNotBeforeSkew), + NotAfter: time.Now().Add(cmp.Or(opts.Spec.Duration, defaultCertificateDuration)), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: opts.Spec.DNSNames, + BasicConstraintsValid: true, + } + for _, s := range opts.Spec.IPAddresses { + if ip := net.ParseIP(s); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } + } + + // 6. sign server cert with the CA + derBytes, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &serverKey.PublicKey, caPriv) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to create certificate: %w", err) + } + + // 7. PEM-encode outputs + serverCertPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + serverKeyPEM = pem.EncodeToMemory( + &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}, + ) + + return serverCertPEM, serverKeyPEM, caCertPEM, nil +} + +// copyTLSCertificateSecretToRemoteCluster copies the registry TLS certificate Secret to the remote cluster. +func copyTLSCertificateSecretToRemoteCluster( + ctx context.Context, + c ctrlclient.Client, + cluster *clusterv1.Cluster, + key ctrlclient.ObjectKey, + certPEM, keyPEM, caPEM []byte, +) error { + err := handlersutils.EnsureSecretOnRemoteCluster( + ctx, + c, buildRegistryTLSCertificateSecret(key, certPEM, keyPEM, caPEM), + cluster, + ) + if err != nil { + return fmt.Errorf("failed to create registry addon TLS secret on remote cluster: %w", err) + } + + return nil +} + +func buildRegistryTLSCertificateSecret( + key ctrlclient.ObjectKey, + certPEM, keyPEM, caPEM []byte, +) *corev1.Secret { + data := map[string][]byte{ + corev1.TLSCertKey: certPEM, + corev1.TLSPrivateKeyKey: keyPEM, + caCrtKey: caPEM, + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Data: data, + Type: corev1.SecretTypeTLS, + } +} diff --git a/pkg/handlers/generic/lifecycle/registry/utils/tls_integration_test.go b/pkg/handlers/generic/lifecycle/registry/utils/tls_integration_test.go new file mode 100644 index 000000000..f2cad8268 --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/utils/tls_integration_test.go @@ -0,0 +1,279 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/remote" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +var ( + testCrt = []byte(`-----BEGIN CERTIFICATE----- +MIIC/jCCAeagAwIBAgIQNIZl/oI199Zgn4a2c6pAADANBgkqhkiG9w0BAQsFADAZ +MRcwFQYDVQQDEw5yZWdpc3RyeS1hZGRvbjAeFw0yNTA1MTUxOTU2MDhaFw0zNTA1 +MTMxOTU2MDhaMBkxFzAVBgNVBAMTDnJlZ2lzdHJ5LWFkZG9uMIIBIjANBgkqhkiG +9w0BAQEFAAOCAQ8AMIIBCgKCAQEAo5eCNCJwyZf66WJw5+YjsHRKvtFzMx08Q2cn +doCaRj1/enTo2h48o2NMvG2MNYg0SAyAvSrSpy8fWZhXyU/SdUJv+DGCczwMtkH0 +DHXONlFYBkbe16v4QwB5TCX0G+IWZgzfFwX+vT/KVXxjJMmkdSdkvJN6kpD/8knM +jmxTAxjUIKMygfaut21MHI5YlD2h3gfyEsyJrlhhxZK8sWEfdEMd9z259PnthgW/ +ZWVf30L1xy40ErPiZwVZ/X8Y+99+xGXn5unQkTclHDvdXMX1xi7XkhN6kYuJ2CKZ +mWk160lydz0nSP8v7TIv7Mj78WbPnkQH19I9G938mB7d/L7IXQIDAQABo0IwQDAO +BgNVHQ8BAf8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUwp+UqNe3 +CN+mT8CMYf+QvgBHo1swDQYJKoZIhvcNAQELBQADggEBAGcbZb34EEhm9kCfus/c +N53oX15qpiwIKpddE5Vi/MXspLfncRMFqWagoSgvP8zVO5FxyOcMAIPO1lVgdxwU +ATGGHj84AeKQ5wNYyIiZkac/cL2sBjWSogHivCHmMbLIngx2km3LO0iKPF1H6eGp +c9J7CRrehChgLD1Fy4v6CbIf5lzUwhelJtRgXZW0G/LPY3q9DAEwQ0IUFsZz/S3v +H+n9yKYUvKRDCSRoHpL12Jw4XibHfpoQWi3GiaWo+wtInD6/gZ3wOo+2lvz4e1qx +PZHZMQ492XZprH0DkOLIj+oiDKE/RNZAtPfaE2hFr0cs3EnN6NlGOesNtVTn0avf +3YA= +-----END CERTIFICATE-----`) + + testKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAo5eCNCJwyZf66WJw5+YjsHRKvtFzMx08Q2cndoCaRj1/enTo +2h48o2NMvG2MNYg0SAyAvSrSpy8fWZhXyU/SdUJv+DGCczwMtkH0DHXONlFYBkbe +16v4QwB5TCX0G+IWZgzfFwX+vT/KVXxjJMmkdSdkvJN6kpD/8knMjmxTAxjUIKMy +gfaut21MHI5YlD2h3gfyEsyJrlhhxZK8sWEfdEMd9z259PnthgW/ZWVf30L1xy40 +ErPiZwVZ/X8Y+99+xGXn5unQkTclHDvdXMX1xi7XkhN6kYuJ2CKZmWk160lydz0n +SP8v7TIv7Mj78WbPnkQH19I9G938mB7d/L7IXQIDAQABAoIBAEU2xQ/pwm6IrtAv +pjV3WYI+saEqXOMza1vZOQkaQCuXuWfGLv6Z7G30hXLzpm6/wd756z4d8CJr/Yea +vQmfjBuwkE8iI189+OLj5K2g6i5xHB0Lvxzg1ZkDik59gFqLvY5Pw9Op5a2MX77r +cccOyVYH5MckXqfEUYXhU3quujCEk4ha5C+cVEJlnfeO4hTAoKXqhQ3eQ70hQURi +x9zxXOuHZbPr//YVtjeEYcLTeJe7YtV4+pFE8J2QS1L3CXzw1j4uULCUf0yPypXQ +LZL9Z0HIkMrm470G7semL39EbNJhsnFW/SWtgwfwQpH42JcQGWjISZZpNfKws8eo +f7wn9gECgYEA0I4MLibtJZrlvSJVi7ixn5fLSnl0h7DqwIivHhGZP2xHIzUzNh2/ +rW6+7DKQqw6/K2BCkO7gNIBvKEe3XdlzP5+3PZeChFHOt+vWuEVo4v52IiYY7ing +aL8HhLVBUMROm2dpVJlVFtfRVb6SxJtINyQVLB0sM44fYn8ZSh+aEPkCgYEAyM7f +4BEsEQa5MHZWG48meZKBZCkxcBWbu8aHCIrd8TILepRhMPM9+j6ex+e9jnOPzmHv +xehMyvqFRZ9OCrgdxw4vyJpl0l5KObsBZvIJXmuxclMXOAUwHdx+lFLpAJDC9QVm +YDvAxD3Sy67rOWfECdhD9eM+9j14hchmj9UYb4UCgYAkXRAsn+brjqWOI8Vstkhq +PkpY8vJpkmRsK6j1AjaJQ3Tn46fJQMiiEdRCVNK6sLiOdJtGsA/xt48qI88KExcw +OcX2fEtqjOURVpK60IdoRNwOOjxQkoapXN2PuxbnYUMff5lzAcU/VWQPoknu8/BU +hPsYFQIW/ynjv6uGLBpt6QKBgGRVydMBgY04WMv4NOosSsMwCurrEkK46UmX1tzT +1jWwFcA356A3yd4B8ABesH4/C7nJga7XdZduOa0h/jKo8GgHlKSdUQceCeRypi6z +/S5qjQ1cqxtYrEQfajfefYHE00TuX8rx0E29vlf7nJjgWjm5D6wK0ejjqhbenTB8 +/2qpAoGAZCXNflc2JVU1Lx235KdJg8jIdo8rutowBQ7GtDrJXzqPpIwv6M9hEsnr +I0EWIHsyNzyT3eqsXIO2ZIFKtqN83bPEFmVkKXAUV6lb0/nVSLrRTppFWlNyg1o5 +FYnq6/jDVxCbWmmP2u4TT557gMqao0DaJstf/NSXlK0bhA2B64M= +-----END RSA PRIVATE KEY-----`) + + testRegistryAddonRootCASecretName = "registry-addon-root-ca" +) + +var _ = Describe("Test EnsureCASecretForCluster", func() { + clientScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) + + It("CA Secret should be created for a cluster", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + + globalSecret := testGlobalRegistryAddonTLSCertificate() + Expect(c.Create(ctx, globalSecret)).To(Succeed()) + + Expect(EnsureCASecretForCluster(ctx, c, cluster)).To(Succeed()) + caSecretKey := ctrlclient.ObjectKey{ + Name: fmt.Sprintf("%s-registry-addon-ca", cluster.Name), + Namespace: corev1.NamespaceDefault, + } + caSecret := &corev1.Secret{} + Expect(c.Get(ctx, caSecretKey, caSecret)).To(Succeed()) + Expect(caSecret.OwnerReferences).To( + ContainElement( + metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: clusterv1.ClusterKind, + Name: cluster.Name, + UID: cluster.UID, + }, + ), + ) + Expect(caSecret.Data["ca.crt"]).To(Equal(testCrt)) + Expect(caSecret.Data[corev1.TLSCertKey]).To(BeEmpty()) + Expect(caSecret.Data[corev1.TLSPrivateKeyKey]).To(BeEmpty()) + }) + It("CA Secret should not be created when missing global CA Secret", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + + Expect(EnsureCASecretForCluster(ctx, c, cluster)).To( + MatchError("error getting registry addon root CA secret: " + + "secrets \"registry-addon-root-ca\" not found", + ), + ) + caSecretKey := ctrlclient.ObjectKey{ + Name: fmt.Sprintf("%s-registry-addon-ca", cluster.Name), + Namespace: corev1.NamespaceDefault, + } + caSecret := &corev1.Secret{} + Expect(c.Get(ctx, caSecretKey, caSecret)).ToNot(Succeed()) + }) + + AfterEach(func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + globalSecret := testGlobalRegistryAddonTLSCertificate() + Expect(c.Delete(ctx, globalSecret)).To( + Or( + Succeed(), + MatchError("secrets \"registry-addon-root-ca\" not found"), + ), + ) + }) +}) + +var _ = Describe("Test EnsureRegistryServerCertificateSecretOnRemoteCluster", func() { + clientScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) + + It("TLS Secret should be created/updated on the remote cluster", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + Expect(helpers.TestEnv.WithFakeRemoteClusterClient(cluster)).To(Succeed()) + + globalSecret := testGlobalRegistryAddonTLSCertificate() + Expect(c.Create(ctx, globalSecret)).To(Succeed()) + + remoteTLSSecretName := "registry-tls" + opts := &EnsureCertificateOpts{ + RemoteSecretKey: ctrlclient.ObjectKey{ + Name: remoteTLSSecretName, + Namespace: corev1.NamespaceDefault, + }, + Spec: CertificateSpec{ + CommonName: "registry", + DNSNames: []string{"registry"}, + IPAddresses: []string{"127.0.0.1"}, + Duration: 24 * 365 * time.Hour, + }, + } + remoteTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: remoteTLSSecretName, + Namespace: corev1.NamespaceDefault, + }, + } + + remoteClient, err := remote.NewClusterClient(ctx, "", c, ctrlclient.ObjectKeyFromObject(cluster)) + Expect(err).To(BeNil()) + + // Create the initial TLS secret on the remote cluster. + Expect(EnsureRegistryServerCertificateSecretOnRemoteCluster(ctx, c, cluster, opts)).To(Succeed()) + err = remoteClient.Get(ctx, ctrlclient.ObjectKeyFromObject(remoteTLSSecret), remoteTLSSecret) + Expect(err).To(BeNil()) + + initialCA := remoteTLSSecret.Data["ca.crt"] + initialCert := remoteTLSSecret.Data["tls.crt"] + initialKey := remoteTLSSecret.Data["tls.key"] + Expect(initialCA).To(Equal(testCrt)) + Expect(initialCert).ToNot(BeEmpty()) + Expect(initialKey).ToNot(BeEmpty()) + + // Run the function again to update the TLS secret. + Expect(EnsureRegistryServerCertificateSecretOnRemoteCluster(ctx, c, cluster, opts)).To(Succeed()) + err = remoteClient.Get(ctx, ctrlclient.ObjectKeyFromObject(remoteTLSSecret), remoteTLSSecret) + Expect(err).To(BeNil()) + + updatedCA := remoteTLSSecret.Data["ca.crt"] + updatedCert := remoteTLSSecret.Data["tls.crt"] + updatedKey := remoteTLSSecret.Data["tls.key"] + Expect(updatedCA).To(Equal(testCrt)) + Expect(updatedCert).ToNot(BeEmpty()) + Expect(updatedKey).ToNot(BeEmpty()) + Expect(updatedCert).ToNot(Equal(initialCert)) + Expect(updatedKey).ToNot(Equal(initialKey)) + }) + + It("Should error when missing global CA Secret", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + + // Expect this to fail because the global CA secret is missing. + Expect(EnsureRegistryServerCertificateSecretOnRemoteCluster(ctx, c, cluster, nil)).To( + MatchError("failed to get TLS secret used to sign the certificate: " + + "error getting registry addon root CA secret: " + + "secrets \"registry-addon-root-ca\" not found"), + ) + }) + + AfterEach(func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + globalSecret := testGlobalRegistryAddonTLSCertificate() + Expect(c.Delete(ctx, globalSecret)).To( + Or( + Succeed(), + MatchError("secrets \"registry-addon-root-ca\" not found"), + ), + ) + }) +}) + +func testGlobalRegistryAddonTLSCertificate() *corev1.Secret { + secretData := map[string][]byte{ + "ca.crt": testCrt, + "tls.crt": testCrt, + "tls.key": testKey, + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: testRegistryAddonRootCASecretName, + Namespace: corev1.NamespaceDefault, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} diff --git a/pkg/handlers/generic/lifecycle/registry/utils/tls_test.go b/pkg/handlers/generic/lifecycle/registry/utils/tls_test.go new file mode 100644 index 000000000..734eaa32b --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/utils/tls_test.go @@ -0,0 +1,136 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" +) + +func Test_generateCertificateData(t *testing.T) { + t.Parallel() + tests := []struct { + name string + opts *EnsureCertificateOpts + wantErr error + }{ + { + name: "valid certificate with a set duration", + opts: &EnsureCertificateOpts{ + Spec: CertificateSpec{ + CommonName: "common-name", + DNSNames: []string{"myregistry.example.com"}, + IPAddresses: []string{"192.168.0.20"}, + Duration: 30 * 24 * time.Hour, + }, + }, + }, + { + name: "valid certificate with a default duration", + opts: &EnsureCertificateOpts{ + Spec: CertificateSpec{ + CommonName: "common-name", + DNSNames: []string{"myregistry.example.com"}, + IPAddresses: []string{"192.168.0.20"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + globalCASecret := testGlobalRegistryAddonTLSCertificate() + serverCertPEM, serverKeyPEM, caCertPEM, err := generateCertificateData(globalCASecret, tt.opts) + if tt.wantErr != nil { + require.EqualError(t, err, tt.wantErr.Error()) + } else { + require.NoError(t, err) + } + require.NotEmpty(t, serverCertPEM) + require.NotEmpty(t, serverKeyPEM) + require.NotEmpty(t, caCertPEM) + + assertCertificateData(t, globalCASecret, serverCertPEM, serverKeyPEM, caCertPEM, tt.opts.Spec) + }) + } +} + +func assertCertificateData( + t *testing.T, + globalCASecret *corev1.Secret, + serverCertPEM, serverKeyPEM, caCertPEM []byte, + opts CertificateSpec, +) { + t.Helper() + + rootCACertBytes := globalCASecret.Data[caCrtKey] + rootCACert := parseCertPEM(t, rootCACertBytes) + + assert.Equal(t, rootCACertBytes, caCertPEM) + + // Decode and parse server cert + cert := parseCertPEM(t, serverCertPEM) + key := parseKeyPEM(t, serverKeyPEM) + + require.NoError(t, cert.CheckSignatureFrom(rootCACert), "server cert not signed by CA") + assert.Equal(t, opts.CommonName, cert.Subject.CommonName) + assert.Equal(t, opts.DNSNames, cert.DNSNames) + gotCertIPAddresses := make([]string, 0, len(opts.IPAddresses)) + for _, ipAddress := range cert.IPAddresses { + gotCertIPAddresses = append(gotCertIPAddresses, ipAddress.String()) + } + assert.Equal(t, opts.IPAddresses, gotCertIPAddresses) + assert.GreaterOrEqual(t, key.N.BitLen(), 2048) + + wantDuration := opts.Duration + if wantDuration == 0 { + wantDuration = defaultCertificateDuration + } + ttl := cert.NotAfter.Sub(cert.NotBefore) + // Assert that the duration is about what was requested + assert.InDelta(t, wantDuration, ttl, float64(defaultCertificateNotBeforeSkew)) +} + +// parseCertPEM takes a PEM‐encoded cert and returns the parsed *x509.Certificate. +func parseCertPEM( + t *testing.T, + pemBytes []byte, +) *x509.Certificate { + t.Helper() + + block, rest := pem.Decode(pemBytes) + require.NotNil(t, block, "failed to decode PEM block") + require.Equal(t, "CERTIFICATE", block.Type, "expected PEM block type to be CERTIFICATE") + assert.Empty(t, rest, "extra data after first PEM block") + + cert, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err, "failed to parse certificate") + + return cert +} + +func parseKeyPEM( + t *testing.T, + pemBytes []byte, +) *rsa.PrivateKey { + t.Helper() + + block, rest := pem.Decode(pemBytes) + require.NotNil(t, block, "failed to decode PEM block") + assert.Empty(t, rest, "extra data after first PEM block") + + key, err := x509.ParsePKCS1PrivateKey(block.Bytes) + require.NoError(t, err, "failed to parse private key") + + return key +} diff --git a/pkg/handlers/generic/lifecycle/registry/utils/utils_suite_test.go b/pkg/handlers/generic/lifecycle/registry/utils/utils_suite_test.go new file mode 100644 index 000000000..b3f51df1e --- /dev/null +++ b/pkg/handlers/generic/lifecycle/registry/utils/utils_suite_test.go @@ -0,0 +1,17 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// TestUtils is the entrypoint for integration (envtest) tests. +func TestUtils(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Utils") +} diff --git a/pkg/handlers/generic/mutation/mirrors/inject.go b/pkg/handlers/generic/mutation/mirrors/inject.go index d33f5267f..fe1231711 100644 --- a/pkg/handlers/generic/mutation/mirrors/inject.go +++ b/pkg/handlers/generic/mutation/mirrors/inject.go @@ -5,6 +5,7 @@ package mirrors import ( "context" + "errors" "fmt" corev1 "k8s.io/api/core/v1" @@ -143,7 +144,11 @@ func (h *globalMirrorPatchHandler) Mutate( return err } - registryConfig, err := containerdConfigFromRegistryAddon(cluster) + registryConfig, err := containerdConfigFromRegistryAddon( + ctx, + h.client, + cluster, + ) if err != nil { return err } @@ -262,16 +267,27 @@ func containerdConfigFromImageRegistry( return configWithOptionalCACert, nil } -func containerdConfigFromRegistryAddon(cluster *clusterv1.Cluster) (containerdConfig, error) { +func containerdConfigFromRegistryAddon( + ctx context.Context, + c ctrlclient.Client, + cluster *clusterv1.Cluster, +) (containerdConfig, error) { serviceIP, err := registryutils.ServiceIPForCluster(cluster) if err != nil { return containerdConfig{}, fmt.Errorf("error getting service IP for the registry addon: %w", err) } - + secret, err := handlersutils.SecretForClusterRegistryAddonCA(ctx, c, cluster) + if err != nil { + return containerdConfig{}, fmt.Errorf("error getting CA secret for registry addon: %w", err) + } + if !secretHasCACert(secret) { + return containerdConfig{}, errors.New("CA certificate not found in the secret") + } config := containerdConfig{ - // FIXME: Generate a self-signed CA. - URL: fmt.Sprintf("http://%s", serviceIP), - Mirror: true, + URL: fmt.Sprintf("https://%s", serviceIP), + Mirror: true, + CASecretName: secret.Name, + CACert: string(secret.Data[secretKeyForCACert]), } return config, nil diff --git a/pkg/handlers/generic/mutation/mirrors/inject_test.go b/pkg/handlers/generic/mutation/mirrors/inject_test.go index e0b12c6c7..057a0c4d5 100644 --- a/pkg/handlers/generic/mutation/mirrors/inject_test.go +++ b/pkg/handlers/generic/mutation/mirrors/inject_test.go @@ -4,6 +4,7 @@ package mirrors import ( + "context" "fmt" "testing" @@ -19,6 +20,8 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" @@ -30,6 +33,8 @@ import ( const ( validMirrorCASecretName = "myregistry-mirror-cacert" validMirrorNoCASecretName = "myregistry-mirror-no-cacert" + + registryAddonCAForCluster = "test-cluster-registry-addon-ca" ) func TestMirrorsPatch(t *testing.T) { @@ -358,6 +363,9 @@ var _ = Describe("Generate Global mirror patches", func() { gomega.HaveKeyWithValue( "path", "/etc/containerd/certs.d/_default/hosts.toml", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/192.168.0.20/ca.crt", + ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", ), @@ -391,6 +399,9 @@ var _ = Describe("Generate Global mirror patches", func() { gomega.HaveKeyWithValue( "path", "/etc/containerd/certs.d/_default/hosts.toml", ), + gomega.HaveKeyWithValue( + "path", "/etc/containerd/certs.d/192.168.0.20/ca.crt", + ), gomega.HaveKeyWithValue( "path", "/etc/caren/containerd/patches/registry-config.toml", ), @@ -406,11 +417,15 @@ var _ = Describe("Generate Global mirror patches", func() { gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(client.Create( ctx, - newMirrorSecretWithCA(validMirrorCASecretName, request.Namespace), + newRegistrySecretWithCA(validMirrorCASecretName), + )).To(gomega.BeNil()) + gomega.Expect(client.Create( + ctx, + newRegistrySecretWithoutCA(validMirrorNoCASecretName), )).To(gomega.BeNil()) gomega.Expect(client.Create( ctx, - newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace), + newRegistrySecretWithCA(registryAddonCAForCluster), )).To(gomega.BeNil()) gomega.Expect(client.Create( @@ -437,11 +452,15 @@ var _ = Describe("Generate Global mirror patches", func() { gomega.Expect(err).To(gomega.BeNil()) gomega.Expect(client.Delete( ctx, - newMirrorSecretWithCA(validMirrorCASecretName, request.Namespace), + newRegistrySecretWithCA(validMirrorCASecretName), + )).To(gomega.BeNil()) + gomega.Expect(client.Delete( + ctx, + newRegistrySecretWithoutCA(validMirrorNoCASecretName), )).To(gomega.BeNil()) gomega.Expect(client.Delete( ctx, - newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace), + newRegistrySecretWithCA(registryAddonCAForCluster), )).To(gomega.BeNil()) gomega.Expect(client.Delete( @@ -464,57 +483,24 @@ var _ = Describe("Generate Global mirror patches", func() { } }) -func newMirrorSecretWithCA(name, namespace string) *corev1.Secret { - secretData := map[string][]byte{ - "ca.crt": []byte("myCACert"), - } - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: secretData, - Type: corev1.SecretTypeOpaque, - } -} - -func newMirrorSecretWithoutCA(name, namespace string) *corev1.Secret { - secretData := map[string][]byte{ - "username": []byte("user"), - "password": []byte("pass"), - } - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Data: secretData, - Type: corev1.SecretTypeOpaque, - } -} - func Test_containerdConfigFromRegistryAddon(t *testing.T) { t.Parallel() tests := []struct { name string + c ctrlclient.Client cluster *clusterv1.Cluster want containerdConfig wantErr error }{ { - name: "valid input", + name: "valid input with a CA certificate", + c: fake.NewClientBuilder().WithObjects( + newRegistrySecretWithCA(registryAddonCAForCluster), + ).Build(), cluster: &clusterv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ - Name: request.ClusterName, - Namespace: request.Namespace, + Name: "test-cluster", + Namespace: corev1.NamespaceDefault, }, Spec: clusterv1.ClusterSpec{ ClusterNetwork: &clusterv1.ClusterNetwork{ @@ -525,16 +511,19 @@ func Test_containerdConfigFromRegistryAddon(t *testing.T) { }, }, want: containerdConfig{ - URL: "http://192.168.0.20", - Mirror: true, + URL: "https://192.168.0.20", + Mirror: true, + CASecretName: "test-cluster-registry-addon-ca", + CACert: "myCACert", }, }, { - name: "missing Services CIDR", + name: "error: missing Services CIDR", + c: fake.NewClientBuilder().Build(), cluster: &clusterv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ - Name: request.ClusterName, - Namespace: request.Namespace, + Name: "test-cluster", + Namespace: corev1.NamespaceDefault, }, Spec: clusterv1.ClusterSpec{ ClusterNetwork: &clusterv1.ClusterNetwork{}, @@ -546,12 +535,32 @@ func Test_containerdConfigFromRegistryAddon(t *testing.T) { "unexpected empty service Subnets", ), }, + { + name: "error: missing certificate in the secret", + // The suffix "-ca" is misleading here because we expect the generated secret to always have a CA. + c: fake.NewClientBuilder().WithObjects( + newRegistrySecretWithoutCA("test-cluster-registry-addon-ca"), + ).Build(), + cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: corev1.NamespaceDefault, + }, + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + Services: &clusterv1.NetworkRanges{ + CIDRBlocks: []string{"192.168.0.1/16"}, + }, + }, + }, + }, + wantErr: fmt.Errorf("CA certificate not found in the secret"), + }, } - for idx := range tests { - tt := tests[idx] + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got, err := containerdConfigFromRegistryAddon(tt.cluster) + got, err := containerdConfigFromRegistryAddon(context.Background(), tt.c, tt.cluster) if tt.wantErr != nil { require.EqualError(t, err, tt.wantErr.Error()) } else { @@ -637,3 +646,40 @@ func Test_needContainerdConfiguration(t *testing.T) { }) } } + +func newRegistrySecretWithCA(name string) *corev1.Secret { + secretData := map[string][]byte{ + "ca.crt": []byte("myCACert"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: corev1.NamespaceDefault, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} + +func newRegistrySecretWithoutCA(name string) *corev1.Secret { + secretData := map[string][]byte{ + "username": []byte("user"), + "password": []byte("pass"), + } + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: corev1.NamespaceDefault, + }, + Data: secretData, + Type: corev1.SecretTypeOpaque, + } +} diff --git a/pkg/handlers/utils/owner_reference.go b/pkg/handlers/utils/owner_reference.go new file mode 100644 index 000000000..ad1c222e3 --- /dev/null +++ b/pkg/handlers/utils/owner_reference.go @@ -0,0 +1,72 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/external" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// EnsureClusterOwnerReferenceForObject ensures that OwnerReference of the cluster is added on provided object. +func EnsureClusterOwnerReferenceForObject( + ctx context.Context, + cl ctrlclient.Client, + objectRef corev1.TypedLocalObjectReference, + cluster *clusterv1.Cluster, +) error { + targetObj, err := getResourceFromTypedLocalObjectReference( + ctx, + cl, + objectRef, + cluster.Namespace, + ) + if err != nil { + return err + } + + err = controllerutil.SetOwnerReference(cluster, targetObj, cl.Scheme()) + if err != nil { + return fmt.Errorf("failed to set cluster's owner reference on object: %w", err) + } + + err = cl.Update(ctx, targetObj) + if err != nil { + return fmt.Errorf("failed to update object with cluster's owner reference: %w", err) + } + return nil +} + +// getResourceFromTypedLocalObjectReference gets the resource from the provided TypedLocalObjectReference. +func getResourceFromTypedLocalObjectReference( + ctx context.Context, + cl ctrlclient.Client, + typedLocalObjectRef corev1.TypedLocalObjectReference, + ns string, +) (*unstructured.Unstructured, error) { + apiVersion := corev1.SchemeGroupVersion.String() + if typedLocalObjectRef.APIGroup != nil { + apiVersion = *typedLocalObjectRef.APIGroup + } + + objectRef := &corev1.ObjectReference{ + APIVersion: apiVersion, + Kind: typedLocalObjectRef.Kind, + Name: typedLocalObjectRef.Name, + Namespace: ns, + } + + targetObj, err := external.Get(ctx, cl, objectRef) + if err != nil { + return nil, fmt.Errorf("failed to get resource from object reference: %w", err) + } + + return targetObj, nil +} diff --git a/pkg/handlers/utils/secrets_test.go b/pkg/handlers/utils/owner_reference_test.go similarity index 98% rename from pkg/handlers/utils/secrets_test.go rename to pkg/handlers/utils/owner_reference_test.go index cf8f8c496..e0e3b6cb5 100644 --- a/pkg/handlers/utils/secrets_test.go +++ b/pkg/handlers/utils/owner_reference_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 Nutanix. All rights reserved. +// Copyright 2025 Nutanix. All rights reserved. // SPDX-License-Identifier: Apache-2.0 package utils diff --git a/pkg/handlers/utils/secrets.go b/pkg/handlers/utils/secrets.go index ec1c129ae..883959db7 100644 --- a/pkg/handlers/utils/secrets.go +++ b/pkg/handlers/utils/secrets.go @@ -9,9 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/external" "sigs.k8s.io/cluster-api/controllers/remote" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -20,6 +18,10 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/k8s/client" ) +const ( + registryAddonRootCASecretName = "registry-addon-root-ca" +) + // CopySecretToRemoteCluster will get the Secret from srcSecretName // and create it on the remote cluster, copying Data and StringData to dstSecretKey Secret. func CopySecretToRemoteCluster( @@ -34,7 +36,7 @@ func CopySecretToRemoteCluster( return err } - credentialsOnRemote := &corev1.Secret{ + secretOnRemote := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "Secret", @@ -58,7 +60,7 @@ func CopySecretToRemoteCluster( return fmt.Errorf("error creating namespace on the remote cluster: %w", err) } - err = client.ServerSideApply(ctx, remoteClient, credentialsOnRemote, client.ForceOwnership) + err = client.ServerSideApply(ctx, remoteClient, secretOnRemote, client.ForceOwnership) if err != nil { return fmt.Errorf("error creating Secret on the remote cluster: %w", err) } @@ -66,79 +68,58 @@ func CopySecretToRemoteCluster( return nil } -// EnsureClusterOwnerReferenceForObject ensures that OwnerReference of the cluster is added on provided object. -func EnsureClusterOwnerReferenceForObject( +// EnsureSecretOnRemoteCluster ensures that the given Secret exists on the remote cluster. +func EnsureSecretOnRemoteCluster( ctx context.Context, cl ctrlclient.Client, - objectRef corev1.TypedLocalObjectReference, + secret *corev1.Secret, cluster *clusterv1.Cluster, ) error { - targetObj, err := GetResourceFromTypedLocalObjectReference( - ctx, - cl, - objectRef, - cluster.Namespace, - ) + clusterKey := ctrlclient.ObjectKeyFromObject(cluster) + remoteClient, err := remote.NewClusterClient(ctx, "", cl, clusterKey) if err != nil { - return err + return fmt.Errorf("error creating client for remote cluster: %w", err) } - err = controllerutil.SetOwnerReference(cluster, targetObj, cl.Scheme()) + err = EnsureNamespaceWithName(ctx, remoteClient, secret.Namespace) if err != nil { - return fmt.Errorf("failed to set cluster's owner reference on object: %w", err) + return fmt.Errorf("error creating namespace on the remote cluster: %w", err) } - err = cl.Update(ctx, targetObj) + err = client.ServerSideApply(ctx, remoteClient, secret, client.ForceOwnership) if err != nil { - return fmt.Errorf("failed to update object with cluster's owner reference: %w", err) + return fmt.Errorf("error creating Secret on the remote cluster: %w", err) } + return nil } -// GetResourceFromTypedLocalObjectReference gets the resource from the provided TypedLocalObjectReference. -func GetResourceFromTypedLocalObjectReference( +func EnsureSecretForLocalCluster( ctx context.Context, cl ctrlclient.Client, - typedLocalObjectRef corev1.TypedLocalObjectReference, - ns string, -) (*unstructured.Unstructured, error) { - apiVersion := corev1.SchemeGroupVersion.String() - if typedLocalObjectRef.APIGroup != nil { - apiVersion = *typedLocalObjectRef.APIGroup + secret *corev1.Secret, + cluster *clusterv1.Cluster, +) error { + if secret.Namespace != "" && + secret.Namespace != cluster.Namespace { + return fmt.Errorf( + "secret namespace %q does not match cluster namespace %q", + secret.Namespace, + cluster.Namespace, + ) } - objectRef := &corev1.ObjectReference{ - APIVersion: apiVersion, - Kind: typedLocalObjectRef.Kind, - Name: typedLocalObjectRef.Name, - Namespace: ns, + err := controllerutil.SetOwnerReference(cluster, secret, cl.Scheme()) + if err != nil { + return fmt.Errorf("failed to set cluster's owner reference on Secret: %w", err) } - targetObj, err := external.Get(ctx, cl, objectRef) + err = client.ServerSideApply(ctx, cl, secret, client.ForceOwnership) if err != nil { - return nil, fmt.Errorf("failed to get resource from object reference: %w", err) + return fmt.Errorf("error creating Secret for cluster: %w", err) } - return targetObj, nil -} - -func getSecretForCluster( - ctx context.Context, - c ctrlclient.Client, - secretName string, - cluster *clusterv1.Cluster, -) (*corev1.Secret, error) { - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: cluster.Namespace, - }, - } - return secret, c.Get(ctx, ctrlclient.ObjectKeyFromObject(secret), secret) + return nil } // SecretForImageRegistryCredentials returns the Secret for the given ImageRegistryCredentials. @@ -171,3 +152,59 @@ func SecretNameForImageRegistryCredentials(credentials *v1alpha1.RegistryCredent } return credentials.SecretRef.Name } + +func SecretForRegistryAddonRootCA( + ctx context.Context, + c ctrlclient.Reader, +) (*corev1.Secret, error) { + secret, err := getSecret(ctx, c, registryAddonRootCASecretName, GetDeploymentNamespace()) + if err != nil { + return nil, fmt.Errorf("error getting registry addon root CA secret: %w", err) + } + return secret, nil +} + +func SecretForClusterRegistryAddonCA( + ctx context.Context, + c ctrlclient.Reader, + cluster *clusterv1.Cluster, +) (*corev1.Secret, error) { + secret, err := getSecretForCluster(ctx, c, SecretNameForRegistryAddonCA(cluster), cluster) + if err != nil { + return nil, fmt.Errorf("error getting registry addon CA secret for cluster: %w", err) + } + return secret, nil +} + +// SecretNameForRegistryAddonCA returns the name of the registry addon CA Secret. +func SecretNameForRegistryAddonCA(cluster *clusterv1.Cluster) string { + return fmt.Sprintf("%s-registry-addon-ca", cluster.Name) +} + +func getSecretForCluster( + ctx context.Context, + c ctrlclient.Reader, + secretName string, + cluster *clusterv1.Cluster, +) (*corev1.Secret, error) { + return getSecret(ctx, c, secretName, cluster.Namespace) +} + +func getSecret( + ctx context.Context, + c ctrlclient.Reader, + secretName string, + secretNamespace string, +) (*corev1.Secret, error) { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: secretNamespace, + }, + } + return secret, c.Get(ctx, ctrlclient.ObjectKeyFromObject(secret), secret) +} diff --git a/pkg/handlers/utils/secrets_integration_test.go b/pkg/handlers/utils/secrets_integration_test.go new file mode 100644 index 000000000..034caf529 --- /dev/null +++ b/pkg/handlers/utils/secrets_integration_test.go @@ -0,0 +1,215 @@ +// Copyright 2025 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/remote" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +var _ = Describe("Test EnsureSecretOnRemoteCluster", func() { + clientScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) + + It("Secret should be created in the default namespace on the remote cluster", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + Expect(helpers.TestEnv.WithFakeRemoteClusterClient(cluster)).To(Succeed()) + + remoteClient, err := remote.NewClusterClient(ctx, "", c, ctrlclient.ObjectKeyFromObject(cluster)) + Expect(err).To(BeNil()) + + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: corev1.NamespaceDefault, + }, + Data: map[string][]byte{ + "key": []byte("value"), + }, + } + + Expect(EnsureSecretOnRemoteCluster(ctx, c, secret, cluster)).To(Succeed()) + + // Verify that the secret was created on the remote cluster. + Expect(remoteClient.Get(ctx, ctrlclient.ObjectKeyFromObject(secret), secret)).To(Succeed()) + Expect(secret.Name).To(Equal("test-secret")) + Expect(secret.Namespace).To(Equal(corev1.NamespaceDefault)) + Expect(secret.Data).To(Equal(secret.Data)) + }) + + It("Secret should be created in a new namespace on the remote cluster", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + Expect(helpers.TestEnv.WithFakeRemoteClusterClient(cluster)).To(Succeed()) + + remoteClient, err := remote.NewClusterClient(ctx, "", c, ctrlclient.ObjectKeyFromObject(cluster)) + Expect(err).To(BeNil()) + + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-namespace", + }, + Data: map[string][]byte{ + "key": []byte("value"), + }, + } + + Expect(EnsureSecretOnRemoteCluster(ctx, c, secret, cluster)).To(Succeed()) + + // Verify that the secret was created on the remote cluster. + Expect(remoteClient.Get(ctx, ctrlclient.ObjectKeyFromObject(secret), secret)).To(Succeed()) + Expect(secret.Name).To(Equal("test-secret")) + Expect(secret.Namespace).To(Equal("test-namespace")) + Expect(secret.Data).To(Equal(secret.Data)) + }) + + It("Should error if can't get remote cluster's kubeconfig", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-namespace", + }, + } + + Expect(EnsureSecretOnRemoteCluster(ctx, c, secret, cluster)).To( + MatchError( + fmt.Sprintf("error creating client for remote cluster: "+ + "failed to retrieve kubeconfig secret for Cluster default/%s: secrets \"%s-kubeconfig\" not found", + cluster.Name, cluster.Name), + ), + ) + }) +}) + +var _ = Describe("Test EnsureSecretForLocalCluster", func() { + clientScheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) + + It("Secret should be created in the cluster", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: corev1.NamespaceDefault, + }, + Data: map[string][]byte{ + "key": []byte("value"), + }, + } + + Expect(EnsureSecretForLocalCluster(ctx, c, secret, cluster)).To(Succeed()) + + // Verify that the secret was created on the local cluster. + Expect(c.Get(ctx, ctrlclient.ObjectKeyFromObject(secret), secret)).To(Succeed()) + Expect(secret.OwnerReferences).To( + ContainElement( + metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: clusterv1.ClusterKind, + Name: cluster.Name, + UID: cluster.UID, + }, + ), + ) + }) + It("Secret error if namespaces don't match", func(ctx SpecContext) { + c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme) + Expect(err).To(BeNil()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-cluster-", + Namespace: corev1.NamespaceDefault, + }, + } + Expect(c.Create(ctx, cluster)).To(Succeed()) + + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-namespace", + }, + Data: map[string][]byte{ + "key": []byte("value"), + }, + } + + Expect(EnsureSecretForLocalCluster(ctx, c, secret, cluster)).To( + MatchError("secret namespace \"test-namespace\" does not match cluster namespace \"default\""), + ) + }) +}) diff --git a/pkg/handlers/utils/utils.go b/pkg/handlers/utils/utils.go index c287abf1d..4e41e3b6b 100644 --- a/pkg/handlers/utils/utils.go +++ b/pkg/handlers/utils/utils.go @@ -200,17 +200,21 @@ func RetrieveValuesTemplate( } func SetTLSConfigForHelmChartProxyIfNeeded(hcp *caaphv1.HelmChartProxy) { - // this is set as an environment variable from the downward API on deployment - deploymentNS := os.Getenv("POD_NAMESPACE") - if deploymentNS == "" { - deploymentNS = metav1.NamespaceDefault - } if strings.Contains(hcp.Spec.RepoURL, "helm-repository") { hcp.Spec.TLSConfig = &caaphv1.TLSConfig{ CASecretRef: &corev1.SecretReference{ Name: "helm-repository-tls", - Namespace: deploymentNS, + Namespace: GetDeploymentNamespace(), }, } } } + +func GetDeploymentNamespace() string { + // this is set as an environment variable from the downward API on deployment + deploymentNamespace := os.Getenv("POD_NAMESPACE") + if deploymentNamespace == "" { + deploymentNamespace = metav1.NamespaceDefault + } + return deploymentNamespace +} diff --git a/test/helpers/envtest.go b/test/helpers/envtest.go index 0e0f294aa..8d322c509 100644 --- a/test/helpers/envtest.go +++ b/test/helpers/envtest.go @@ -20,11 +20,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/klog/v2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/kubeconfig" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -146,6 +150,36 @@ func (t *TestEnvironment) GetK8sClientWithScheme( return client.New(t.GetConfig(), client.Options{Scheme: clientScheme}) } +// WithFakeRemoteClusterClient creates a fake remote cluster client Secret pointing to the test API server. +func (t *TestEnvironment) WithFakeRemoteClusterClient(cluster *clusterv1.Cluster) error { + clientScheme := runtime.NewScheme() + utilruntime.Must(scheme.AddToScheme(clientScheme)) + utilruntime.Must(clusterv1.AddToScheme(clientScheme)) + + cfg := t.GetConfig() + c, err := client.New(cfg, client.Options{Scheme: clientScheme}) + if err != nil { + return err + } + + kubeconfigBytes := kubeconfig.FromEnvTestConfig(cfg, cluster) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-kubeconfig", cluster.Name), + Namespace: cluster.Namespace, + }, + Data: map[string][]byte{ + "value": kubeconfigBytes, + }, + } + err = controllerutil.SetOwnerReference(cluster, secret, c.Scheme()) + if err != nil { + return fmt.Errorf("failed to set cluster's owner reference on kubeconfig secret: %w", err) + } + + return c.Create(context.Background(), secret) +} + // StartManager starts the test controller against the local API server. func (t *TestEnvironment) StartManager(ctx context.Context) error { return t.Start(ctx)