feat: in-cluster zot registry as part of deployment

Makefile

@@ -121,8 +121,8 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform
 .PHONY: build-installer
 build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment.
 	mkdir -p dist
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default > dist/install.yaml
+	$(call set-images)
+	$(KUSTOMIZE) build config/default-zot-https > dist/install.yaml
 
 ##@ Deployment
 
@@ -138,14 +138,26 @@ install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(IGNORE_NOT_FOUND) -f -
 
-.PHONY: deploy
-deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+.PHONY: deploy-dev
+deploy-dev: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. In-cluster zot registry is accessible via http. If you need https, use deploy target.
+	$(call set-images)
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
+.PHONY: undeploy-dev
+undeploy-dev: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(IGNORE_NOT_FOUND) -f -
+
+.PHONY: deploy
+deploy: deploy-cert-manager manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. In-cluster zot registry is accessible via https. If you need http, use deploy-dev target.
+	$(call set-images)
+	$(KUSTOMIZE) build config/default-zot-https | $(KUBECTL) apply -f -
+
+# Undeploy target undeploys the controller, its zot regostry and related certificates. 
+# However, it does not undeploy the cert-manager, which might still be needed by other applications in the cluster.
+# If you wish to undeploy cert manager as well, execute 'make undeploy-cert-manager' in addition.
 .PHONY: undeploy
 undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(IGNORE_NOT_FOUND) -f -
+	$(KUSTOMIZE) build config/default-zot-https | $(KUBECTL) delete --ignore-not-found=$(IGNORE_NOT_FOUND) -f -
 
 ##@ Dependencies
 
@@ -167,6 +179,14 @@ CONTROLLER_TOOLS_VERSION ?= v0.16.0
 ENVTEST_VERSION ?= release-0.18
 GOLANGCI_LINT_VERSION ?= v1.61.0
 
+## ZOT OCI Registry
+ZOT_VERSION ?= v2.1.2
+ZOT_IMG ?= ghcr.io/project-zot/zot-minimal:$(ZOT_VERSION)
+
+## cert-manager
+CERT-MANAGER_VERSION ?= v1.16.3
+CERT-MANAGER_YAML ?= https://github.com/cert-manager/cert-manager/releases/download/$(CERT-MANAGER_VERSION)/cert-manager.yaml
+
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
 $(KUSTOMIZE): $(LOCALBIN)
@@ -182,6 +202,17 @@ envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION))
 
+.PHONY: deploy-cert-manager
+deploy-cert-manager: ## Deploy cert-manager to the K8s cluster specified in ~/.kube/config.
+	$(KUBECTL) apply -f $(CERT-MANAGER_YAML)
+	$(KUBECTL) wait --for=condition=Available=True Deployment/cert-manager -n cert-manager --timeout=60s
+	$(KUBECTL) wait --for=condition=Available=True Deployment/cert-manager-webhook -n cert-manager --timeout=60s
+	$(KUBECTL) wait --for=condition=Available=True Deployment/cert-manager-cainjector -n cert-manager --timeout=60s
+
+.PHONY: undeploy-cert-manager
+undeploy-cert-manager: ## Undeploy cert-manager from the K8s cluster specified in ~/.kube/config.
+	$(KUBECTL) delete --ignore-not-found=$(IGNORE_NOT_FOUND) -f $(CERT-MANAGER_YAML)
+
 .PHONY: golangci-lint
 golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
 $(GOLANGCI_LINT): $(LOCALBIN)
@@ -200,3 +231,10 @@ GOBIN=$(LOCALBIN) go install $${package} ;\
 mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\
 }
 endef
+
+# set-images will set use kustomize to set the specified images for the controller and zot registry
+define set-images
+set -e
+cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+cd config/zot && $(KUSTOMIZE) edit set image zot-minimal=${ZOT_IMG}
+endef

config/default-zot-https/certificate.yaml

@@ -0,0 +1,44 @@
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: ocm-k8s-toolkit-bootstrap-issuer
+  namespace: ocm-k8s-toolkit-system
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: ocm-k8s-toolkit-bootstrap-certificate
+  namespace: ocm-k8s-toolkit-system
+spec:
+  # this is discouraged but required by ios
+  commonName: cert-manager-ocm-tls
+  isCA: true
+  secretName: ocm-k8s-toolkit-registry-tls-certs
+  subject:
+    organizations:
+      - ocm.software
+  dnsNames:
+    - ocm-k8s-toolkit-zot-registry.ocm-k8s-toolkit-system.svc.cluster.local
+    - localhost
+  ipAddresses:
+    - 127.0.0.1
+    - ::1
+  privateKey:
+    algorithm: RSA
+    encoding: PKCS8
+    size: 2048
+  issuerRef:
+    name: ocm-k8s-toolkit-bootstrap-issuer
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: ocm-k8s-toolkit-certificate-issuer
+  namespace: ocm-k8s-toolkit-system
+spec:
+  ca:
+    secretName: ocm-k8s-toolkit-registry-tls-certs

config/default-zot-https/configmap-patch.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: zot-config
+data:
+  config.json: |
+    {
+      "storage": {
+        "rootDirectory": "/tmp/zot",
+        "commit": true,
+        "dedupe": true,
+        "gc": true,
+        "gcDelay": "1h",
+        "gcInterval": "24h"
+      },
+      "http": {
+        "address":"0.0.0.0",
+        "port": "5000",
+        "tls": {
+          "cert":"/etc/zot/tls/tls.crt",
+          "key":"/etc/zot/tls/tls.key"
+        }
+      },
+      "log": {
+          "level": "debug"
+      }
+    }

config/default-zot-https/deployment-patch.yaml

@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: zot-registry
+spec:
+  template:
+    spec:
+      containers:
+      - name: zot-registry
+        volumeMounts:
+        - name: zot-tls
+          mountPath: /etc/zot/tls
+          readOnly: true
+      volumes:
+      - name: zot-tls
+        secret:
+          secretName: ocm-k8s-toolkit-registry-tls-certs
+# Registry is accessible in-cluster under:
+# https://ocm-k8s-toolkit-zot-registry.ocm-k8s-toolkit-system.svc.cluster.local:5000/v2/_catalog

config/default-zot-https/get_secret.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Hack: extract secret from cluster to local dir, e.g. for testing or debugging
+kubectl get secret ocm-k8s-toolkit-registry-tls-certs -n ocm-k8s-toolkit-system -o jsonpath="{.data['tls\.crt']}" | base64 -d > config/zot-https/rootCA.pem
+
+# Alternatively `curl` can be used (https://curl.se/docs/sslcerts.html):
+# curl -k -w %{certs} https://localhost:31000/v2/_catalog > config/zot-https/rootCA.pem
+# curl --cacert config/zot-https/rootCA.pem https://localhost:31000/v2/_catalog
+
+# The pem file can be added to the system trust store (https://github.com/FiloSottile/mkcert):
+# CAROOT=config/zot-https mkcert -install

config/default-zot-https/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- certificate.yaml
+- ../default
+patches:
+- path: configmap-patch.yaml
+- path: deployment-patch.yaml

config/default/kustomization.yaml

@@ -17,6 +17,7 @@ namePrefix: ocm-k8s-toolkit-
 resources:
 - ../crd
 - ../rbac
+- ../zot
 - ../manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml

config/zot/configmap.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: zot-config
+data:
+  # for docker compatibility, add '"http": { "compat": "docker2s2" }'
+  config.json: |
+    {
+      "storage": {
+        "rootDirectory": "/tmp/zot",
+        "commit": true,
+        "dedupe": true,
+        "gc": true,
+        "gcDelay": "1h",
+        "gcInterval": "24h"
+      },
+      "http": {
+        "address":"0.0.0.0",
+        "port": "5000"
+      },
+      "log": {
+          "level": "debug"
+      }
+    }

config/zot/deployment.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: zot-registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: zot
+  template:
+    metadata:
+      labels:
+        app: zot
+    spec:
+      containers:
+      - name: zot-registry
+        image: zot-minimal:latest
+        ports:
+        - containerPort: 5000
+        volumeMounts:
+        - name: zot-config-volume
+          mountPath: /etc/zot/config.json
+          subPath: config.json
+        - name: zot-storage
+          mountPath: /tmp/zot
+      volumes:
+      - name: zot-config-volume
+        configMap:
+          name: zot-config
+      - name: zot-storage
+        emptyDir: {}
+# Manual test if the registry is running:
+# % kubectl run -i --tty --rm debug --image=busybox --restart=Never -- /bin/sh
+# # wget -qO- http://ocm-k8s-toolkit-zot-registry.ocm-k8s-toolkit-system.svc.cluster.local:5000/v2/_catalog
+# {"repositories":[]}
+# # exit

config/zot/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- configmap.yaml
+- deployment.yaml
+- service.yaml
+images:
+- name: zot-minimal
+  newName: ghcr.io/project-zot/zot-minimal
+  newTag: latest

config/zot/service.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: zot-registry
+spec:
+  type: ClusterIP
+  ports:
+    - port: 5000
+      targetPort: 5000
+  selector:
+    app: zot
+# ---
+# Uncomment if you need an external port to the image registry
+# Can be reached e.g. from the host with 'localhost:31000'
+# apiVersion: v1
+# kind: Service
+# metadata:
+#   name: zot-registry-external
+# spec:
+#   type: NodePort
+#   ports:
+#     - port: 5000
+#       targetPort: 5000
+#       nodePort: 31000
+#   selector:
+#     app: zot