diff --git a/README.md b/README.md index b7c8e265..3b300bbd 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Note that persistence requires your cluster to have some PersistentVolumes. If y Sometimes, you want images to stay cached even when they are not used anymore (for instance when you run a workload for a fixed amount of time, stop it, and run it again later). You can choose to prevent `CachedImages` from expiring by manually setting the `spec.retain` flag to `true` like shown below: ```yaml -apiVersion: kuik.enix.io/v1alpha1 +apiVersion: kuik.enix.io/v1alpha1ext1 kind: CachedImage metadata: name: docker.io-library-nginx-1.25 diff --git a/api/kuik/v1alpha1/cachedimage_types.go b/api/kuik/v1alpha1ext1/cachedimage_types.go similarity index 93% rename from api/kuik/v1alpha1/cachedimage_types.go rename to api/kuik/v1alpha1ext1/cachedimage_types.go index 9ddf41ef..5ef16b46 100644 --- a/api/kuik/v1alpha1/cachedimage_types.go +++ b/api/kuik/v1alpha1ext1/cachedimage_types.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1alpha1ext1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,12 +26,19 @@ type UsedBy struct { Count int `json:"count,omitempty"` } +type Progress struct { + Total int64 `json:"total,omitempty"` + Available int64 `json:"available,omitempty"` +} + // CachedImageStatus defines the observed state of CachedImage type CachedImageStatus struct { IsCached bool `json:"isCached,omitempty"` Phase string `json:"phase,omitempty"` UsedBy UsedBy `json:"usedBy,omitempty"` + Progress Progress `json:"progress,omitempty"` + Digest string `json:"digest,omitempty"` UpstreamDigest string `json:"upstreamDigest,omitempty"` UpToDate bool `json:"upToDate,omitempty"` diff --git a/api/kuik/v1alpha1/cachedimage_utils.go b/api/kuik/v1alpha1ext1/cachedimage_utils.go similarity index 97% rename from api/kuik/v1alpha1/cachedimage_utils.go rename to api/kuik/v1alpha1ext1/cachedimage_utils.go index cd37905b..b27669ed 100644 --- a/api/kuik/v1alpha1/cachedimage_utils.go +++ b/api/kuik/v1alpha1ext1/cachedimage_utils.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1alpha1ext1 import ( "context" diff --git a/api/kuik/v1alpha1/cachedimage_webhook.go b/api/kuik/v1alpha1ext1/cachedimage_webhook.go similarity index 98% rename from api/kuik/v1alpha1/cachedimage_webhook.go rename to api/kuik/v1alpha1ext1/cachedimage_webhook.go index baa98b58..39012b91 100644 --- a/api/kuik/v1alpha1/cachedimage_webhook.go +++ b/api/kuik/v1alpha1ext1/cachedimage_webhook.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1alpha1ext1 import ( "context" diff --git a/api/kuik/v1alpha1/cachedimage_webhook_test.go b/api/kuik/v1alpha1ext1/cachedimage_webhook_test.go similarity index 98% rename from api/kuik/v1alpha1/cachedimage_webhook_test.go rename to api/kuik/v1alpha1ext1/cachedimage_webhook_test.go index dc061f72..7dc48f84 100644 --- a/api/kuik/v1alpha1/cachedimage_webhook_test.go +++ b/api/kuik/v1alpha1ext1/cachedimage_webhook_test.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1alpha1ext1 import ( "context" diff --git a/api/kuik/v1alpha1/groupversion_info.go b/api/kuik/v1alpha1ext1/groupversion_info.go similarity index 79% rename from api/kuik/v1alpha1/groupversion_info.go rename to api/kuik/v1alpha1ext1/groupversion_info.go index 634c4880..8ea706eb 100644 --- a/api/kuik/v1alpha1/groupversion_info.go +++ b/api/kuik/v1alpha1ext1/groupversion_info.go @@ -1,7 +1,7 @@ -// Package v1alpha1 contains API Schema definitions for the kuik.enix.io v1alpha1 API group +// Package v1alpha1ext1 contains API Schema definitions for the kuik.enix.io v1alpha1ext1 API group // +kubebuilder:object:generate=true // +groupName=kuik.enix.io -package v1alpha1 +package v1alpha1ext1 import ( "k8s.io/apimachinery/pkg/runtime/schema" @@ -10,7 +10,7 @@ import ( var ( // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "kuik.enix.io", Version: "v1alpha1"} + GroupVersion = schema.GroupVersion{Group: "kuik.enix.io", Version: "v1alpha1ext1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} diff --git a/api/kuik/v1alpha1/repository_types.go b/api/kuik/v1alpha1ext1/repository_types.go similarity index 98% rename from api/kuik/v1alpha1/repository_types.go rename to api/kuik/v1alpha1ext1/repository_types.go index 7b4528a1..8f0606db 100644 --- a/api/kuik/v1alpha1/repository_types.go +++ b/api/kuik/v1alpha1ext1/repository_types.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1alpha1ext1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/api/kuik/v1alpha1/repository_utils.go b/api/kuik/v1alpha1ext1/repository_utils.go similarity index 97% rename from api/kuik/v1alpha1/repository_utils.go rename to api/kuik/v1alpha1ext1/repository_utils.go index a057cc57..7eb05fea 100644 --- a/api/kuik/v1alpha1/repository_utils.go +++ b/api/kuik/v1alpha1ext1/repository_utils.go @@ -1,4 +1,4 @@ -package v1alpha1 +package v1alpha1ext1 import ( "regexp" diff --git a/cmd/cache/main.go b/cmd/cache/main.go index 2140bef6..8311b6cb 100644 --- a/cmd/cache/main.go +++ b/cmd/cache/main.go @@ -18,7 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" kuikenixiov1 "github.com/enix/kube-image-keeper/api/core/v1" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" "github.com/enix/kube-image-keeper/internal" kuikController "github.com/enix/kube-image-keeper/internal/controller" "github.com/enix/kube-image-keeper/internal/controller/core" @@ -116,7 +116,7 @@ func main() { Decoder: admission.NewDecoder(mgr.GetScheme()), } mgr.GetWebhookServer().Register("/mutate-core-v1-pod", &webhook.Admission{Handler: &imageRewriter}) - if err = (&kuikv1alpha1.CachedImage{}).SetupWebhookWithManager(mgr); err != nil { + if err = (&kuikv1alpha1ext1.CachedImage{}).SetupWebhookWithManager(mgr); err != nil { setupLog.Error(err, "unable to create webhook", "webhook", "CachedImage") os.Exit(1) } diff --git a/config/crd/bases/kuik.enix.io_cachedimages.yaml b/config/crd/bases/kuik.enix.io_cachedimages.yaml index efbe1f37..ed026612 100644 --- a/config/crd/bases/kuik.enix.io_cachedimages.yaml +++ b/config/crd/bases/kuik.enix.io_cachedimages.yaml @@ -35,7 +35,7 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - name: v1alpha1 + name: v1alpha1ext1 schema: openAPIV3Schema: description: CachedImage is the Schema for the cachedimages API @@ -90,6 +90,15 @@ spec: type: string phase: type: string + progress: + properties: + available: + format: int64 + type: integer + total: + format: int64 + type: integer + type: object upToDate: type: boolean upstreamDigest: diff --git a/config/crd/bases/kuik.enix.io_repositories.yaml b/config/crd/bases/kuik.enix.io_repositories.yaml index 0a72227c..9fb46748 100644 --- a/config/crd/bases/kuik.enix.io_repositories.yaml +++ b/config/crd/bases/kuik.enix.io_repositories.yaml @@ -26,7 +26,7 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - name: v1alpha1 + name: v1alpha1ext1 schema: openAPIV3Schema: description: Repository is the Schema for the repositories API diff --git a/config/samples/kuik_v1alpha1_cachedimage.yaml b/config/samples/kuik_v1alpha1_cachedimage.yaml index 4041c88e..15156eea 100644 --- a/config/samples/kuik_v1alpha1_cachedimage.yaml +++ b/config/samples/kuik_v1alpha1_cachedimage.yaml @@ -1,4 +1,4 @@ -apiVersion: kuik.enix.io/v1alpha1 +apiVersion: kuik.enix.io/v1alpha1ext1 kind: CachedImage metadata: labels: diff --git a/config/samples/kuik_v1alpha1_repository.yaml b/config/samples/kuik_v1alpha1_repository.yaml index 5da5193e..8056d0b1 100644 --- a/config/samples/kuik_v1alpha1_repository.yaml +++ b/config/samples/kuik_v1alpha1_repository.yaml @@ -1,4 +1,4 @@ -apiVersion: kuik.enix.io/v1alpha1 +apiVersion: kuik.enix.io/v1alpha1ext1 kind: Repository metadata: labels: diff --git a/helm/kube-image-keeper/crds/cachedimage-crd.yaml b/helm/kube-image-keeper/crds/cachedimage-crd.yaml index cb9a882e..e71415e8 100644 --- a/helm/kube-image-keeper/crds/cachedimage-crd.yaml +++ b/helm/kube-image-keeper/crds/cachedimage-crd.yaml @@ -34,7 +34,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - name: v1alpha1 + - jsonPath: .status.progress.available + name: Downloaded + type: integer + name: v1alpha1ext1 schema: openAPIV3Schema: description: CachedImage is the Schema for the cachedimages API @@ -93,6 +96,15 @@ spec: type: boolean upstreamDigest: type: string + progress: + type: object + properties: + total: + type: integer + description: Total size of the compressed blob in bytes, including all layers. + available: + type: integer + description: Total downloaded / available size of the compressed blob in bytes, including all layers. usedBy: properties: count: diff --git a/helm/kube-image-keeper/crds/repository-crd.yaml b/helm/kube-image-keeper/crds/repository-crd.yaml index 5bf51081..599b28d1 100644 --- a/helm/kube-image-keeper/crds/repository-crd.yaml +++ b/helm/kube-image-keeper/crds/repository-crd.yaml @@ -25,7 +25,7 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date - name: v1alpha1 + name: v1alpha1ext1 schema: openAPIV3Schema: description: Repository is the Schema for the repositories API diff --git a/internal/controller/collector.go b/internal/controller/collector.go index f3b971a8..50227ce9 100644 --- a/internal/controller/collector.go +++ b/internal/controller/collector.go @@ -4,7 +4,7 @@ import ( "context" "strconv" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" kuikMetrics "github.com/enix/kube-image-keeper/internal/metrics" "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/client" @@ -70,7 +70,7 @@ func RegisterMetrics(client client.Client) { ) } -func cachedImagesWithLabelValues(gaugeVec *prometheus.GaugeVec, cachedImage *kuikv1alpha1.CachedImage) prometheus.Gauge { +func cachedImagesWithLabelValues(gaugeVec *prometheus.GaugeVec, cachedImage *kuikv1alpha1ext1.CachedImage) prometheus.Gauge { return gaugeVec.WithLabelValues(strconv.FormatBool(cachedImage.Status.IsCached), strconv.FormatBool(cachedImage.Spec.ExpiresAt != nil)) } @@ -83,7 +83,7 @@ func (c *ControllerCollector) Describe(ch chan<- *prometheus.Desc) { } func (c *ControllerCollector) Collect(ch chan<- prometheus.Metric) { - cachedImageList := &kuikv1alpha1.CachedImageList{} + cachedImageList := &kuikv1alpha1ext1.CachedImageList{} if err := c.List(context.Background(), cachedImageList); err == nil { cachedImageGaugeVec := prometheus.NewGaugeVec( prometheus.GaugeOpts{ diff --git a/internal/controller/core/pod_controller.go b/internal/controller/core/pod_controller.go index 15839a6c..1b7820f8 100644 --- a/internal/controller/core/pod_controller.go +++ b/internal/controller/core/pod_controller.go @@ -12,7 +12,7 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/distribution/reference" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" "github.com/enix/kube-image-keeper/internal/registry" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -92,7 +92,7 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } for _, cachedImage := range cachedImages { - var ci kuikv1alpha1.CachedImage + var ci kuikv1alpha1ext1.CachedImage err := r.Get(ctx, client.ObjectKeyFromObject(&cachedImage), &ci) if err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -141,7 +141,7 @@ func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { return ok }))). Watches( - &kuikv1alpha1.CachedImage{}, + &kuikv1alpha1ext1.CachedImage{}, handler.EnqueueRequestsFromMapFunc(r.podsWithDeletingCachedImages), builder.WithPredicates(p), ). @@ -154,8 +154,8 @@ func (r *PodReconciler) podsWithDeletingCachedImages(ctx context.Context, obj cl WithName("controller-runtime.manager.controller.pod.deletingCachedImages"). WithValues("cachedImage", klog.KObj(obj)) - cachedImage := obj.(*kuikv1alpha1.CachedImage) - var currentCachedImage kuikv1alpha1.CachedImage + cachedImage := obj.(*kuikv1alpha1ext1.CachedImage) + var currentCachedImage kuikv1alpha1ext1.CachedImage // wait for the CachedImage to be really deleted if err := r.Get(ctx, client.ObjectKeyFromObject(cachedImage), ¤tCachedImage); err == nil || !apierrors.IsNotFound(err) { return make([]ctrl.Request, 0) @@ -187,8 +187,8 @@ func (r *PodReconciler) podsWithDeletingCachedImages(ctx context.Context, obj cl return make([]ctrl.Request, 0) } -func (r *PodReconciler) desiredRepositories(ctx context.Context, pod *corev1.Pod, cachedImages []kuikv1alpha1.CachedImage) ([]kuikv1alpha1.Repository, error) { - repositories := map[string]kuikv1alpha1.Repository{} +func (r *PodReconciler) desiredRepositories(ctx context.Context, pod *corev1.Pod, cachedImages []kuikv1alpha1ext1.CachedImage) ([]kuikv1alpha1ext1.Repository, error) { + repositories := map[string]kuikv1alpha1ext1.Repository{} pullSecretNames, err := r.imagePullSecretNamesFromPod(ctx, pod) if err != nil { @@ -201,11 +201,11 @@ func (r *PodReconciler) desiredRepositories(ctx context.Context, pod *corev1.Pod return nil, err } repositoryName := named.Name() - repositories[repositoryName] = kuikv1alpha1.Repository{ + repositories[repositoryName] = kuikv1alpha1ext1.Repository{ ObjectMeta: metav1.ObjectMeta{ Name: registry.SanitizeName(repositoryName), }, - Spec: kuikv1alpha1.RepositorySpec{ + Spec: kuikv1alpha1ext1.RepositorySpec{ Name: repositoryName, PullSecretNames: pullSecretNames, PullSecretsNamespace: pod.Namespace, @@ -216,15 +216,15 @@ func (r *PodReconciler) desiredRepositories(ctx context.Context, pod *corev1.Pod return maps.Values(repositories), nil } -func DesiredCachedImages(ctx context.Context, pod *corev1.Pod) []kuikv1alpha1.CachedImage { +func DesiredCachedImages(ctx context.Context, pod *corev1.Pod) []kuikv1alpha1ext1.CachedImage { cachedImages := desiredCachedImagesForContainers(ctx, pod.Spec.Containers, pod.Annotations, false) cachedImages = append(cachedImages, desiredCachedImagesForContainers(ctx, pod.Spec.InitContainers, pod.Annotations, true)...) return cachedImages } -func desiredCachedImagesForContainers(ctx context.Context, containers []corev1.Container, annotations map[string]string, initContainer bool) []kuikv1alpha1.CachedImage { +func desiredCachedImagesForContainers(ctx context.Context, containers []corev1.Container, annotations map[string]string, initContainer bool) []kuikv1alpha1ext1.CachedImage { log := log.FromContext(ctx) - cachedImages := []kuikv1alpha1.CachedImage{} + cachedImages := []kuikv1alpha1ext1.CachedImage{} for _, container := range containers { annotationKey := registry.ContainerAnnotationKey(container.Name, initContainer) @@ -249,7 +249,7 @@ func desiredCachedImagesForContainers(ctx context.Context, containers []corev1.C return cachedImages } -func cachedImageFromSourceImage(sourceImage string) (*kuikv1alpha1.CachedImage, error) { +func cachedImageFromSourceImage(sourceImage string) (*kuikv1alpha1ext1.CachedImage, error) { ref, err := reference.ParseAnyReference(sourceImage) if err != nil { return nil, err @@ -260,12 +260,12 @@ func cachedImageFromSourceImage(sourceImage string) (*kuikv1alpha1.CachedImage, sanitizedName += "-latest" } - cachedImage := kuikv1alpha1.CachedImage{ - TypeMeta: metav1.TypeMeta{APIVersion: kuikv1alpha1.GroupVersion.String(), Kind: "CachedImage"}, + cachedImage := kuikv1alpha1ext1.CachedImage{ + TypeMeta: metav1.TypeMeta{APIVersion: kuikv1alpha1ext1.GroupVersion.String(), Kind: "CachedImage"}, ObjectMeta: metav1.ObjectMeta{ Name: sanitizedName, }, - Spec: kuikv1alpha1.CachedImageSpec{ + Spec: kuikv1alpha1ext1.CachedImageSpec{ SourceImage: sourceImage, }, } diff --git a/internal/controller/core/pod_controller_test.go b/internal/controller/core/pod_controller_test.go index 54216cf1..2e640730 100644 --- a/internal/controller/core/pod_controller_test.go +++ b/internal/controller/core/pod_controller_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" "github.com/enix/kube-image-keeper/internal/registry" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -57,19 +57,19 @@ func TestDesiredCachedImages(t *testing.T) { tests := []struct { name string pod corev1.Pod - cachedImages []kuikv1alpha1.CachedImage + cachedImages []kuikv1alpha1ext1.CachedImage }{ { name: "basic", pod: podStub, - cachedImages: []kuikv1alpha1.CachedImage{ - {Spec: kuikv1alpha1.CachedImageSpec{ + cachedImages: []kuikv1alpha1ext1.CachedImage{ + {Spec: kuikv1alpha1ext1.CachedImageSpec{ SourceImage: "nginx", }}, - {Spec: kuikv1alpha1.CachedImageSpec{ + {Spec: kuikv1alpha1ext1.CachedImageSpec{ SourceImage: "busybox", }}, - {Spec: kuikv1alpha1.CachedImageSpec{ + {Spec: kuikv1alpha1ext1.CachedImageSpec{ SourceImage: "alpine", }}, }, @@ -146,7 +146,7 @@ var _ = Describe("Pod Controller", func() { podStubNotRewritten.ResourceVersion = "" By("Deleting all cached images") - Expect(k8sClient.DeleteAllOf(context.Background(), &kuikv1alpha1.CachedImage{})).Should(Succeed()) + Expect(k8sClient.DeleteAllOf(context.Background(), &kuikv1alpha1ext1.CachedImage{})).Should(Succeed()) }) Context("Pod with containers and init containers", func() { @@ -154,8 +154,8 @@ var _ = Describe("Pod Controller", func() { By("Creating a pod") Expect(k8sClient.Create(context.Background(), &podStub)).Should(Succeed()) - fetched := &kuikv1alpha1.CachedImageList{} - Eventually(func() []kuikv1alpha1.CachedImage { + fetched := &kuikv1alpha1ext1.CachedImageList{} + Eventually(func() []kuikv1alpha1ext1.CachedImage { _ = k8sClient.List(context.Background(), fetched) return fetched.Items }, timeout, interval).Should(HaveLen(len(podStub.Spec.Containers) + len(podStub.Spec.InitContainers))) @@ -177,8 +177,8 @@ var _ = Describe("Pod Controller", func() { By("Creating a pod without rewriting images") Expect(k8sClient.Create(context.Background(), &podStubNotRewritten)).Should(Succeed()) - fetched := &kuikv1alpha1.CachedImageList{} - Eventually(func() []kuikv1alpha1.CachedImage { + fetched := &kuikv1alpha1ext1.CachedImageList{} + Eventually(func() []kuikv1alpha1ext1.CachedImage { _ = k8sClient.List(context.Background(), fetched) return fetched.Items }, timeout, interval).Should(HaveLen(0)) diff --git a/internal/controller/core/suite_test.go b/internal/controller/core/suite_test.go index 21cedb86..73c11fac 100644 --- a/internal/controller/core/suite_test.go +++ b/internal/controller/core/suite_test.go @@ -19,7 +19,7 @@ import ( corev1 "k8s.io/api/core/v1" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" //+kubebuilder:scaffold:imports ) @@ -55,7 +55,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = kuikv1alpha1.AddToScheme(scheme.Scheme) + err = kuikv1alpha1ext1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) err = corev1.AddToScheme(scheme.Scheme) diff --git a/internal/controller/kuik/cachedimage_controller.go b/internal/controller/kuik/cachedimage_controller.go index 8aa44153..61c696ee 100644 --- a/internal/controller/kuik/cachedimage_controller.go +++ b/internal/controller/kuik/cachedimage_controller.go @@ -9,6 +9,7 @@ import ( "github.com/distribution/reference" "github.com/go-logr/logr" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -28,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" kuikController "github.com/enix/kube-image-keeper/internal/controller" "github.com/enix/kube-image-keeper/internal/controller/core" "github.com/enix/kube-image-keeper/internal/registry" @@ -78,7 +79,7 @@ type CachedImageReconciler struct { func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) - var cachedImage kuikv1alpha1.CachedImage + var cachedImage kuikv1alpha1ext1.CachedImage if err := r.Get(ctx, req.NamespacedName, &cachedImage); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -102,7 +103,7 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) } repositoryName := named.Name() - repository := kuikv1alpha1.Repository{ObjectMeta: metav1.ObjectMeta{Name: registry.SanitizeName(repositoryName)}} + repository := kuikv1alpha1ext1.Repository{ObjectMeta: metav1.ObjectMeta{Name: registry.SanitizeName(repositoryName)}} operation, err := controllerutil.CreateOrPatch(ctx, r.Client, &repository, func() error { repository.Spec.Name = repositoryName return nil @@ -114,7 +115,7 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) log.Info("repository updated", "repository", klog.KObj(&repository), "operation", operation) // Set owner reference - owner := &kuikv1alpha1.Repository{} + owner := &kuikv1alpha1ext1.Repository{} if err := r.Get(ctx, client.ObjectKeyFromObject(&repository), owner); err != nil { return ctrl.Result{}, err } @@ -211,7 +212,7 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - err = updateStatusRaw(r.Client, &cachedImage, func(status *kuikv1alpha1.CachedImageStatus) { + err = updateStatusRaw(r.Client, &cachedImage, func(status *kuikv1alpha1ext1.CachedImageStatus) { cachedImage.Status.IsCached = isCached }) if err != nil { @@ -270,8 +271,8 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } -func updateStatus(c client.Client, cachedImage *kuikv1alpha1.CachedImage, upstreamDescriptor *remote.Descriptor, update func(*kuikv1alpha1.CachedImageStatus)) error { - return updateStatusRaw(c, cachedImage, func(status *kuikv1alpha1.CachedImageStatus) { +func updateStatus(c client.Client, cachedImage *kuikv1alpha1ext1.CachedImage, upstreamDescriptor *remote.Descriptor, update func(*kuikv1alpha1ext1.CachedImageStatus)) error { + return updateStatusRaw(c, cachedImage, func(status *kuikv1alpha1ext1.CachedImageStatus) { cachedImage.Status.AvailableUpstream = upstreamDescriptor != nil cachedImage.Status.LastSync = metav1.NewTime(time.Now()) @@ -287,13 +288,13 @@ func updateStatus(c client.Client, cachedImage *kuikv1alpha1.CachedImage, upstre }) } -func updateStatusRaw(c client.Client, cachedImage *kuikv1alpha1.CachedImage, update func(*kuikv1alpha1.CachedImageStatus)) error { +func updateStatusRaw(c client.Client, cachedImage *kuikv1alpha1ext1.CachedImage, update func(*kuikv1alpha1ext1.CachedImageStatus)) error { patch := client.MergeFrom(cachedImage.DeepCopy()) update(&cachedImage.Status) return c.Status().Patch(context.Background(), cachedImage, patch) } -func getSanitizedName(cachedImage *kuikv1alpha1.CachedImage) (string, error) { +func getSanitizedName(cachedImage *kuikv1alpha1ext1.CachedImage) (string, error) { ref, err := reference.ParseAnyReference(cachedImage.Spec.SourceImage) if err != nil { return "", err @@ -307,7 +308,7 @@ func getSanitizedName(cachedImage *kuikv1alpha1.CachedImage) (string, error) { return sanitizedName, nil } -func (r *CachedImageReconciler) cacheImage(cachedImage *kuikv1alpha1.CachedImage) error { +func (r *CachedImageReconciler) cacheImage(cachedImage *kuikv1alpha1ext1.CachedImage) error { if err := r.patchPhase(cachedImage, cachedImagePhaseSynchronizing); err != nil { return err } @@ -319,7 +320,7 @@ func (r *CachedImageReconciler) cacheImage(cachedImage *kuikv1alpha1.CachedImage desc, err := registry.GetDescriptor(cachedImage.Spec.SourceImage, pullSecrets, r.InsecureRegistries, r.RootCAs) - statusErr := updateStatus(r.Client, cachedImage, desc, func(status *kuikv1alpha1.CachedImageStatus) { + statusErr := updateStatus(r.Client, cachedImage, desc, func(status *kuikv1alpha1ext1.CachedImageStatus) { _, err := registry.GetLocalDescriptor(cachedImage.Spec.SourceImage) cachedImage.Status.IsCached = err == nil @@ -343,9 +344,32 @@ func (r *CachedImageReconciler) cacheImage(cachedImage *kuikv1alpha1.CachedImage return err } - err = registry.CacheImage(cachedImage.Spec.SourceImage, desc, r.Architectures) + lastUpdateTime := time.Now() + lastWriteComplete := int64(0) + onUpdated := func(update v1.Update) { + needUpdate := false + if lastWriteComplete != update.Complete && update.Complete == update.Total { + // Update is needed whenever the writing complmetes. + needUpdate = true + } + + if time.Since(lastUpdateTime).Seconds() >= 5 { + // Update is needed if last update is more than 5 seconds ago + needUpdate = true + } + if needUpdate { + updateStatus(r.Client, cachedImage, desc, func(status *kuikv1alpha1ext1.CachedImageStatus) { + cachedImage.Status.Progress.Total = update.Total + cachedImage.Status.Progress.Available = update.Complete + }) + + lastUpdateTime = time.Now() + } + lastWriteComplete = update.Complete + } + err = registry.CacheImage(cachedImage.Spec.SourceImage, desc, r.Architectures, onUpdated) - statusErr = updateStatus(r.Client, cachedImage, desc, func(status *kuikv1alpha1.CachedImageStatus) { + statusErr = updateStatus(r.Client, cachedImage, desc, func(status *kuikv1alpha1ext1.CachedImageStatus) { if err == nil { cachedImage.Status.IsCached = true cachedImage.Status.Digest = desc.Digest.Hex @@ -363,7 +387,7 @@ func (r *CachedImageReconciler) cacheImage(cachedImage *kuikv1alpha1.CachedImage return nil } -func (r *CachedImageReconciler) patchPhase(cachedImage *kuikv1alpha1.CachedImage, phase string) error { +func (r *CachedImageReconciler) patchPhase(cachedImage *kuikv1alpha1ext1.CachedImage, phase string) error { patch := client.MergeFrom(cachedImage.DeepCopy()) cachedImage.Status.Phase = phase return r.Status().Patch(context.Background(), cachedImage, patch) @@ -396,7 +420,7 @@ func (r *CachedImageReconciler) SetupWithManager(mgr ctrl.Manager, maxConcurrent } return ctrl.NewControllerManagedBy(mgr). - For(&kuikv1alpha1.CachedImage{}). + For(&kuikv1alpha1ext1.CachedImage{}). Watches( &corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(r.cachedImagesRequestFromPod), @@ -435,21 +459,21 @@ func (r *CachedImageReconciler) SetupWithManager(mgr ctrl.Manager, maxConcurrent } // updatePodCount update CachedImage UsedBy status -func (r *CachedImageReconciler) updatePodCount(ctx context.Context, cachedImage *kuikv1alpha1.CachedImage) (requeue bool, err error) { +func (r *CachedImageReconciler) updatePodCount(ctx context.Context, cachedImage *kuikv1alpha1ext1.CachedImage) (requeue bool, err error) { var podsList corev1.PodList if err = r.List(ctx, &podsList, client.MatchingFields{core.CachedImageOwnerKey: cachedImage.Name}); err != nil && !apierrors.IsNotFound(err) { return } - pods := []kuikv1alpha1.PodReference{} + pods := []kuikv1alpha1ext1.PodReference{} for _, pod := range podsList.Items { if !pod.DeletionTimestamp.IsZero() { continue } - pods = append(pods, kuikv1alpha1.PodReference{NamespacedName: pod.Namespace + "/" + pod.Name}) + pods = append(pods, kuikv1alpha1ext1.PodReference{NamespacedName: pod.Namespace + "/" + pod.Name}) } - cachedImage.Status.UsedBy = kuikv1alpha1.UsedBy{ + cachedImage.Status.UsedBy = kuikv1alpha1ext1.UsedBy{ Pods: pods, Count: len(pods), } diff --git a/internal/controller/kuik/cachedimage_controller_test.go b/internal/controller/kuik/cachedimage_controller_test.go index ad847fba..3ca7d931 100644 --- a/internal/controller/kuik/cachedimage_controller_test.go +++ b/internal/controller/kuik/cachedimage_controller_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -18,20 +18,20 @@ var _ = Describe("CachedImage Controller", func() { Context("When creating CachedImages", func() { It("Should expire image that are not retained only", func() { - fetched := &kuikv1alpha1.CachedImageList{} + fetched := &kuikv1alpha1ext1.CachedImageList{} By("Creating an image without the retain flag", func() { - Expect(k8sClient.Create(context.Background(), &kuikv1alpha1.CachedImage{ + Expect(k8sClient.Create(context.Background(), &kuikv1alpha1ext1.CachedImage{ ObjectMeta: v1.ObjectMeta{ Name: "nginx", }, - Spec: kuikv1alpha1.CachedImageSpec{ + Spec: kuikv1alpha1ext1.CachedImageSpec{ SourceImage: "nginx", }, })).Should(Succeed()) - Eventually(func() []kuikv1alpha1.CachedImage { - expiringCachedImages := []kuikv1alpha1.CachedImage{} + Eventually(func() []kuikv1alpha1ext1.CachedImage { + expiringCachedImages := []kuikv1alpha1ext1.CachedImage{} _ = k8sClient.List(context.Background(), fetched) for _, cachedImage := range fetched.Items { if cachedImage.Spec.ExpiresAt != nil { @@ -43,19 +43,19 @@ var _ = Describe("CachedImage Controller", func() { }) By("Creating an expiring image with the retain flag", func() { - Expect(k8sClient.Create(context.Background(), &kuikv1alpha1.CachedImage{ + Expect(k8sClient.Create(context.Background(), &kuikv1alpha1ext1.CachedImage{ ObjectMeta: v1.ObjectMeta{ Name: "alpine", }, - Spec: kuikv1alpha1.CachedImageSpec{ + Spec: kuikv1alpha1ext1.CachedImageSpec{ SourceImage: "alpine", Retain: true, ExpiresAt: &v1.Time{Time: time.Now().Add(time.Hour)}, }, })).Should(Succeed()) - Eventually(func() []kuikv1alpha1.CachedImage { - expiringCachedImages := []kuikv1alpha1.CachedImage{} + Eventually(func() []kuikv1alpha1ext1.CachedImage { + expiringCachedImages := []kuikv1alpha1ext1.CachedImage{} _ = k8sClient.List(context.Background(), fetched) for _, cachedImage := range fetched.Items { if cachedImage.Spec.ExpiresAt != nil { diff --git a/internal/controller/kuik/repository_controller.go b/internal/controller/kuik/repository_controller.go index cd451ee6..315aefc8 100644 --- a/internal/controller/kuik/repository_controller.go +++ b/internal/controller/kuik/repository_controller.go @@ -20,7 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" kuikController "github.com/enix/kube-image-keeper/internal/controller" "github.com/enix/kube-image-keeper/internal/registry" ) @@ -53,7 +53,7 @@ type RepositoryReconciler struct { func (r *RepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) - var repository kuikv1alpha1.Repository + var repository kuikv1alpha1ext1.Repository if err := r.Get(ctx, req.NamespacedName, &repository); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -67,7 +67,7 @@ func (r *RepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - var cachedImageList kuikv1alpha1.CachedImageList + var cachedImageList kuikv1alpha1ext1.CachedImageList if err := r.List(ctx, &cachedImageList, client.MatchingFields{repositoryOwnerKey: repository.Name}); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } @@ -196,7 +196,7 @@ func (r *RepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } -func (r *RepositoryReconciler) UpdateStatus(ctx context.Context, repository *kuikv1alpha1.Repository, conditions []metav1.Condition) error { +func (r *RepositoryReconciler) UpdateStatus(ctx context.Context, repository *kuikv1alpha1ext1.Repository, conditions []metav1.Condition) error { log := log.FromContext(ctx) for _, condition := range conditions { @@ -223,12 +223,12 @@ func (r *RepositoryReconciler) UpdateStatus(ctx context.Context, repository *kui // SetupWithManager sets up the controller with the Manager. func (r *RepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { // Create an index to list CachedImage by Repository - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &kuikv1alpha1.CachedImage{}, repositoryOwnerKey, func(rawObj client.Object) []string { - cachedImage := rawObj.(*kuikv1alpha1.CachedImage) + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &kuikv1alpha1ext1.CachedImage{}, repositoryOwnerKey, func(rawObj client.Object) []string { + cachedImage := rawObj.(*kuikv1alpha1ext1.CachedImage) owners := cachedImage.GetOwnerReferences() for _, owner := range owners { - if owner.APIVersion != kuikv1alpha1.GroupVersion.String() || owner.Kind != "Repository" { + if owner.APIVersion != kuikv1alpha1ext1.GroupVersion.String() || owner.Kind != "Repository" { return nil } @@ -241,9 +241,9 @@ func (r *RepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { } return ctrl.NewControllerManagedBy(mgr). - For(&kuikv1alpha1.Repository{}). + For(&kuikv1alpha1ext1.Repository{}). Watches( - &kuikv1alpha1.CachedImage{}, + &kuikv1alpha1ext1.CachedImage{}, handler.EnqueueRequestsFromMapFunc(r.repositoryWithDeletingCachedImages), builder.WithPredicates(predicate.Funcs{ DeleteFunc: func(e event.DeleteEvent) bool { @@ -252,7 +252,7 @@ func (r *RepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { }), ). Watches( - &kuikv1alpha1.CachedImage{}, + &kuikv1alpha1ext1.CachedImage{}, handler.EnqueueRequestsFromMapFunc(requestRepositoryFromCachedImage), builder.WithPredicates(predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { @@ -267,8 +267,8 @@ func (r *RepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error { } func (r *RepositoryReconciler) repositoryWithDeletingCachedImages(ctx context.Context, obj client.Object) []ctrl.Request { - cachedImage := obj.(*kuikv1alpha1.CachedImage) - var currentCachedImage kuikv1alpha1.CachedImage + cachedImage := obj.(*kuikv1alpha1ext1.CachedImage) + var currentCachedImage kuikv1alpha1ext1.CachedImage // wait for the CachedImage to be really deleted if err := r.Get(ctx, client.ObjectKeyFromObject(cachedImage), ¤tCachedImage); err == nil || !apierrors.IsNotFound(err) { return nil @@ -278,8 +278,8 @@ func (r *RepositoryReconciler) repositoryWithDeletingCachedImages(ctx context.Co } func requestRepositoryFromCachedImage(ctx context.Context, obj client.Object) []ctrl.Request { - cachedImage := obj.(*kuikv1alpha1.CachedImage) - repositoryName, ok := cachedImage.Labels[kuikv1alpha1.RepositoryLabelName] + cachedImage := obj.(*kuikv1alpha1ext1.CachedImage) + repositoryName, ok := cachedImage.Labels[kuikv1alpha1ext1.RepositoryLabelName] if !ok { return nil } diff --git a/internal/controller/kuik/suite_test.go b/internal/controller/kuik/suite_test.go index 85e6ed2a..48e3b909 100644 --- a/internal/controller/kuik/suite_test.go +++ b/internal/controller/kuik/suite_test.go @@ -39,7 +39,7 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" "github.com/enix/kube-image-keeper/internal/registry" //+kubebuilder:scaffold:imports ) @@ -134,7 +134,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) - err = kuikv1alpha1.AddToScheme(scheme.Scheme) + err = kuikv1alpha1ext1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) //+kubebuilder:scaffold:scheme diff --git a/internal/proxy/server.go b/internal/proxy/server.go index d68d85a9..1e3d600c 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -13,7 +13,7 @@ import ( "strings" "github.com/distribution/reference" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" "github.com/enix/kube-image-keeper/internal/metrics" "github.com/enix/kube-image-keeper/internal/registry" "github.com/gin-gonic/gin" @@ -251,10 +251,10 @@ func (p *Proxy) proxyRegistry(c *gin.Context, endpoint string, endpointIsOrigin return proxyError } -func (p *Proxy) getRepository(registryDomain string, repositoryName string) (*kuikv1alpha1.Repository, error) { +func (p *Proxy) getRepository(registryDomain string, repositoryName string) (*kuikv1alpha1ext1.Repository, error) { sanitizedName := registry.SanitizeName(registryDomain + "/" + repositoryName) - repository := &kuikv1alpha1.Repository{} + repository := &kuikv1alpha1ext1.Repository{} if err := p.k8sClient.Get(context.Background(), types.NamespacedName{Name: sanitizedName}, repository); err != nil { return nil, err } @@ -262,7 +262,7 @@ func (p *Proxy) getRepository(registryDomain string, repositoryName string) (*ku return repository, nil } -func (p *Proxy) getKeychains(repository *kuikv1alpha1.Repository) ([]authn.Keychain, error) { +func (p *Proxy) getKeychains(repository *kuikv1alpha1ext1.Repository) ([]authn.Keychain, error) { pullSecrets, err := repository.GetPullSecrets(p.k8sClient) if err != nil { return nil, err @@ -271,7 +271,7 @@ func (p *Proxy) getKeychains(repository *kuikv1alpha1.Repository) ([]authn.Keych return registry.GetKeychains(repository.Spec.Name, pullSecrets) } -func (p *Proxy) getAuthentifiedTransport(repository *kuikv1alpha1.Repository, originRegistry string) (http.RoundTripper, error) { +func (p *Proxy) getAuthentifiedTransport(repository *kuikv1alpha1ext1.Repository, originRegistry string) (http.RoundTripper, error) { imageRef, err := name.ParseReference(repository.Spec.Name) if err != nil { return nil, err diff --git a/internal/registry/registry.go b/internal/registry/registry.go index 3c187601..48e68819 100644 --- a/internal/registry/registry.go +++ b/internal/registry/registry.go @@ -123,12 +123,21 @@ func DeleteImage(imageName string) error { return remote.Delete(digest) } -func CacheImage(imageName string, desc *remote.Descriptor, architectures []string) error { +func CacheImage(imageName string, desc *remote.Descriptor, architectures []string, callback func(v1.Update)) error { destRef, err := parseLocalReference(imageName) if err != nil { return err } + progressUpdate := make(chan v1.Update, 100) + go func() { + for update := range progressUpdate { + if callback != nil { + callback(update) + } + } + }() + switch desc.MediaType { case types.OCIImageIndex, types.DockerManifestList: index, err := desc.ImageIndex() @@ -145,7 +154,7 @@ func CacheImage(imageName string, desc *remote.Descriptor, architectures []strin return true }) - if err := remote.WriteIndex(destRef, filteredIndex); err != nil { + if err := remote.WriteIndex(destRef, filteredIndex, remote.WithProgress(progressUpdate)); err != nil { return err } default: @@ -153,7 +162,8 @@ func CacheImage(imageName string, desc *remote.Descriptor, architectures []strin if err != nil { return err } - if err := remote.Write(destRef, image); err != nil { + + if err := remote.Write(destRef, image, remote.WithProgress(progressUpdate)); err != nil { return err } } diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go index a7635388..fdce494d 100644 --- a/internal/registry/registry_test.go +++ b/internal/registry/registry_test.go @@ -320,7 +320,7 @@ func Test_CacheImage(t *testing.T) { desc, err := remote.Get(sourceRef) g.Expect(err).To(BeNil()) - err = CacheImage(imageName, desc, []string{"amd64"}) + err = CacheImage(imageName, desc, []string{"amd64"}, nil) if tt.wantErr != "" { g.Expect(err).To(BeAssignableToTypeOf(tt.errType)) g.Expect(err).To(MatchError(ContainSubstring(tt.wantErr))) diff --git a/internal/scheme/scheme.go b/internal/scheme/scheme.go index a685106d..6f31c795 100644 --- a/internal/scheme/scheme.go +++ b/internal/scheme/scheme.go @@ -10,7 +10,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + kuikv1alpha1ext1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1ext1" //+kubebuilder:scaffold:imports ) @@ -19,7 +19,7 @@ func NewScheme() *runtime.Scheme { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(kuikv1alpha1.AddToScheme(scheme)) + utilruntime.Must(kuikv1alpha1ext1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme return scheme