diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 7e67d0f8..417d5129 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -113,3 +113,16 @@ type ResourceInfo struct { // +required Digest string `json:"digest,omitempty"` } + +type BlobInfo struct { + // Digest is the digest of the blob in the form of ':'. + // +kubebuilder:validation:Pattern="^[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+$" + Digest string `json:"digest"` + + // Tag/Version of the blob + Tag string `json:"tag"` + + // Size is the number of bytes of the blob. + // Can be used to determine how to file should be handled when downloaded (memory/disk) + Size int64 `json:"size"` +} diff --git a/api/v1alpha1/component_types.go b/api/v1alpha1/component_types.go index c005248a..46b41b4c 100644 --- a/api/v1alpha1/component_types.go +++ b/api/v1alpha1/component_types.go @@ -100,10 +100,13 @@ type ComponentStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // ArtifactRef references the generated artifact containing a list of + // SnapshotRef references the generated snapshot containing a list of // component descriptors. This list can be used by other controllers to // avoid re-downloading (and potentially also re-verifying) the components. // +optional + SnapshotRef corev1.LocalObjectReference `json:"snapshotRef,omitempty"` + + // TODO: Remove ArtifactRef corev1.LocalObjectReference `json:"artifactRef,omitempty"` // Component specifies the concrete version of the component that was @@ -180,6 +183,10 @@ func (in *Component) GetVerifications() []Verification { return in.Spec.Verify } +func (in *Component) GetSnapshotName() string { + return in.Status.SnapshotRef.Name +} + // +kubebuilder:object:root=true // ComponentList contains a list of Component. diff --git a/api/v1alpha1/resource_types.go b/api/v1alpha1/resource_types.go index b8ec88a1..5e9824f4 100644 --- a/api/v1alpha1/resource_types.go +++ b/api/v1alpha1/resource_types.go @@ -62,9 +62,13 @@ type ResourceStatus struct { // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` - // ArtifactRef points to the Artifact which represents the output of the - // last successful Resource sync. + // SnapshotRef references the generated snapshot containing a list of + // component descriptors. This list can be used by other controllers to + // avoid re-downloading (and potentially also re-verifying) the components. // +optional + SnapshotRef corev1.LocalObjectReference `json:"snapshotRef,omitempty"` + + // TODO: Remove ArtifactRef corev1.LocalObjectReference `json:"artifactRef,omitempty"` // +optional @@ -131,6 +135,10 @@ func (in *Resource) GetEffectiveOCMConfig() []OCMConfiguration { return in.Status.EffectiveOCMConfig } +func (in *Resource) GetSnapshotName() string { + return in.Status.SnapshotRef.Name +} + // +kubebuilder:object:root=true // ResourceList contains a list of Resource. diff --git a/api/v1alpha1/snapshot_types.go b/api/v1alpha1/snapshot_types.go index 8643c44c..82bcf079 100644 --- a/api/v1alpha1/snapshot_types.go +++ b/api/v1alpha1/snapshot_types.go @@ -4,24 +4,31 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ocmmetav1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" ) // SnapshotWriter defines any object which produces a snapshot // +k8s:deepcopy-gen=false type SnapshotWriter interface { client.Object - GetSnapshotDigest() string GetSnapshotName() string + GetKind() string + GetNamespace() string + GetName() string } // SnapshotSpec defines the desired state of Snapshot. type SnapshotSpec struct { - Identity ocmmetav1.Identity `json:"identity"` + // OCI repository name + // +required + Repository string `json:"repository"` + // Manifest digest + // +required Digest string `json:"digest"` - Tag string `json:"tag"` + // Blob + // +required + Blob BlobInfo `json:"blob"` // Suspend stops all operations on this object. // +optional @@ -41,10 +48,6 @@ type SnapshotStatus struct { // +optional LastReconciledTag string `json:"tag,omitempty"` - // RepositoryURL has the concrete URL pointing to the local registry including the service name. - // +optional - RepositoryURL string `json:"repositoryURL,omitempty"` - // ObservedGeneration is the last reconciled generation. // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` @@ -61,17 +64,6 @@ func (in *Snapshot) SetObservedGeneration(v int64) { in.Status.ObservedGeneration = v } -// TODO: Check purpose -//// GetComponentVersion returns the component version for the snapshot. -// func (in Snapshot) GetComponentVersion() string { -// return in.Spec.Identity[ComponentVersionKey] -//} -// -//// GetComponentResourceVersion returns the resource version for the snapshot. -// func (in Snapshot) GetComponentResourceVersion() string { -// return in.Spec.Identity[ResourceVersionKey] -//} - // GetDigest returns the last reconciled digest for the snapshot. func (in Snapshot) GetDigest() string { return in.Status.LastReconciledDigest diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index da203423..541d3ca6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -11,6 +11,21 @@ import ( "ocm.software/ocm/api/ocm/compdesc/meta/v1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlobInfo) DeepCopyInto(out *BlobInfo) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlobInfo. +func (in *BlobInfo) DeepCopy() *BlobInfo { + if in == nil { + return nil + } + out := new(BlobInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Component) DeepCopyInto(out *Component) { *out = *in @@ -127,6 +142,7 @@ func (in *ComponentStatus) DeepCopyInto(out *ComponentStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.SnapshotRef = in.SnapshotRef out.ArtifactRef = in.ArtifactRef in.Component.DeepCopyInto(&out.Component) if in.EffectiveOCMConfig != nil { @@ -1247,6 +1263,7 @@ func (in *ResourceStatus) DeepCopyInto(out *ResourceStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.SnapshotRef = in.SnapshotRef out.ArtifactRef = in.ArtifactRef if in.Resource != nil { in, out := &in.Resource, &out.Resource @@ -1275,7 +1292,7 @@ func (in *Snapshot) DeepCopyInto(out *Snapshot) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) + out.Spec = in.Spec in.Status.DeepCopyInto(&out.Status) } @@ -1332,13 +1349,7 @@ func (in *SnapshotList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SnapshotSpec) DeepCopyInto(out *SnapshotSpec) { *out = *in - if in.Identity != nil { - in, out := &in.Identity, &out.Identity - *out = make(v1.Identity, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } + out.Blob = in.Blob } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SnapshotSpec. diff --git a/cmd/main.go b/cmd/main.go index 78599620..9ef96d92 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -52,6 +52,7 @@ import ( "github.com/open-component-model/ocm-k8s-toolkit/internal/controller/resource" "github.com/open-component-model/ocm-k8s-toolkit/internal/controller/snapshot" "github.com/open-component-model/ocm-k8s-toolkit/pkg/ocm" + snapshotRegistry "github.com/open-component-model/ocm-k8s-toolkit/pkg/snapshot" ) var ( @@ -67,7 +68,7 @@ func init() { // +kubebuilder:scaffold:scheme } -//nolint:funlen // this is the main function +//nolint:funlen,maintidx // this is the main function func main() { var ( metricsAddr string @@ -171,6 +172,7 @@ func main() { os.Exit(1) } + // TODO: Replace storage, artifactServer, err := server.NewArtifactStore(mgr.GetClient(), mgr.GetScheme(), storagePath, storageAddr, storageAdvAddr, artifactRetentionTTL, artifactRetentionRecords) if err != nil { @@ -178,13 +180,26 @@ func main() { os.Exit(1) } + // TODO: Adjust hardcode with CLI param + registry, err := snapshotRegistry.NewRegistry("ocm-k8s-toolkit-zot-registry.ocm-k8s-toolkit-system.svc.cluster.local:5000") + registry.PlainHTTP = true + if err != nil { + setupLog.Error(err, "unable to initialize registry object") + os.Exit(1) + } + + if err := registry.Ping(ctx); err != nil { + setupLog.Error(err, "unable to ping OCI registry") + os.Exit(1) + } + if err = (&component.Reconciler{ BaseReconciler: &ocm.BaseReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), EventRecorder: eventsRecorder, }, - Storage: storage, + Registry: registry, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Component") os.Exit(1) diff --git a/config/crd/bases/delivery.ocm.software_components.yaml b/config/crd/bases/delivery.ocm.software_components.yaml index 5325f24f..4603e20b 100644 --- a/config/crd/bases/delivery.ocm.software_components.yaml +++ b/config/crd/bases/delivery.ocm.software_components.yaml @@ -182,9 +182,8 @@ spec: properties: artifactRef: description: |- - ArtifactRef references the generated artifact containing a list of - component descriptors. This list can be used by other controllers to - avoid re-downloading (and potentially also re-verifying) the components. + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: name: default: "" @@ -327,6 +326,23 @@ spec: object. format: int64 type: integer + snapshotRef: + description: |- + SnapshotRef references the generated snapshot containing a list of + component descriptors. This list can be used by other controllers to + avoid re-downloading (and potentially also re-verifying) the components. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic type: object required: - spec diff --git a/config/crd/bases/delivery.ocm.software_resources.yaml b/config/crd/bases/delivery.ocm.software_resources.yaml index 9bb536e1..edfed58a 100644 --- a/config/crd/bases/delivery.ocm.software_resources.yaml +++ b/config/crd/bases/delivery.ocm.software_resources.yaml @@ -151,8 +151,8 @@ spec: properties: artifactRef: description: |- - ArtifactRef points to the Artifact which represents the output of the - last successful Resource sync. + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. properties: name: default: "" @@ -300,6 +300,23 @@ spec: - name - type type: object + snapshotRef: + description: |- + SnapshotRef references the generated snapshot containing a list of + component descriptors. This list can be used by other controllers to + avoid re-downloading (and potentially also re-verifying) the components. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic type: object required: - spec diff --git a/config/crd/bases/delivery.ocm.software_snapshots.yaml b/config/crd/bases/delivery.ocm.software_snapshots.yaml index b8c00bda..b9e1b23c 100644 --- a/config/crd/bases/delivery.ocm.software_snapshots.yaml +++ b/config/crd/bases/delivery.ocm.software_snapshots.yaml @@ -48,24 +48,40 @@ spec: spec: description: SnapshotSpec defines the desired state of Snapshot. properties: + blob: + description: Blob + properties: + digest: + description: Digest is the digest of the blob in the form of ':'. + pattern: ^[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+$ + type: string + size: + description: |- + Size is the number of bytes of the blob. + Can be used to determine how to file should be handled when downloaded (memory/disk) + format: int64 + type: integer + tag: + description: Tag/Version of the blob + type: string + required: + - digest + - size + - tag + type: object digest: + description: Manifest digest + type: string + repository: + description: OCI repository name type: string - identity: - additionalProperties: - type: string - description: |- - Identity describes the identity of an object. - Only ascii characters are allowed - type: object suspend: description: Suspend stops all operations on this object. type: boolean - tag: - type: string required: + - blob - digest - - identity - - tag + - repository type: object status: description: SnapshotStatus defines the observed state of Snapshot. @@ -133,10 +149,6 @@ spec: description: ObservedGeneration is the last reconciled generation. format: int64 type: integer - repositoryURL: - description: RepositoryURL has the concrete URL pointing to the local - registry including the service name. - type: string tag: description: Tag defines the explicit tag that was used to create the related snapshot and cache entry. diff --git a/go.mod b/go.mod index f067cfee..36cb1b01 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/fluxcd/pkg/apis/meta v1.10.0 github.com/fluxcd/pkg/runtime v0.53.0 github.com/fluxcd/pkg/tar v0.11.0 - github.com/distribution/distribution/v3 v3.0.0-beta.1 github.com/google/go-containerregistry v0.20.3 github.com/mandelsoft/filepath v0.0.0-20240223090642-3e2777258aa3 github.com/mandelsoft/goutils v0.0.0-20241005173814-114fa825bbdc @@ -26,15 +25,14 @@ require ( github.com/opencontainers/image-spec v1.1.0 github.com/openfluxcd/artifact v0.1.1 github.com/openfluxcd/controller-manager v0.1.2 - github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/stretchr/testify v1.10.0 github.com/ulikunitz/xz v0.5.12 - helm.sh/helm/v3 v3.16.3 k8s.io/api v0.32.1 k8s.io/apiextensions-apiserver v0.32.1 k8s.io/apimachinery v0.32.1 k8s.io/client-go v0.32.1 ocm.software/ocm v0.19.1 + oras.land/oras-go/v2 v2.5.0 sigs.k8s.io/controller-runtime v0.20.1 sigs.k8s.io/yaml v1.4.0 ) @@ -137,7 +135,6 @@ require ( github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-connections v0.5.0 // indirect - github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/drone/envsubst v1.0.3 // indirect @@ -198,8 +195,6 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.14.1 // indirect - github.com/googleapis/gax-go/v2 v2.14.0 // indirect - github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gowebpki/jcs v1.0.1 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect @@ -209,7 +204,6 @@ require ( github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect github.com/hashicorp/golang-lru/arc/v2 v2.0.7 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/hcl v1.0.1-vault-5 // indirect github.com/hashicorp/vault-client-go v0.4.3 // indirect github.com/huandu/xstrings v1.5.0 // indirect @@ -267,8 +261,6 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3 // indirect - github.com/redis/go-redis/extra/redisotel/v9 v9.5.3 // indirect github.com/redis/go-redis/v9 v9.7.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect @@ -318,25 +310,17 @@ require ( github.com/zeebo/errs v1.4.0 // indirect go.mongodb.org/mongo-driver v1.17.1 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/exporters/autoexport v0.46.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 // indirect - go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.33.0 // indirect go.opentelemetry.io/otel/sdk v1.33.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.31.0 // indirect go.opentelemetry.io/otel/trace v1.33.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect - go.step.sm/crypto v0.56.0 // indirect go.opentelemetry.io/proto/otlp v1.4.0 // indirect - go.step.sm/crypto v0.54.2 // indirect + go.step.sm/crypto v0.56.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.32.0 // indirect @@ -356,11 +340,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d // indirect google.golang.org/grpc v1.69.4 // indirect google.golang.org/protobuf v1.36.4 // indirect - google.golang.org/api v0.206.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.36.3 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect @@ -370,7 +349,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect helm.sh/helm/v3 v3.16.3 // indirect k8s.io/cli-runtime v0.32.1 // indirect - k8s.io/cli-runtime v0.32.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect diff --git a/go.sum b/go.sum index 2e705ce1..d5b6b48a 100644 --- a/go.sum +++ b/go.sum @@ -602,6 +602,7 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.5 h1:dvk7TIXCZpmfOlM+9mlcrWmWjw/wlKT+VDq2wMvfPJU= github.com/hashicorp/go-sockaddr v1.0.5/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3lyjsj6+CCykpaxI= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru/arc/v2 v2.0.7 h1:QxkVTxwColcduO+LP7eJO56r2hFiG8zEbfAAzRv52KQ= github.com/hashicorp/golang-lru/arc/v2 v2.0.7/go.mod h1:Pe7gBlGdc8clY5LJ0LpJXMt5AmgmWNH1g+oFFVUHOEc= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -1086,17 +1087,10 @@ go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4Jjx go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= -<<<<<<< HEAD -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= -go.step.sm/crypto v0.56.0 h1:KcFfV76cI9Xaw8bdSc9x55skyuSdcHcTdL37vvVZnvY= -go.step.sm/crypto v0.56.0/go.mod h1:snWNloxY9s1W+HsFqcviq55nvzbqqX6LxVt0Vktv5mw= -======= go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= -go.step.sm/crypto v0.54.2 h1:3LSA5nYDQvcd484OSx7xsS3XDqQ7/WZjVqvq0+a0fWc= -go.step.sm/crypto v0.54.2/go.mod h1:1+OjUozd5aA3TkBJfr5Aobd6vNt9F70n1DagcoBh3Pc= ->>>>>>> c926578 (copy and refactor oci-library from ocm-controller v1) +go.step.sm/crypto v0.56.0 h1:KcFfV76cI9Xaw8bdSc9x55skyuSdcHcTdL37vvVZnvY= +go.step.sm/crypto v0.56.0/go.mod h1:snWNloxY9s1W+HsFqcviq55nvzbqqX6LxVt0Vktv5mw= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -1257,35 +1251,19 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -<<<<<<< HEAD google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d h1:xJJRGY7TJcvIlpSrN3K6LAWgNFUILlO+OMAqtg9aqnw= google.golang.org/genproto/googleapis/rpc v0.0.0-20250102185135-69823020774d/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4= -======= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20241104194629-dd2ea8efbc28 h1:KJjNNclfpIkVqrZlTWcgOOaVQ00LdBnoEaRfkUx760s= -google.golang.org/genproto v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:mt9/MofW7AWQ+Gy179ChOnvmJatV8YHUmrcedo9CIFI= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= ->>>>>>> c926578 (copy and refactor oci-library from ocm-controller v1) google.golang.org/grpc v1.0.5/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -<<<<<<< HEAD google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -======= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= ->>>>>>> c926578 (copy and refactor oci-library from ocm-controller v1) google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1362,6 +1340,8 @@ ocm.software/ocm v0.19.1 h1:sWcQB+G9zcURqZfKvTnAfeA+rcDwlbI222o/fPkm6ls= ocm.software/ocm v0.19.1/go.mod h1:JCGMa/y8PPXvRhD+8SnnNHco4aAXMaXxiJKe8gyRHTQ= oras.land/oras-go v1.2.6 h1:z8cmxQXBU8yZ4mkytWqXfo6tZcamPwjsuxYU81xJ8Lk= oras.land/oras-go v1.2.6/go.mod h1:OVPc1PegSEe/K8YiLfosrlqlqTN9PUyFvOw5Y9gwrT8= +oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= +oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= sigs.k8s.io/controller-runtime v0.20.1 h1:JbGMAG/X94NeM3xvjenVUaBjy6Ui4Ogd/J5ZtjZnHaE= sigs.k8s.io/controller-runtime v0.20.1/go.mod h1:BrP3w158MwvB3ZbNpaAcIKkHQ7YGpYnzpoSTZ8E14WU= sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= diff --git a/internal/controller/component/component_controller.go b/internal/controller/component/component_controller.go index 70c179a4..03c8ec7b 100644 --- a/internal/controller/component/component_controller.go +++ b/internal/controller/component/component_controller.go @@ -20,39 +20,37 @@ import ( "context" "errors" "fmt" - "os" - "path/filepath" - "strings" "github.com/Masterminds/semver/v3" "github.com/fluxcd/pkg/runtime/conditions" "github.com/fluxcd/pkg/runtime/patch" "github.com/mandelsoft/goutils/sliceutils" - "github.com/openfluxcd/controller-manager/storage" + "github.com/opencontainers/go-digest" "k8s.io/apimachinery/pkg/types" "ocm.software/ocm/api/datacontext" "ocm.software/ocm/api/ocm/resolvers" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" - artifactv1 "github.com/openfluxcd/artifact/api/v1alpha1" corev1 "k8s.io/api/core/v1" ocmctx "ocm.software/ocm/api/ocm" ctrl "sigs.k8s.io/controller-runtime" "github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1" "github.com/open-component-model/ocm-k8s-toolkit/pkg/ocm" + "github.com/open-component-model/ocm-k8s-toolkit/pkg/snapshot" "github.com/open-component-model/ocm-k8s-toolkit/pkg/status" ) // Reconciler reconciles a Component object. type Reconciler struct { *ocm.BaseReconciler - Storage *storage.Storage + Registry snapshot.RegistryType } var _ ocm.Reconciler = (*Reconciler)(nil) @@ -167,6 +165,7 @@ func (r *Reconciler) reconcileOCM(ctx context.Context, component *v1alpha1.Compo return result, nil } +//nolint:funlen // we do not want to cut function at an arbitrary point func (r *Reconciler) reconcileComponent(ctx context.Context, octx ocmctx.Context, component *v1alpha1.Component, repository *v1alpha1.OCMRepository) (ctrl.Result, error) { logger := log.FromContext(ctx) @@ -238,26 +237,67 @@ func (r *Reconciler) reconcileComponent(ctx context.Context, octx ocmctx.Context return ctrl.Result{}, err } - err = r.Storage.ReconcileStorage(ctx, component) + // Store descriptors and create snapshot + // TODO: Can I check beforehand if the CD is already downloaded and in the OCI Registry (cached)? + // Compare digest/hash from manifest of the CD from the source storage + + ociRepositoryName, err := snapshot.CreateRepositoryName(component.Spec.RepositoryRef.Name, component.GetName()) + if err != nil { + status.MarkNotReady(r.EventRecorder, component, v1alpha1.ReconcileArtifactFailedReason, err.Error()) + + return ctrl.Result{}, err + } + + ociRepository, err := r.Registry.NewRepository(ctx, ociRepositoryName) if err != nil { - status.MarkNotReady(r.EventRecorder, component, v1alpha1.StorageReconcileFailedReason, err.Error()) + status.MarkNotReady(r.EventRecorder, component, v1alpha1.ReconcileArtifactFailedReason, err.Error()) - return ctrl.Result{}, fmt.Errorf("failed to reconcileComponent storage: %w", err) + return ctrl.Result{}, err + } + + descriptorBytes, err := yaml.Marshal(descriptors) + if err != nil { + status.MarkNotReady(r.EventRecorder, component, v1alpha1.ReconcileArtifactFailedReason, err.Error()) + + return ctrl.Result{}, err } - err = r.createArtifactForDescriptors(ctx, octx, component, cv, descriptors) + manifestDigest, err := ociRepository.PushSnapshot(ctx, version, descriptorBytes) if err != nil { status.MarkNotReady(r.EventRecorder, component, v1alpha1.ReconcileArtifactFailedReason, err.Error()) return ctrl.Result{}, err } - // Update status - r.setComponentStatus(component, configs, v1alpha1.ComponentInfo{ + // Create snapshot + snapshotCR := snapshot.Create(component, ociRepositoryName, manifestDigest.String(), version, digest.FromBytes(descriptorBytes).String(), int64(len(descriptorBytes))) + + if _, err = controllerutil.CreateOrUpdate(ctx, r.GetClient(), &snapshotCR, func() error { + if snapshotCR.ObjectMeta.CreationTimestamp.IsZero() { + if err := controllerutil.SetControllerReference(component, &snapshotCR, r.GetScheme()); err != nil { + return fmt.Errorf("failed to set controller reference: %w", err) + } + } + + return nil + }); err != nil { + status.MarkNotReady(r.EventRecorder, component, v1alpha1.ReconcileArtifactFailedReason, err.Error()) + + return ctrl.Result{}, err + } + + // Update component status + component.Status.Component = v1alpha1.ComponentInfo{ RepositorySpec: repository.Spec.RepositorySpec, Component: component.Spec.Component, Version: version, - }) + } + + component.Status.SnapshotRef = corev1.LocalObjectReference{ + Name: snapshotCR.GetName(), + } + + component.Status.EffectiveOCMConfig = configs status.MarkReady(r.EventRecorder, component, "Applied version %s", version) @@ -356,73 +396,3 @@ func (r *Reconciler) verifyComponentVersionAndListDescriptors(ctx context.Contex return descriptors, nil } - -func (r *Reconciler) createArtifactForDescriptors(ctx context.Context, octx ocmctx.Context, - component *v1alpha1.Component, cv ocmctx.ComponentVersionAccess, descriptors *ocm.Descriptors, -) error { - logger := log.FromContext(ctx) - - // Create temp working dir - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-%s-", component.Kind, component.Namespace, component.Name)) - if err != nil { - return reconcile.TerminalError(fmt.Errorf("failed to create temporary working directory: %w", err)) - } - octx.Finalizer().With(func() error { - if err = os.RemoveAll(tmpDir); err != nil { - ctrl.LoggerFrom(ctx).Error(err, "failed to remove temporary working directory") - } - - return nil - }) - - content, err := yaml.Marshal(descriptors) - if err != nil { - return reconcile.TerminalError(fmt.Errorf("failed to marshal content: %w", err)) - } - - const perm = 0o655 - if err := os.WriteFile(filepath.Join(tmpDir, v1alpha1.OCMComponentDescriptorList), content, perm); err != nil { - return reconcile.TerminalError(fmt.Errorf("failed to write file: %w", err)) - } - - revision := r.normalizeComponentVersionName(cv.GetName()) + "-" + cv.GetVersion() - if err := r.Storage.ReconcileArtifact( - ctx, - component, - revision, - tmpDir, - revision+".tar.gz", - func(art *artifactv1.Artifact, _ string) error { - // Archive directory to storage - if err := r.Storage.Archive(art, tmpDir, nil); err != nil { - return fmt.Errorf("unable to archive artifact to storage: %w", err) - } - - component.Status.ArtifactRef = corev1.LocalObjectReference{ - Name: art.Name, - } - - return nil - }, - ); err != nil { - return fmt.Errorf("failed to reconcileComponent artifact: %w", err) - } - - logger.Info("successfully reconciled component", "name", component.Name) - - return nil -} - -func (r *Reconciler) normalizeComponentVersionName(name string) string { - return strings.ReplaceAll(name, "/", "-") -} - -func (r *Reconciler) setComponentStatus( - component *v1alpha1.Component, - configs []v1alpha1.OCMConfiguration, - info v1alpha1.ComponentInfo, -) { - component.Status.Component = info - - component.Status.EffectiveOCMConfig = configs -} diff --git a/internal/controller/component/component_controller_test.go b/internal/controller/component/component_controller_test.go index 8ac65307..562de82f 100644 --- a/internal/controller/component/component_controller_test.go +++ b/internal/controller/component/component_controller_test.go @@ -19,7 +19,6 @@ package component import ( "context" "fmt" - "net/http" "os" "time" @@ -30,25 +29,17 @@ import ( "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/conditions" - "github.com/fluxcd/pkg/tar" - "github.com/mandelsoft/filepath/pkg/filepath" "github.com/mandelsoft/vfs/pkg/osfs" - "github.com/mandelsoft/vfs/pkg/vfs" - "k8s.io/apimachinery/pkg/types" - "ocm.software/ocm/api/ocm/extensions/repositories/ctf" - "ocm.software/ocm/api/utils/accessio" - "ocm.software/ocm/api/utils/accessobj" - "sigs.k8s.io/controller-runtime/pkg/envtest/komega" - "sigs.k8s.io/yaml" - - artifactv1 "github.com/openfluxcd/artifact/api/v1alpha1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" environment "ocm.software/ocm/api/helper/env" + "ocm.software/ocm/api/ocm/extensions/repositories/ctf" + "ocm.software/ocm/api/utils/accessio" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" "github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1" - "github.com/open-component-model/ocm-k8s-toolkit/pkg/ocm" ) const ( @@ -143,36 +134,19 @@ var _ = Describe("Component Controller", func() { } Expect(k8sClient.Create(ctx, component)).To(Succeed()) - By("check that artifact has been created successfully") + By("check that snapshot has been created successfully") Eventually(komega.Object(component), "15s").Should( - HaveField("Status.ArtifactRef.Name", Not(BeEmpty()))) + HaveField("Status.SnapshotRef.Name", Not(BeEmpty()))) - artifact := &artifactv1.Artifact{ + By("checking if the snapshot can be received") + snapshot := &v1alpha1.Snapshot{ ObjectMeta: metav1.ObjectMeta{ Namespace: component.Namespace, - Name: component.Status.ArtifactRef.Name, + Name: component.Status.SnapshotRef.Name, }, } - Eventually(komega.Get(artifact)).Should(Succeed()) - - By("check if the component descriptor list can be retrieved from the artifact server") - r := Must(http.Get(artifact.Spec.URL)) - - tmpdir := Must(os.MkdirTemp("/tmp", "descriptors-")) - DeferCleanup(func() error { - return os.RemoveAll(tmpdir) - }) - MustBeSuccessful(tar.Untar(r.Body, tmpdir)) - - repo := Must(ctf.Open(env, accessobj.ACC_WRITABLE, ctfpath, vfs.FileMode(vfs.O_RDWR), env)) - cv := Must(repo.LookupComponentVersion(Component, Version1)) - expecteddescs := Must(ocm.ListComponentDescriptors(ctx, cv, repo)) - - data := Must(os.ReadFile(filepath.Join(tmpdir, v1alpha1.OCMComponentDescriptorList))) - descs := &ocm.Descriptors{} - MustBeSuccessful(yaml.Unmarshal(data, descs)) - Expect(descs).To(YAMLEqual(expecteddescs)) + Eventually(komega.Get(snapshot)).Should(Succeed()) }) It("does not reconcile when the repository is not ready", func() { @@ -199,9 +173,9 @@ var _ = Describe("Component Controller", func() { } Expect(k8sClient.Create(ctx, component)).To(Succeed()) - By("check that no artifact has been created") + By("check that no snapshot has been created") Eventually(komega.Object(component), "15s").Should( - HaveField("Status.ArtifactRef.Name", BeEmpty())) + HaveField("Status.SnapshotRef.Name", BeEmpty())) }) It("grabs the new version when it becomes available", func() { @@ -224,10 +198,10 @@ var _ = Describe("Component Controller", func() { } Expect(k8sClient.Create(ctx, component)).To(Succeed()) - By("check that artifact has been created successfully") + By("check that snapshot has been created successfully") Eventually(komega.Object(component), "15s").Should( - HaveField("Status.ArtifactRef.Name", Not(BeEmpty()))) + HaveField("Status.SnapshotRef.Name", Not(BeEmpty()))) Expect(k8sClient.Get(ctx, types.NamespacedName{Name: component.Name, Namespace: component.Namespace}, component)).To(Succeed()) Expect(component.Status.Component.Version).To(Equal(Version1)) @@ -281,9 +255,9 @@ var _ = Describe("Component Controller", func() { } Expect(k8sClient.Create(ctx, component)).To(Succeed()) - By("check that artifact has been created successfully") + By("check that snapshot has been created successfully") - Eventually(komega.Object(component), "15s").Should(HaveField("Status.ArtifactRef.Name", Not(BeEmpty()))) + Eventually(komega.Object(component), "15s").Should(HaveField("Status.SnapshotRef.Name", Not(BeEmpty()))) Expect(k8sClient.Get(ctx, types.NamespacedName{Name: component.Name, Namespace: component.Namespace}, component)).To(Succeed()) Expect(component.Status.Component.Version).To(Equal("0.0.3")) @@ -330,8 +304,8 @@ var _ = Describe("Component Controller", func() { } Expect(k8sClient.Create(ctx, component)).To(Succeed()) - By("check that artifact has been created successfully") - Eventually(komega.Object(component), "15s").Should(HaveField("Status.ArtifactRef.Name", Not(BeEmpty()))) + By("check that snapshot has been created successfully") + Eventually(komega.Object(component), "15s").Should(HaveField("Status.SnapshotRef.Name", Not(BeEmpty()))) Expect(k8sClient.Get(ctx, types.NamespacedName{Name: component.Name, Namespace: component.Namespace}, component)).To(Succeed()) Expect(component.Status.Component.Version).To(Equal("0.0.3")) @@ -376,10 +350,10 @@ var _ = Describe("Component Controller", func() { } Expect(k8sClient.Create(ctx, component)).To(Succeed()) - By("check that artifact has been created successfully") + By("check that snapshot has been created successfully") Eventually(komega.Object(component), "15s").Should( - HaveField("Status.ArtifactRef.Name", Not(BeEmpty()))) + HaveField("Status.SnapshotRef.Name", Not(BeEmpty()))) Expect(k8sClient.Get(ctx, types.NamespacedName{Name: component.Name, Namespace: component.Namespace}, component)).To(Succeed()) Expect(component.Status.Component.Version).To(Equal("0.0.3")) diff --git a/internal/controller/component/suite_test.go b/internal/controller/component/suite_test.go index c7c66166..bd8f01d1 100644 --- a/internal/controller/component/suite_test.go +++ b/internal/controller/component/suite_test.go @@ -16,37 +16,28 @@ package component import ( "context" "fmt" - "io" - "net/http" - "os" "path/filepath" "runtime" "testing" - "time" - . "github.com/mandelsoft/goutils/testutils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "github.com/openfluxcd/controller-manager/server" + artifactv1 "github.com/openfluxcd/artifact/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/komega" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/yaml" - - artifactv1 "github.com/openfluxcd/artifact/api/v1alpha1" - corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" metricserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1" + "github.com/open-component-model/ocm-k8s-toolkit/pkg/mocks" "github.com/open-component-model/ocm-k8s-toolkit/pkg/ocm" ) @@ -55,11 +46,6 @@ import ( // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. -const ( - ARTIFACT_PATH = "ocm-k8s-artifactstore--*" - ARTIFACT_SERVER = "localhost:8080" -) - var cfg *rest.Config var k8sClient client.Client var k8sManager ctrl.Manager @@ -76,26 +62,10 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") - // Get external artifact CRD - resp, err := http.Get(v1alpha1.ArtifactCrd) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func() error { - return resp.Body.Close() - }) - - crdByte, err := io.ReadAll(resp.Body) - Expect(err).NotTo(HaveOccurred()) - - artifactCRD := &apiextensionsv1.CustomResourceDefinition{} - err = yaml.Unmarshal(crdByte, artifactCRD) - Expect(err).NotTo(HaveOccurred()) - testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true, - CRDs: []*apiextensionsv1.CustomResourceDefinition{artifactCRD}, - // The BinaryAssetsDirectory is only required if you want to run the tests directly // without call the makefile target test. If not informed it will look for the // default path defined in controller-runtime which is /usr/local/kubebuilder/. @@ -106,7 +76,7 @@ var _ = BeforeSuite(func() { } // cfg is defined in this file globally. - cfg, err = testEnv.Start() + cfg, err := testEnv.Start() Expect(err).NotTo(HaveOccurred()) Expect(cfg).NotTo(BeNil()) DeferCleanup(testEnv.Stop) @@ -130,10 +100,8 @@ var _ = BeforeSuite(func() { }) Expect(err).ToNot(HaveOccurred()) - tmpdir := Must(os.MkdirTemp("", ARTIFACT_PATH)) - address := ARTIFACT_SERVER - storage := Must(server.NewStorage(k8sClient, testEnv.Scheme, tmpdir, address, 0, 0)) - artifactServer := Must(server.NewArtifactServer(tmpdir, address, time.Millisecond)) + mockRegistry, err := mocks.NewRegistry("") + Expect(err).NotTo(HaveOccurred()) Expect((&Reconciler{ BaseReconciler: &ocm.BaseReconciler{ @@ -144,7 +112,7 @@ var _ = BeforeSuite(func() { IncludeObject: true, }, }, - Storage: storage, + Registry: mockRegistry, }).SetupWithManager(k8sManager)).To(Succeed()) ctx, cancel := context.WithCancel(context.Background()) @@ -157,10 +125,6 @@ var _ = BeforeSuite(func() { } Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) - go func() { - defer GinkgoRecover() - Expect(artifactServer.Start(ctx)).To(Succeed()) - }() go func() { defer GinkgoRecover() Expect(k8sManager.Start(ctx)).To(Succeed()) diff --git a/internal/controller/ocmrepository/controller.go b/internal/controller/ocmrepository/controller.go index 68aca8d3..85dba704 100644 --- a/internal/controller/ocmrepository/controller.go +++ b/internal/controller/ocmrepository/controller.go @@ -96,7 +96,7 @@ func (r *Reconciler) reconcileExists(ctx context.Context, ocmRepo *v1alpha1.OCMR } if ocmRepo.Spec.Suspend { - logger.Info("component is suspended, skipping reconciliation") + logger.Info("OCMRepository is suspended, skipping reconciliation") return ctrl.Result{}, nil } diff --git a/pkg/mocks/snapshot.go b/pkg/mocks/snapshot.go new file mode 100644 index 00000000..7bfcfb9a --- /dev/null +++ b/pkg/mocks/snapshot.go @@ -0,0 +1,40 @@ +package mocks + +import ( + "context" + + "github.com/opencontainers/go-digest" + "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/open-component-model/ocm-k8s-toolkit/pkg/snapshot" +) + +type Registry struct{} + +var _ snapshot.RegistryType = (*Registry)(nil) + +func NewRegistry(_ string) (snapshot.RegistryType, error) { + return &Registry{}, nil +} + +func (r *Registry) NewRepository(ctx context.Context, name string) (snapshot.RepositoryType, error) { + log.FromContext(ctx).Info("mocking repository creation", "name", name) + + return &Repository{}, nil +} + +type Repository struct{} + +var _ snapshot.RepositoryType = (*Repository)(nil) + +func (r *Repository) PushSnapshot(ctx context.Context, _ string, _ []byte) (digest.Digest, error) { + log.FromContext(ctx).Info("mocking snapshot push") + + return digest.FromString("mock"), nil +} + +func (r *Repository) FetchSnapshot(ctx context.Context, _ string) ([]byte, error) { + log.FromContext(ctx).Info("mocking snapshot fetch") + + return []byte{}, nil +} diff --git a/pkg/oci/client.go b/pkg/oci/client.go deleted file mode 100644 index ad44416a..00000000 --- a/pkg/oci/client.go +++ /dev/null @@ -1,244 +0,0 @@ -package oci - -import ( - "context" - "crypto/tls" - "crypto/x509" - "fmt" - "io" - "net/http" - - "github.com/google/go-containerregistry/pkg/v1/remote" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - - corev1 "k8s.io/api/core/v1" - apitypes "k8s.io/apimachinery/pkg/types" - - "github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1" -) - -// ClientOptsFunc options are used to leave the cache backwards compatible. -// If the certificate isn't defined, we will use `WithInsecure`. -type ClientOptsFunc func(opts *Client) - -// WithCertificateSecret defines the name of the secret holding the certificates. -func WithCertificateSecret(name string) ClientOptsFunc { - return func(opts *Client) { - opts.CertSecretName = name - } -} - -// WithNamespace sets up certificates for the client. -func WithNamespace(namespace string) ClientOptsFunc { - return func(opts *Client) { - opts.Namespace = namespace - } -} - -// WithInsecureSkipVerify sets up certificates for the client. -func WithInsecureSkipVerify(value bool) ClientOptsFunc { - return func(opts *Client) { - opts.InsecureSkipVerify = value - } -} - -// WithClient sets up certificates for the client. -func WithClient(client client.Client) ClientOptsFunc { - return func(opts *Client) { - opts.Client = client - } -} - -// Client implements the caching layer and the OCI layer. -type Client struct { - Client client.Client - OCIRepositoryAddr string - InsecureSkipVerify bool - Namespace string - CertSecretName string - - certPem []byte - keyPem []byte - ca []byte -} - -// WithTransport sets up insecure TLS so the library is forced to use HTTPS. -func (c *Client) WithTransport(ctx context.Context) Option { - return func(o *options) error { - if c.InsecureSkipVerify { - return nil - } - - if c.certPem == nil && c.keyPem == nil { - if err := c.setupCertificates(ctx); err != nil { - return fmt.Errorf("failed to set up certificates for transport: %w", err) - } - } - - o.remoteOpts = append(o.remoteOpts, remote.WithTransport(c.constructTLSRoundTripper())) - - return nil - } -} - -func (c *Client) setupCertificates(ctx context.Context) error { - if c.Client == nil { - return fmt.Errorf("client must not be nil if certificate is requested, please set WithClient when creating the oci cache") - } - registryCerts := &corev1.Secret{} - if err := c.Client.Get(ctx, apitypes.NamespacedName{Name: c.CertSecretName, Namespace: c.Namespace}, registryCerts); err != nil { - return fmt.Errorf("unable to find the secret containing the registry certificates: %w", err) - } - - certFile, ok := registryCerts.Data["tls.crt"] - if !ok { - return fmt.Errorf("tls.crt data not found in registry certificate secret") - } - - keyFile, ok := registryCerts.Data["tls.key"] - if !ok { - return fmt.Errorf("tls.key data not found in registry certificate secret") - } - - caFile, ok := registryCerts.Data["ca.crt"] - if !ok { - return fmt.Errorf("ca.crt data not found in registry certificate secret") - } - - c.certPem = certFile - c.keyPem = keyFile - c.ca = caFile - - return nil -} - -func (c *Client) constructTLSRoundTripper() http.RoundTripper { - tlsConfig := &tls.Config{} //nolint:gosec // must provide lower version for quay.io - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(c.ca) - - tlsConfig.Certificates = []tls.Certificate{ - { - Certificate: [][]byte{c.certPem}, - PrivateKey: c.keyPem, - }, - } - - tlsConfig.RootCAs = caCertPool - tlsConfig.InsecureSkipVerify = c.InsecureSkipVerify - - // Create a new HTTP transport with the TLS configuration - return &http.Transport{ - TLSClientConfig: tlsConfig, - } -} - -// NewClient creates a new OCI Client. -func NewClient(ociAddress string, opts ...ClientOptsFunc) *Client { - c := &Client{ - OCIRepositoryAddr: ociAddress, - } - - for _, opt := range opts { - opt(c) - } - - return c -} - -// PushData takes a blob of data and caches it using OCI as a background. -func (c *Client) PushData(ctx context.Context, data io.ReadCloser, mediaType, name, tag string) (string, int64, error) { - repositoryName := fmt.Sprintf("%s/%s", c.OCIRepositoryAddr, name) - repo, err := NewRepository(repositoryName, c.WithTransport(ctx)) - if err != nil { - return "", -1, fmt.Errorf("failed create new repository: %w", err) - } - - manifest, err := repo.PushStreamingImage(tag, data, mediaType, nil) - if err != nil { - return "", -1, fmt.Errorf("failed to push image: %w", err) - } - - layers := manifest.Layers - if len(layers) == 0 { - return "", -1, fmt.Errorf("no layers returned by manifest: %w", err) - } - - return layers[0].Digest.String(), layers[0].Size, nil -} - -// FetchDataByIdentity fetches an existing resource. Errors if there is no resource available. It's advised to call IsCached -// before fetching. Returns the digest of the resource alongside the data for further processing. -func (c *Client) FetchDataByIdentity(ctx context.Context, name, tag string) (io.ReadCloser, string, int64, error) { - logger := log.FromContext(ctx).WithName("cache") - repositoryName := fmt.Sprintf("%s/%s", c.OCIRepositoryAddr, name) - logger.V(v1alpha1.LevelDebug).Info("cache hit for data", "name", name, "tag", tag, "repository", repositoryName) - repo, err := NewRepository(repositoryName, c.WithTransport(ctx)) - if err != nil { - return nil, "", -1, fmt.Errorf("failed to get repository: %w", err) - } - - manifest, _, err := repo.FetchManifest(tag, nil) - if err != nil { - return nil, "", -1, fmt.Errorf("failed to fetch manifest to obtain layers: %w", err) - } - logger.V(v1alpha1.LevelDebug).Info("got the manifest", "manifest", manifest) - layers := manifest.Layers - if len(layers) == 0 { - return nil, "", -1, fmt.Errorf("layers for repository is empty") - } - - digest := layers[0].Digest - - reader, err := repo.FetchBlob(digest.String()) - if err != nil { - return nil, "", -1, fmt.Errorf("failed to fetch reader for digest of the 0th layer: %w", err) - } - - // decompresses the data coming from the cache. Because a streaming layer doesn't support decompression - // and a static layer returns the data AS IS, we have to decompress it ourselves. - return reader, digest.String(), layers[0].Size, nil -} - -// FetchDataByDigest returns a reader for a given digest. -func (c *Client) FetchDataByDigest(ctx context.Context, name, digest string) (io.ReadCloser, error) { - repositoryName := fmt.Sprintf("%s/%s", c.OCIRepositoryAddr, name) - - repo, err := NewRepository(repositoryName, c.WithTransport(ctx)) - if err != nil { - return nil, fmt.Errorf("failed to get repository: %w", err) - } - - reader, err := repo.FetchBlob(digest) - if err != nil { - return nil, fmt.Errorf("failed to fetch blob: %w", err) - } - - // decompresses the data coming from the cache. Because a streaming layer doesn't support decompression - // and a static layer returns the data AS IS, we have to decompress it ourselves. - return reader, nil -} - -// IsCached returns whether a certain tag with a given name exists in cache. -func (c *Client) IsCached(ctx context.Context, name, tag string) (bool, error) { - repositoryName := fmt.Sprintf("%s/%s", c.OCIRepositoryAddr, name) - - repo, err := NewRepository(repositoryName, c.WithTransport(ctx)) - if err != nil { - return false, fmt.Errorf("failed to get repository: %w", err) - } - - return repo.head(tag) -} - -// DeleteData removes a specific tag from the cache. -func (c *Client) DeleteData(ctx context.Context, name, tag string) error { - repositoryName := fmt.Sprintf("%s/%s", c.OCIRepositoryAddr, name) - repo, err := NewRepository(repositoryName, c.WithTransport(ctx)) - if err != nil { - return fmt.Errorf("failed create new repository: %w", err) - } - - return repo.deleteTag(tag) -} diff --git a/pkg/oci/client_test.go b/pkg/oci/client_test.go deleted file mode 100644 index 4a53a1a9..00000000 --- a/pkg/oci/client_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package oci - -import ( - "bytes" - "context" - "fmt" - "io" - "strings" - "testing" - - _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" - "github.com/mitchellh/hashstructure/v2" - . "github.com/onsi/gomega" - "github.com/stretchr/testify/assert" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - ocmmetav1 "ocm.software/ocm/api/ocm/compdesc/meta/v1" - - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1" -) - -func TestClient_FetchPush(t *testing.T) { - scheme := runtime.NewScheme() - assert.NoError(t, v1alpha1.AddToScheme(scheme)) - assert.NoError(t, v1.AddToScheme(scheme)) - - addr := strings.TrimPrefix(testServer.URL, "http://") - testCases := []struct { - name string - blob []byte - expected []byte - resource v1alpha1.ResourceReference - objects []client.Object - push bool - }{ - { - name: "image", - blob: []byte("image"), - expected: []byte("image"), - resource: v1alpha1.ResourceReference{ - Resource: ocmmetav1.Identity{ - "name": "test-resource-1", - "version": "v0.0.1", - }, - }, - push: true, - }, - { - name: "empty image", - blob: []byte(""), - expected: []byte(""), - resource: v1alpha1.ResourceReference{ - Resource: ocmmetav1.Identity{ - "name": "test-resource-2", - "version": "v0.0.2", - }, - }, - push: true, - }, - { - name: "data doesn't exist", - blob: []byte(""), - expected: []byte(""), - resource: v1alpha1.ResourceReference{ - Resource: ocmmetav1.Identity{ - "name": "test-resource-3", - "version": "v0.0.3", - }, - }, - }, - } - - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ocm-registry-tls-certs", - Namespace: "default", - }, - Data: map[string][]byte{ - "ca.crt": []byte("file"), - "tls.crt": []byte("file"), - "tls.key": []byte("file"), - }, - Type: "Opaque", - } - fakeClient := fake.NewClientBuilder().WithObjects(secret).WithScheme(scheme).Build() - c := NewClient(addr, WithClient(fakeClient), WithCertificateSecret("ocm-registry-tls-certs"), WithNamespace("default")) - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Helper() - - g := NewWithT(t) - obj := &v1alpha1.Component{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-name", - Namespace: "default", - }, - Spec: v1alpha1.ComponentSpec{ - Component: "github.com/open-component-model/root", - Semver: "v0.0.1", - }, - Status: v1alpha1.ComponentStatus{ - Component: v1alpha1.ComponentInfo{ - Version: "v0.0.1", - }, - }, - } - identity := ocmmetav1.Identity{ - "component-version": obj.Status.Component.Version, - "component-name": obj.Spec.Component, - "resource-name": tc.resource.Resource["name"], - "resource-version": tc.resource.Resource["version"], - } - name, err := hashIdentity(identity) - g.Expect(err).NotTo(HaveOccurred()) - if tc.push { - _, _, err := c.PushData(context.Background(), io.NopCloser(bytes.NewBuffer(tc.blob)), "", name, tc.resource.Resource["version"]) - g.Expect(err).NotTo(HaveOccurred()) - blob, _, _, err := c.FetchDataByIdentity(context.Background(), name, tc.resource.Resource["version"]) - g.Expect(err).NotTo(HaveOccurred()) - content, err := io.ReadAll(blob) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(content).To(Equal(tc.expected)) - } else { - exists, err := c.IsCached(context.Background(), name, tc.resource.Resource["version"]) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(exists).To(BeFalse()) - } - }) - } -} - -func TestClient_DeleteData(t *testing.T) { - scheme := runtime.NewScheme() - assert.NoError(t, v1.AddToScheme(scheme)) - assert.NoError(t, v1alpha1.AddToScheme(scheme)) - - addr := strings.TrimPrefix(testServer.URL, "http://") - testCases := []struct { - name string - blob []byte - expected []byte - resource v1alpha1.ResourceReference - objects []client.Object - push bool - }{ - { - name: "image", - blob: []byte("image"), - expected: []byte("image"), - resource: v1alpha1.ResourceReference{ - Resource: ocmmetav1.Identity{ - "name": "test-resource-1", - "version": "v0.0.1", - }, - }, - push: true, - }, - } - - secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ocm-registry-tls-certs", - Namespace: "default", - }, - Data: map[string][]byte{ - "ca.crt": []byte("file"), - "tls.crt": []byte("file"), - "tls.key": []byte("file"), - }, - Type: "Opaque", - } - fakeClient := fake.NewClientBuilder().WithObjects(secret).WithScheme(scheme).Build() - c := NewClient(addr, WithClient(fakeClient), WithCertificateSecret("ocm-registry-tls-certs"), WithNamespace("default")) - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Helper() - - g := NewWithT(t) - - obj := &v1alpha1.Component{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-name", - Namespace: "default", - }, - Spec: v1alpha1.ComponentSpec{ - Component: "github.com/open-component-model/root", - Semver: "v0.0.1", - }, - Status: v1alpha1.ComponentStatus{ - Component: v1alpha1.ComponentInfo{ - Version: "v0.0.1", - }, - }, - } - identity := ocmmetav1.Identity{ - "component-version": obj.Status.Component.Version, - "component-name": obj.Spec.Component, - "resource-name": tc.resource.Resource["name"], - "resource-version": tc.resource.Resource["version"], - } - name, err := hashIdentity(identity) - g.Expect(err).NotTo(HaveOccurred()) - _, _, err = c.PushData(context.Background(), io.NopCloser(bytes.NewBuffer(tc.blob)), "", name, tc.resource.Resource["version"]) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(c.DeleteData(context.Background(), name, tc.resource.Resource["version"])).To(Succeed()) - exists, err := c.IsCached(context.Background(), name, tc.resource.Resource["version"]) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(exists).To(BeFalse()) - }) - } -} - -// TODO: Check purpose -// hashIdentity returns the string hash of an ocm identity. -func hashIdentity(id ocmmetav1.Identity) (string, error) { - hash, err := hashstructure.Hash(id, hashstructure.FormatV2, nil) - if err != nil { - return "", fmt.Errorf("failed to hash identity: %w", err) - } - - return fmt.Sprintf("sha-%d", hash), nil -} diff --git a/pkg/oci/repository.go b/pkg/oci/repository.go deleted file mode 100644 index 14fb9285..00000000 --- a/pkg/oci/repository.go +++ /dev/null @@ -1,371 +0,0 @@ -package oci - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" - "github.com/google/go-containerregistry/pkg/v1/stream" - "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/opencontainers/go-digest" - "helm.sh/helm/v3/pkg/registry" - - ociname "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1" -) - -// Option is a functional option for Repository. -type Option func(o *options) error - -type options struct { - // remoteOpts are the options to use when fetching and pushing blobs. - remoteOpts []remote.Option -} - -// ResourceOptions contains all parameters necessary to fetch / push resources. -type ResourceOptions struct { - // TODO: Check for replacement - // ComponentVersion *v1alpha1.ComponentVersion - Resource v1alpha1.ResourceReference - Owner metav1.Object - SnapshotName string -} - -// Repository is a wrapper around go-container registry's name.Repository. -// It provides a few convenience methods for interacting with OCI registries. -type Repository struct { - ociname.Repository - options -} - -// NewRepository returns a new Repository. It points to the given remote repository. -// It accepts a list of options to configure the repository and the underlying remote client. -func NewRepository(repositoryName string, opts ...Option) (*Repository, error) { - opt, err := makeOptions(opts...) - if err != nil { - return nil, fmt.Errorf("failed to make options: %w", err) - } - repoOpts := make([]ociname.Option, 0) - - repo, err := ociname.NewRepository(repositoryName, repoOpts...) - if err != nil { - return nil, fmt.Errorf("failed to parse Repository name %q: %w", repositoryName, err) - } - - return &Repository{repo, opt}, nil -} - -// head does an authenticated call with the repo context to see if a tag in a repository already exists or not. -func (r *Repository) head(tag string) (bool, error) { - reference, err := ociname.ParseReference(fmt.Sprintf("%s:%s", r.Repository, tag)) - if err != nil { - return false, fmt.Errorf("failed to parse repository and tag name: %w", err) - } - - if _, err := remote.Head(reference, r.remoteOpts...); err != nil { - terr := &transport.Error{} - if ok := errors.As(err, &terr); ok { - if terr.StatusCode == http.StatusNotFound { - return false, nil - } - } - - return false, err - } - - return true, nil -} - -// deleteTag fetches the latest digest for a tag. This will delete the whole Manifest. -// This is done because docker registry doesn't technically support deleting a single Tag. -// But since we have a 1:1 relationship between a tag and a manifest, it's safe to delete -// the complete manifest. -func (r *Repository) deleteTag(tag string) error { - ref, err := ociname.NewTag(fmt.Sprintf("%s:%s", r.Repository, tag)) - if err != nil { - return fmt.Errorf("failed to parse reference: %w", err) - } - desc, err := remote.Head(ref, r.remoteOpts...) - if err != nil { - return fmt.Errorf("failed to fetch head for reference: %w", err) - } - - deleteRef, err := parseReference(desc.Digest.String(), r) - if err != nil { - return fmt.Errorf("failed to construct reference for calculated digest: %w", err) - } - - if err := remote.Delete(deleteRef, r.remoteOpts...); err != nil { - return fmt.Errorf("failed to delete ref '%s': %w", ref, err) - } - - return nil -} - -// fetchBlob fetches a blob from the repository. -func (r *Repository) fetchBlob(digest string) (v1.Layer, error) { - ref, err := ociname.NewDigest(fmt.Sprintf("%s@%s", r.Repository, digest)) - if err != nil { - return nil, fmt.Errorf("failed to parse digest %q: %w", digest, err) - } - - return remote.Layer(ref, r.remoteOpts...) -} - -// FetchBlob fetches a blob from the repository. -func (r *Repository) FetchBlob(digest string) (io.ReadCloser, error) { - l, err := r.fetchBlob(digest) - if err != nil { - return nil, fmt.Errorf("failed to fetch layer: %w", err) - } - - return l.Uncompressed() -} - -// PushStreamBlob pushes by streaming a blob to the repository. It accepts an io.ReadCloser interface. -// A media type can be specified to override the default media type. -// Default media type is "application/vnd.oci.image.layer.v1.tar+gzip". -func (r *Repository) PushStreamBlob(blob io.ReadCloser, mediaType string) (*ocispec.Descriptor, error) { - t := types.MediaType(mediaType) - if t == "" { - t = types.OCILayer - } - layer := stream.NewLayer(blob, stream.WithMediaType(t)) - err := r.pushBlob(layer) - if err != nil { - return nil, fmt.Errorf("failed to push layer: %w", err) - } - desc, err := layerToOCIDescriptor(layer) - if err != nil { - return nil, fmt.Errorf("failed to get layer descriptor: %w", err) - } - - return desc, nil -} - -// pushBlob pushes a blob to the repository. It accepts a v1.Layer interface. -func (r *Repository) pushBlob(layer v1.Layer) error { - return remote.WriteLayer(r.Repository, layer, r.remoteOpts...) -} - -// PushStreamingImage pushes a reader to the repository as a streaming OCI image. -// It accepts a media type and a byte slice as the blob. -// Default media type is "application/vnd.oci.image.layer.v1.tar+gzip". -// Annotations can be passed to the image manifest. -func (r *Repository) PushStreamingImage( - reference string, - reader io.ReadCloser, - mediaType string, - annotations map[string]string, -) (*v1.Manifest, error) { - ref, err := parseReference(reference, r) - if err != nil { - return nil, fmt.Errorf("failed to parse reference: %w", err) - } - image, err := computeStreamImage(reader, mediaType) - if err != nil { - return nil, fmt.Errorf("failed to compute image: %w", err) - } - if len(annotations) > 0 { - i, ok := mutate.Annotations(image, annotations).(v1.Image) - if !ok { - return nil, fmt.Errorf("returned object was not an Image") - } - - image = i - } - - // These MediaTypes are required to create a Helm compliant OCI repository. - if mediaType == registry.ChartLayerMediaType { - image = mutate.ConfigMediaType(image, registry.ConfigMediaType) - image = mutate.MediaType(image, ocispec.MediaTypeImageManifest) - } - - if err := r.pushImage(image, ref); err != nil { - return nil, fmt.Errorf("failed to push image: %w", err) - } - - return image.Manifest() -} - -// pushImage pushes an OCI image to the repository. It accepts a v1.RepositoryURL interface. -func (r *Repository) pushImage(image v1.Image, reference ociname.Reference) error { - return remote.Write(reference, image, r.remoteOpts...) -} - -// FetchManifest fetches a manifest from the repository. -// It returns the manifest as an oci.Manifest struct and the raw manifest as a byte slice. -// The oci.Manifest struct can be used to retrieve the layers digests. -// Optionally, the manifest annotations can be verified against the given slice of strings keys. -func (r *Repository) FetchManifest(reference string, filters []string) (*ocispec.Manifest, []byte, error) { - ref, err := parseReference(reference, r) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse reference: %w", err) - } - m, err := r.fetchManifestDescriptor(ref.String()) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch manifest: %w", err) - } - raw, err := m.RawManifest() - if err != nil { - return nil, nil, fmt.Errorf("failed to get raw manifest: %w", err) - } - - // check if the manifest annotations match the given filters - var annotations map[string]string - if len(filters) > 0 { - // get descriptor from manifest - desc, err := getDescriptor(raw) - if err != nil { - return nil, nil, fmt.Errorf("failed to get descriptor: %w", err) - } - annotations = filterAnnotations(desc.Annotations, filters) - if len(annotations) == 0 { - return nil, nil, fmt.Errorf("no matching annotations found") - } - } - - desc, err := manifestToOCIDescriptor(m) - if err != nil { - return nil, nil, fmt.Errorf("failed to get manifest descriptor: %w", err) - } - - return desc, raw, nil -} - -func (r *Repository) fetchManifestDescriptor(s string) (*remote.Descriptor, error) { - return fetchManifestDescriptorFrom(s, r.remoteOpts...) -} - -// manifestToOCIDescriptor converts a manifest to an OCI Manifest struct. -// It contains the layers descriptors. -func manifestToOCIDescriptor(m *remote.Descriptor) (*ocispec.Manifest, error) { - ociManifest := &ocispec.Manifest{} - ociManifest.MediaType = string(m.MediaType) - image, err := m.Image() - if err != nil { - return nil, fmt.Errorf("failed to get image: %w", err) - } - layers, err := image.Layers() - if err != nil { - return nil, fmt.Errorf("failed to get layers: %w", err) - } - for _, layer := range layers { - ociLayer, err := layerToOCIDescriptor(layer) - if err != nil { - return nil, fmt.Errorf("failed to get layer: %w", err) - } - ociManifest.Layers = append(ociManifest.Layers, *ociLayer) - } - - return ociManifest, nil -} - -func fetchManifestDescriptorFrom(s string, opts ...remote.Option) (*remote.Descriptor, error) { - // a manifest reference can be a tag or a digest - ref, err := ociname.ParseReference(s) - if err != nil { - return nil, fmt.Errorf("failed to parse reference: %w", err) - } - // fetch manifest - // Get performs a digest verification - return remote.Get(ref, opts...) -} - -func parseReference(reference string, r *Repository) (ociname.Reference, error) { - if reference == "" { - return nil, fmt.Errorf("reference must be specified") - } - if strings.Contains(reference, "sha256:") { - reference = fmt.Sprintf("%s@%s", r.Repository, reference) - } else { - reference = fmt.Sprintf("%s:%s", r.Repository, reference) - } - ref, err := ociname.ParseReference(reference) - if err != nil { - return nil, fmt.Errorf("failed to parse reference: %w", err) - } - - return ref, nil -} - -// layerToOCIDescriptor converts a layer to an OCI Layer struct. -func layerToOCIDescriptor(layer v1.Layer) (*ocispec.Descriptor, error) { - ociLayer := &ocispec.Descriptor{} - mediaType, err := layer.MediaType() - if err != nil { - return nil, fmt.Errorf("failed to get media type: %w", err) - } - d, err := layer.Digest() - if err != nil { - return nil, fmt.Errorf("failed to get digest: %w", err) - } - size, err := layer.Size() - if err != nil { - return nil, fmt.Errorf("failed to get size: %w", err) - } - ociLayer.MediaType = string(mediaType) - ociLayer.Digest = digest.NewDigestFromHex(d.Algorithm, d.Hex) - ociLayer.Size = size - - return ociLayer, nil -} - -func makeOptions(opts ...Option) (options, error) { - opt := options{} - for _, o := range opts { - if err := o(&opt); err != nil { - return options{}, fmt.Errorf("failed to apply option: %w", err) - } - } - - return opt, nil -} - -// filterAnnotations filters the annotations of a map of annotations. -// It returns a map of annotations that match the given entries. -func filterAnnotations(annotations map[string]string, filters []string) map[string]string { - filtered := make(map[string]string) - for k, v := range annotations { - for _, match := range filters { - if strings.EqualFold(k, match) { - filtered[k] = v - } - } - } - - return filtered -} - -func computeStreamImage(reader io.ReadCloser, mediaType string) (v1.Image, error) { - return mutate.AppendLayers(empty.Image, computeStreamBlob(reader, mediaType)) -} - -func computeStreamBlob(reader io.ReadCloser, mediaType string) v1.Layer { - t := types.MediaType(mediaType) - if t == "" { - t = types.OCILayer - } - - return stream.NewLayer(reader, stream.WithMediaType(t)) -} - -func getDescriptor(manifest []byte) (*v1.Descriptor, error) { - desc := &v1.Descriptor{} - if err := json.Unmarshal(manifest, desc); err != nil { - return nil, fmt.Errorf("failed to unmarshal manifest: %w", err) - } - - return desc, nil -} diff --git a/pkg/oci/repository_test.go b/pkg/oci/repository_test.go deleted file mode 100644 index 55d71768..00000000 --- a/pkg/oci/repository_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package oci - -import ( - "bytes" - "io" - "strings" - "testing" - "time" - - . "github.com/onsi/gomega" - - "github.com/google/go-containerregistry/pkg/v1/types" -) - -func TestRepository_Blob(t *testing.T) { - addr := strings.TrimPrefix(testServer.URL, "http://") - testCases := []struct { - name string - blob []byte - expected []byte - }{ - { - name: "blob", - blob: []byte("blob"), - expected: []byte("blob"), - }, - { - name: "empty blob", - blob: []byte(""), - expected: []byte(""), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - // create a repository client - repoName := generateRandomName("testblob") - repo, err := NewRepository(addr + "/" + repoName) - g.Expect(err).NotTo(HaveOccurred()) - - // compute a blob - layer := computeStreamBlob(io.NopCloser(bytes.NewBuffer(tc.blob)), string(types.OCILayer)) - - // push blob to the registry - err = repo.pushBlob(layer) - g.Expect(err).NotTo(HaveOccurred()) - - // fetch the blob from the registry - digest, err := layer.Digest() - g.Expect(err).NotTo(HaveOccurred()) - rc, err := repo.FetchBlob(digest.String()) - g.Expect(err).NotTo(HaveOccurred()) - b, err := io.ReadAll(rc) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(b).To(Equal(tc.expected)) - }) - } -} - -func TestRepository_StreamImage(t *testing.T) { - addr := strings.TrimPrefix(testServer.URL, "http://") - testCases := []struct { - name string - blob []byte - expected []byte - }{ - { - name: "image", - blob: []byte("image"), - expected: []byte("image"), - }, - { - name: "empty image", - blob: []byte(""), - expected: []byte(""), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - // create a repository client - repoName := generateRandomName("testimage") - repo, err := NewRepository(addr + "/" + repoName) - g.Expect(err).NotTo(HaveOccurred()) - - // push image to the registry - blob := tc.blob - reader := io.NopCloser(bytes.NewBuffer(blob)) - manifest, err := repo.PushStreamingImage("latest", reader, string(types.OCILayer), map[string]string{ - "org.opencontainers.artifact.created": time.Now().UTC().Format(time.RFC3339), - }) - g.Expect(err).NotTo(HaveOccurred()) - digest := manifest.Layers[0].Digest.String() - layer, err := repo.FetchBlob(digest) - g.Expect(err).NotTo(HaveOccurred()) - b, err := io.ReadAll(layer) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(b).To(Equal(tc.expected)) - }) - } -} diff --git a/pkg/oci/suite_test.go b/pkg/oci/suite_test.go deleted file mode 100644 index 8a8a8d15..00000000 --- a/pkg/oci/suite_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package oci - -import ( - "context" - "fmt" - "math/rand" - "net/http" - "net/http/httptest" - "os" - "testing" - "time" - - "github.com/distribution/distribution/v3/configuration" - "github.com/distribution/distribution/v3/registry/handlers" - _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" - - "github.com/phayes/freeport" -) - -var testServer *httptest.Server - -// Server is a registry server -// It wraps the http.Server -type Server struct { - http.Server - config *configuration.Configuration -} - -func TestMain(m *testing.M) { - ctx := context.Background() - port, err := freeport.GetFreePort() - if err != nil { - panic(fmt.Errorf("could not get free port: %w", err)) - } - app, err := New(ctx, fmt.Sprintf("127.0.0.1:%d", port)) - if err != nil { - panic(fmt.Errorf("could not create registry server: %w", err)) - } - testServer = httptest.NewServer(app.Handler) - defer testServer.Close() - exitCode := m.Run() - os.Exit(exitCode) -} - -// New creates a new oci registry server -func New(ctx context.Context, addr string) (*Server, error) { - config, err := getConfig(addr) - if err != nil { - return nil, fmt.Errorf("could not get config: %w", err) - } - app := handlers.NewApp(ctx, config) - return &Server{ - http.Server{ - Addr: addr, - Handler: app, - ReadHeaderTimeout: 1 * time.Second, - }, - config, - }, nil -} - -func getConfig(addr string) (*configuration.Configuration, error) { - config := &configuration.Configuration{} - config.HTTP.Addr = addr - config.HTTP.DrainTimeout = time.Duration(10) * time.Second - config.Storage = map[string]configuration.Parameters{ - "inmemory": map[string]interface{}{}, - "delete": map[string]interface{}{ - "enabled": true, - }, - } - config.HTTP.DrainTimeout = time.Duration(10) * time.Second - return config, nil -} - -func generateRandomName(name string) string { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - return fmt.Sprintf("%s-%d", name, r.Intn(1000)) -} diff --git a/pkg/snapshot/registry.go b/pkg/snapshot/registry.go new file mode 100644 index 00000000..132d541a --- /dev/null +++ b/pkg/snapshot/registry.go @@ -0,0 +1,40 @@ +package snapshot + +import ( + "context" + "errors" + + "oras.land/oras-go/v2/registry/remote" +) + +// A RegistryType is something that can create a Repository. +type RegistryType interface { + NewRepository(ctx context.Context, name string) (RepositoryType, error) +} + +type Registry struct { + *remote.Registry +} + +func NewRegistry(url string) (*Registry, error) { + registry, err := remote.NewRegistry(url) + if err != nil { + return nil, err + } + + return &Registry{registry}, nil +} + +func (r *Registry) NewRepository(ctx context.Context, name string) (RepositoryType, error) { + repository, err := r.Repository(ctx, name) + if err != nil { + return nil, err + } + + remoteRepository, ok := repository.(*remote.Repository) + if !ok { + return nil, errors.New("invalid repository type") + } + + return &Repository{remoteRepository}, nil +} diff --git a/pkg/snapshot/repository.go b/pkg/snapshot/repository.go new file mode 100644 index 00000000..c4bf1581 --- /dev/null +++ b/pkg/snapshot/repository.go @@ -0,0 +1,130 @@ +package snapshot + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + "github.com/mitchellh/hashstructure/v2" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/registry/remote" + "sigs.k8s.io/controller-runtime/pkg/log" + + ociV1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +const OCISchemaVersion = 2 + +// A RepositoryType is a type that can push and fetch blobs. +type RepositoryType interface { + // PushSnapshot pushes the blob to its repository. It returns the manifest-digest to retrieve the blob. + PushSnapshot(ctx context.Context, reference string, blob []byte) (digest.Digest, error) + + FetchSnapshot(ctx context.Context, reference string) ([]byte, error) +} + +type Repository struct { + *remote.Repository +} + +func (r *Repository) PushSnapshot(ctx context.Context, tag string, blob []byte) (digest.Digest, error) { + logger := log.FromContext(ctx) + + // Prepare and upload blob + blobDescriptor := ociV1.Descriptor{ + MediaType: ociV1.MediaTypeImageLayer, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + } + + // Check if blob is already present + exists, err := r.Exists(ctx, blobDescriptor) + if err != nil { + return "", fmt.Errorf("error checking existence of blob: %w", err) + } + + if exists { + logger.Info("blob already exists", "descriptor", blobDescriptor) + + return "", nil + } + + logger.Info("pushing blob", "descriptor", blobDescriptor) + if err := r.Push(ctx, blobDescriptor, content.NewVerifyReader( + bytes.NewReader(blob), + blobDescriptor, + )); err != nil { + return "", fmt.Errorf("oci: error pushing blob: %w", err) + } + + // Prepare and upload image config + emptyImageConfig := []byte("{}") + + imageConfigDescriptor := ociV1.Descriptor{ + MediaType: ociV1.MediaTypeImageConfig, + Digest: digest.FromBytes(emptyImageConfig), + Size: int64(len(emptyImageConfig)), + } + + logger.Info("pushing OCI config") + if err := r.Push(ctx, imageConfigDescriptor, content.NewVerifyReader( + bytes.NewReader(emptyImageConfig), + imageConfigDescriptor, + )); err != nil { + return "", fmt.Errorf("oci: error pushing empty config: %w", err) + } + + // Prepare and upload manifest + manifest := ociV1.Manifest{ + Versioned: specs.Versioned{SchemaVersion: OCISchemaVersion}, + MediaType: ociV1.MediaTypeImageManifest, + Config: imageConfigDescriptor, + Layers: []ociV1.Descriptor{blobDescriptor}, + } + + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return "", fmt.Errorf("oci: error marshaling manifest: %w", err) + } + + manifestDigest := digest.FromBytes(manifestBytes) + + manifestDescriptor := ociV1.Descriptor{ + MediaType: manifest.MediaType, + Digest: manifestDigest, + Size: int64(len(manifestBytes)), + } + + logger.Info("pushing OCI manifest") + if err := r.Push(ctx, manifestDescriptor, content.NewVerifyReader( + bytes.NewReader(manifestBytes), + manifestDescriptor, + )); err != nil { + return "", fmt.Errorf("oci: error pushing manifest: %w", err) + } + + // Tag manifest + if err := r.Tag(ctx, manifestDescriptor, tag); err != nil { + return "", fmt.Errorf("oci: error tagging manifest: %w", err) + } + + return manifestDigest, nil +} + +func (*Repository) FetchSnapshot(_ context.Context, _ string) ([]byte, error) { + return []byte{}, nil +} + +// CreateRepositoryName creates a name for an OCI repository and returns a hashed string from the passed arguments. The +// purpose of this function is to sanitize any passed string to an OCI repository compliant name. +func CreateRepositoryName(args ...string) (string, error) { + hash, err := hashstructure.Hash(args, hashstructure.FormatV2, nil) + if err != nil { + return "", fmt.Errorf("failed to hash identity: %w", err) + } + + return fmt.Sprintf("sha-%d", hash), nil +} diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go new file mode 100644 index 00000000..c68aea6e --- /dev/null +++ b/pkg/snapshot/snapshot.go @@ -0,0 +1,42 @@ +package snapshot + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/util/validation" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/open-component-model/ocm-k8s-toolkit/api/v1alpha1" +) + +// generateName generates a name for a snapshot CR. If the name exceeds the character limit, it will be cut off at 256. +func generateName(obj v1alpha1.SnapshotWriter) string { + name := strings.ToLower(fmt.Sprintf("%s-%s", obj.GetKind(), obj.GetName())) + + if len(name) > validation.DNS1123SubdomainMaxLength { + return name[:validation.DNS1123SubdomainMaxLength] + } + + return name +} + +func Create(owner v1alpha1.SnapshotWriter, ociRepository, manifestDigest, blobVersion, blobDigest string, blobSize int64) v1alpha1.Snapshot { + return v1alpha1.Snapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: generateName(owner), + Namespace: owner.GetNamespace(), + }, + Spec: v1alpha1.SnapshotSpec{ + Repository: ociRepository, + Digest: manifestDigest, + Blob: v1alpha1.BlobInfo{ + Digest: blobDigest, + Tag: blobVersion, + Size: blobSize, + }, + }, + Status: v1alpha1.SnapshotStatus{}, + } +}