From c5699dc9ea87186396d45300aca5ecea03072136 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 17 Mar 2025 19:26:23 +0100 Subject: [PATCH] test/e2e: add virtualworkspace test Signed-off-by: Dr. Stefan Schimanski --- envtest/internal/controlplane/plane.go | 6 - envtest/internal/controlplane/shard.go | 1 + test/e2e/apiexport_test.go | 230 +++++++++++++++++++++++++ test/e2e/suite_test.go | 14 ++ virtualworkspace/provider.go | 7 +- 5 files changed, 249 insertions(+), 9 deletions(-) create mode 100644 test/e2e/apiexport_test.go diff --git a/envtest/internal/controlplane/plane.go b/envtest/internal/controlplane/plane.go index 9247e14..60b4fec 100644 --- a/envtest/internal/controlplane/plane.go +++ b/envtest/internal/controlplane/plane.go @@ -22,14 +22,8 @@ import ( kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/rest" - - "github.com/kcp-dev/multicluster-provider/envtest/internal/certs" ) -// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY. -// Don't use this for anything else! -var NewTinyCA = certs.NewTinyCA - // Kcp is a struct that knows how to start your test kcp. // // Right now, that means one kcp shard. This is likely to increase in diff --git a/envtest/internal/controlplane/shard.go b/envtest/internal/controlplane/shard.go index 05aca09..3754240 100644 --- a/envtest/internal/controlplane/shard.go +++ b/envtest/internal/controlplane/shard.go @@ -262,6 +262,7 @@ func (s *Shard) defaultArgs() map[string][]string { "bind-address": {s.SecureServing.Address}, "embedded-etcd-peer-port": {s.EmbeddedEtcd.PeerPort}, "embedded-etcd-client-port": {s.EmbeddedEtcd.ClientPort}, + "external-hostname": {s.SecureServing.Address}, } return args } diff --git a/test/e2e/apiexport_test.go b/test/e2e/apiexport_test.go new file mode 100644 index 0000000..1459d1b --- /dev/null +++ b/test/e2e/apiexport_test.go @@ -0,0 +1,230 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/yaml" + + apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/core" + tenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + "github.com/kcp-dev/logicalcluster/v3" + + mcbuilder "github.com/multicluster-runtime/multicluster-runtime/pkg/builder" + mcmanager "github.com/multicluster-runtime/multicluster-runtime/pkg/manager" + mcreconcile "github.com/multicluster-runtime/multicluster-runtime/pkg/reconcile" + + clusterclient "github.com/kcp-dev/multicluster-provider/client" + "github.com/kcp-dev/multicluster-provider/envtest" + "github.com/kcp-dev/multicluster-provider/virtualworkspace" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("VirtualWorkspace Provider", Ordered, func() { + var ( + ctx context.Context + cancel context.CancelFunc + + cli clusterclient.ClusterClient + provider, consumer logicalcluster.Path + consumerWS *tenancyv1alpha1.Workspace + mgr mcmanager.Manager + vwEndpoint string + ) + + BeforeAll(func() { + ctx, cancel = context.WithCancel(context.Background()) + + var err error + cli, err = clusterclient.New(kcpConfig, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + + _, provider = envtest.NewWorkspaceFixture(GinkgoT(), cli, core.RootCluster.Path(), envtest.WithNamePrefix("provider")) + consumerWS, consumer = envtest.NewWorkspaceFixture(GinkgoT(), cli, core.RootCluster.Path(), envtest.WithNamePrefix("consumer")) + + By(fmt.Sprintf("creating a schema in the provider workspace %q", provider)) + schema := &apisv1alpha1.APIResourceSchema{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v20250317.things.example.com", + }, + Spec: apisv1alpha1.APIResourceSchemaSpec{ + Group: "example.com", + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Kind: "Thing", + ListKind: "ThingList", + Plural: "things", + Singular: "thing", + }, + Scope: apiextensionsv1.ClusterScoped, + Versions: []apisv1alpha1.APIResourceVersion{{ + Name: "v1", + Schema: runtime.RawExtension{ + Raw: []byte(`{"type":"object","properties":{"spec":{"type":"object","properties":{"message":{"type":"string"}}}}}`), + }, + Storage: true, + }}, + }, + } + err = cli.Cluster(provider).Create(ctx, schema) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("creating an APIExport in the provider workspace %q", provider)) + export := &apisv1alpha1.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example.com", + }, + Spec: apisv1alpha1.APIExportSpec{ + LatestResourceSchemas: []string{schema.Name}, + }, + } + err = cli.Cluster(provider).Create(ctx, export) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("creating an APIExportEndpointSlice in the provider workspace %q", provider)) + endpoitns := &apisv1alpha1.APIExportEndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example.com", + }, + Spec: apisv1alpha1.APIExportEndpointSliceSpec{ + APIExport: apisv1alpha1.ExportBindingReference{ + Path: provider.String(), + Name: export.Name, + }, + }, + } + err = cli.Cluster(provider).Create(ctx, endpoitns) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("creating an APIBinding in the consumer workspace %q", consumer)) + binding := &apisv1alpha1.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example.com", + }, + Spec: apisv1alpha1.APIBindingSpec{ + Reference: apisv1alpha1.BindingReference{ + Export: &apisv1alpha1.ExportBindingReference{ + Path: provider.String(), + Name: export.Name, + }, + }, + }, + } + err = cli.Cluster(consumer).Create(ctx, binding) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("waiting until the APIExportEndpointSlice in the provider workspace %q to have endpoints", provider)) + endpoints := &apisv1alpha1.APIExportEndpointSlice{} + envtest.Eventually(GinkgoT(), func() (bool, string) { + err := cli.Cluster(provider).Get(ctx, client.ObjectKey{Name: "example.com"}, endpoints) + if err != nil { + return false, fmt.Sprintf("failed to get APIExportEndpointSlice in %s: %v", provider, err) + } + return len(endpoints.Status.APIExportEndpoints) > 0, toYAML(GinkgoT(), endpoints) + }, wait.ForeverTestTimeout, time.Millisecond*100, "failed to see endpoints in APIExportEndpointSlice in %s", provider) + vwEndpoint = endpoints.Status.APIExportEndpoints[0].URL + }) + + Describe("with a multicluster provider and manager", func() { + var ( + lock sync.RWMutex + engaged = sets.NewString() + g *errgroup.Group + cancelGroup context.CancelFunc + ) + + BeforeAll(func() { + By("creating a multicluster provider for APIBindings against the apiexport virtual workspace") + vwConfig := rest.CopyConfig(kcpConfig) + vwConfig.Host = vwEndpoint + p, err := virtualworkspace.New(vwConfig, &apisv1alpha1.APIBinding{}, virtualworkspace.Options{}) + Expect(err).NotTo(HaveOccurred()) + + By("creating a manager against the provider workspace") + rootConfig := rest.CopyConfig(kcpConfig) + rootConfig.Host += provider.RequestPath() + mgr, err = mcmanager.New(rootConfig, p, mcmanager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + By("creating a reconciler for the APIBinding") + err = mcbuilder.ControllerManagedBy(mgr). + Named("things"). + For(&apisv1alpha1.APIBinding{}). + Complete(mcreconcile.Func(func(ctx context.Context, request mcreconcile.Request) (reconcile.Result, error) { + By(fmt.Sprintf("reconciling APIBinding %s in cluster %q", request.Name, request.ClusterName)) + lock.Lock() + defer lock.Unlock() + engaged.Insert(request.ClusterName) + return reconcile.Result{}, nil + })) + Expect(err).NotTo(HaveOccurred()) + + By("starting the provider and manager") + var groupContext context.Context + groupContext, cancelGroup = context.WithCancel(ctx) + g, groupContext = errgroup.WithContext(groupContext) + g.Go(func() error { + return p.Run(groupContext, mgr) + }) + g.Go(func() error { + return mgr.Start(groupContext) + }) + }) + + It("sees the consumer workspace as a cluster", func() { + By("watching the clusters the reconciler has seen") + envtest.Eventually(GinkgoT(), func() (bool, string) { + lock.RLock() + defer lock.RUnlock() + return engaged.Has(consumerWS.Spec.Cluster), fmt.Sprintf("failed to see the consumer workspace %q as a cluster: %v", consumerWS.Spec.Cluster, engaged.List()) + }, wait.ForeverTestTimeout, time.Millisecond*100, "failed to see the consumer workspace %q as a cluster", consumer) + }) + + AfterAll(func() { + cancelGroup() + err := g.Wait() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + AfterAll(func() { + cancel() + }) +}) + +func toYAML(t require.TestingT, x any) string { + y, err := yaml.Marshal(x) + require.NoError(t, err) + return string(y) +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 7900dae..47127f1 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -21,11 +21,18 @@ import ( "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + tenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + topologyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/topology/v1alpha1" + "github.com/kcp-dev/multicluster-provider/envtest" . "github.com/onsi/ginkgo/v2" @@ -37,6 +44,13 @@ var ( kcpConfig *rest.Config ) +func init() { + runtime.Must(apisv1alpha1.AddToScheme(scheme.Scheme)) + runtime.Must(corev1alpha1.AddToScheme(scheme.Scheme)) + runtime.Must(tenancyv1alpha1.AddToScheme(scheme.Scheme)) + runtime.Must(topologyv1alpha1.AddToScheme(scheme.Scheme)) +} + func TestE2e(t *testing.T) { RegisterFailHandler(Fail) diff --git a/virtualworkspace/provider.go b/virtualworkspace/provider.go index eaa3985..c44fbc4 100644 --- a/virtualworkspace/provider.go +++ b/virtualworkspace/provider.go @@ -108,7 +108,7 @@ func (p *Provider) Run(ctx context.Context, mgr mcmanager.Manager) error { // Watch logical clusters and engage them as clusters in multicluster-runtime. inf, err := p.cache.GetInformer(ctx, p.object, cache.BlockUntilSynced(false)) if err != nil { - return fmt.Errorf("failed to get logical cluster informer: %w", err) + return fmt.Errorf("failed to get %T informer: %w", p.object, err) } shInf, _, _, err := p.cache.getSharedInformer(p.object) if err != nil { @@ -206,8 +206,9 @@ func (p *Provider) Run(ctx context.Context, mgr mcmanager.Manager) error { syncCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - if !p.cache.WaitForCacheSync(syncCtx) { - return fmt.Errorf("failed to sync wildcard cache") + + if _, err := p.cache.GetInformer(syncCtx, p.object, cache.BlockUntilSynced(true)); err != nil { + return fmt.Errorf("failed to sync %T informer: %w", p.object, err) } return g.Wait()