diff --git a/.golangci.yaml b/.golangci.yaml index 422a473..b995c1c 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -31,7 +31,6 @@ linters: - gocritic - gocyclo - gofmt - - goimports - goprintffuncname - gosimple - govet diff --git a/Makefile b/Makefile index 0493cac..e4a6552 100644 --- a/Makefile +++ b/Makefile @@ -106,5 +106,5 @@ verify: ./hack/verify-licenses.sh .PHONY: test -test: +test: $(KCP) ./hack/run-tests.sh diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..66f0272 --- /dev/null +++ b/client/client.go @@ -0,0 +1,84 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + "sync" + + "github.com/hashicorp/golang-lru/v2" + + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kcp-dev/logicalcluster/v3" +) + +// ClusterClient is a cluster-aware client. +type ClusterClient interface { + // Cluster returns the client for the given cluster. + Cluster(cluster logicalcluster.Path) client.Client +} + +// clusterClient is a multi-cluster-aware client. +type clusterClient struct { + baseConfig *rest.Config + opts client.Options + + lock sync.RWMutex + cache *lru.Cache[logicalcluster.Path, client.Client] +} + +// New creates a new cluster-aware client. +func New(cfg *rest.Config, options client.Options) (ClusterClient, error) { + ca, err := lru.New[logicalcluster.Path, client.Client](100) + if err != nil { + return nil, err + } + return &clusterClient{ + opts: options, + baseConfig: cfg, + cache: ca, + }, nil +} + +func (c *clusterClient) Cluster(cluster logicalcluster.Path) client.Client { + // quick path + c.lock.RLock() + cli, ok := c.cache.Get(cluster) + c.lock.RUnlock() + if ok { + return cli + } + + // slow path + c.lock.Lock() + defer c.lock.Unlock() + if cli, ok := c.cache.Get(cluster); ok { + return cli + } + + // cache miss + cfg := rest.CopyConfig(c.baseConfig) + cfg.Host += cluster.RequestPath() + cli, err := client.New(cfg, c.opts) + if err != nil { + panic(fmt.Errorf("failed to create client for cluster %s: %w", cluster, err)) + } + c.cache.Add(cluster, cli) + return cli +} diff --git a/envtest/doc.go b/envtest/doc.go new file mode 100644 index 0000000..411a084 --- /dev/null +++ b/envtest/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package envtest provides a test environment for testing code against a +// kcp control plane. +package envtest diff --git a/envtest/eventually.go b/envtest/eventually.go new file mode 100644 index 0000000..7d54661 --- /dev/null +++ b/envtest/eventually.go @@ -0,0 +1,141 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "fmt" + "time" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" + + conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" +) + +// Eventually asserts that given condition will be met in waitFor time, periodically checking target function +// each tick. In addition to require.Eventually, this function t.Logs the reason string value returned by the condition +// function (eventually after 20% of the wait time) to aid in debugging. +func Eventually(t TestingT, condition func() (success bool, reason string), waitFor time.Duration, tick time.Duration, msgAndArgs ...interface{}) { + t.Helper() + + var last string + start := time.Now() + require.Eventually(t, func() bool { + t.Helper() + + ok, msg := condition() + if time.Since(start) > waitFor/5 { + if !ok && msg != "" && msg != last { + last = msg + t.Logf("Waiting for condition, but got: %s", msg) + } else if ok && msg != "" && last != "" { + t.Logf("Condition became true: %s", msg) + } + } + return ok + }, waitFor, tick, msgAndArgs...) +} + +// EventuallyReady asserts that the object returned by getter() eventually has a ready condition. +func EventuallyReady(t TestingT, getter func() (conditions.Getter, error), msgAndArgs ...interface{}) { + t.Helper() + EventuallyCondition(t, getter, Is(conditionsv1alpha1.ReadyCondition, corev1.ConditionTrue), msgAndArgs...) +} + +// ConditionEvaluator is a helper for evaluating conditions. +type ConditionEvaluator struct { + conditionType conditionsv1alpha1.ConditionType + isStatus *corev1.ConditionStatus + isNotStatus *corev1.ConditionStatus + reason *string +} + +func (c *ConditionEvaluator) matches(object conditions.Getter) (*conditionsv1alpha1.Condition, string, bool) { + condition := conditions.Get(object, c.conditionType) + if condition == nil { + return nil, c.descriptor(), false + } + if c.isStatus != nil && condition.Status != *c.isStatus { + return condition, c.descriptor(), false + } + if c.isNotStatus != nil && condition.Status == *c.isNotStatus { + return condition, c.descriptor(), false + } + if c.reason != nil && condition.Reason != *c.reason { + return condition, c.descriptor(), false + } + return condition, c.descriptor(), true +} + +func (c *ConditionEvaluator) descriptor() string { + var descriptor string + if c.isStatus != nil { + descriptor = fmt.Sprintf("%s to be %s", c.conditionType, *c.isStatus) + } + if c.isNotStatus != nil { + descriptor = fmt.Sprintf("%s not to be %s", c.conditionType, *c.isNotStatus) + } + if c.reason != nil { + descriptor += fmt.Sprintf(" (with reason %s)", *c.reason) + } + return descriptor +} + +// Is matches if the given condition type is of the given value. +func Is(conditionType conditionsv1alpha1.ConditionType, s corev1.ConditionStatus) *ConditionEvaluator { + return &ConditionEvaluator{ + conditionType: conditionType, + isStatus: ptr.To(s), + } +} + +// IsNot matches if the given condition type is not of the given value. +func IsNot(conditionType conditionsv1alpha1.ConditionType, s corev1.ConditionStatus) *ConditionEvaluator { + return &ConditionEvaluator{ + conditionType: conditionType, + isNotStatus: ptr.To(s), + } +} + +// WithReason matches if the given condition type has the given reason. +func (c *ConditionEvaluator) WithReason(reason string) *ConditionEvaluator { + c.reason = &reason + return c +} + +// EventuallyCondition asserts that the object returned by getter() eventually has a condition that matches the evaluator. +func EventuallyCondition(t TestingT, getter func() (conditions.Getter, error), evaluator *ConditionEvaluator, msgAndArgs ...interface{}) { + t.Helper() + Eventually(t, func() (bool, string) { + obj, err := getter() + require.NoError(t, err, "Error fetching object") + condition, descriptor, done := evaluator.matches(obj) + var reason string + if !done { + if condition != nil { + reason = fmt.Sprintf("Not done waiting for object %s: %s: %s", descriptor, condition.Reason, condition.Message) + } else { + reason = fmt.Sprintf("Not done waiting for object %s: no condition present", descriptor) + } + } + return done, reason + }, wait.ForeverTestTimeout, 100*time.Millisecond, msgAndArgs...) +} diff --git a/envtest/internal/addr/addr_suite_test.go b/envtest/internal/addr/addr_suite_test.go new file mode 100644 index 0000000..3869bb0 --- /dev/null +++ b/envtest/internal/addr/addr_suite_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addr_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAddr(t *testing.T) { + t.Parallel() + RegisterFailHandler(Fail) + RunSpecs(t, "Addr Suite") +} diff --git a/envtest/internal/addr/manager.go b/envtest/internal/addr/manager.go new file mode 100644 index 0000000..5513e83 --- /dev/null +++ b/envtest/internal/addr/manager.go @@ -0,0 +1,142 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addr + +import ( + "errors" + "fmt" + "io/fs" + "net" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/flock" +) + +// TODO(directxman12): interface / release functionality for external port managers + +const ( + portReserveTime = 2 * time.Minute + portConflictRetry = 100 + portFilePrefix = "port-" +) + +var ( + cacheDir string +) + +func init() { + baseDir, err := os.UserCacheDir() + if err == nil { + cacheDir = filepath.Join(baseDir, "kubebuilder-envtest") + err = os.MkdirAll(cacheDir, 0o750) + } + if err != nil { + // Either we didn't get a cache directory, or we can't use it + baseDir = os.TempDir() + cacheDir = filepath.Join(baseDir, "kubebuilder-envtest") + err = os.MkdirAll(cacheDir, 0o750) + } + if err != nil { + panic(err) + } +} + +type portCache struct{} + +func (c *portCache) add(port int) (bool, error) { + // Remove outdated ports. + if err := fs.WalkDir(os.DirFS(cacheDir), ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !d.Type().IsRegular() || !strings.HasPrefix(path, portFilePrefix) { + return nil + } + info, err := d.Info() + if err != nil { + // No-op if file no longer exists; may have been deleted by another + // process/thread trying to allocate ports. + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if time.Since(info.ModTime()) > portReserveTime { + if err := os.Remove(filepath.Join(cacheDir, path)); err != nil { + // No-op if file no longer exists; may have been deleted by another + // process/thread trying to allocate ports. + if os.IsNotExist(err) { + return nil + } + return err + } + } + return nil + }); err != nil { + return false, err + } + // Try allocating new port, by acquiring a file. + path := fmt.Sprintf("%s/%s%d", cacheDir, portFilePrefix, port) + if err := flock.Acquire(path); errors.Is(err, flock.ErrAlreadyLocked) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + +var cache = &portCache{} + +func suggest(listenHost string) (*net.TCPListener, int, string, error) { + if listenHost == "" { + listenHost = "localhost" + } + addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(listenHost, "0")) + if err != nil { + return nil, -1, "", err + } + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return nil, -1, "", err + } + return l, l.Addr().(*net.TCPAddr).Port, + addr.IP.String(), + nil +} + +// Suggest suggests an address a process can listen on. It returns +// a tuple consisting of a free port and the hostname resolved to its IP. +// It makes sure that new port allocated does not conflict with old ports +// allocated within 1 minute. +func Suggest(listenHost string) (int, string, error) { + for i := 0; i < portConflictRetry; i++ { + listener, port, resolvedHost, err := suggest(listenHost) + if err != nil { + return -1, "", err + } + defer listener.Close() + if ok, err := cache.add(port); ok { + return port, resolvedHost, nil + } else if err != nil { + return -1, "", err + } + } + return -1, "", fmt.Errorf("no free ports found after %d retries", portConflictRetry) +} diff --git a/envtest/internal/addr/manager_test.go b/envtest/internal/addr/manager_test.go new file mode 100644 index 0000000..c77ba8c --- /dev/null +++ b/envtest/internal/addr/manager_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addr_test + +import ( + "net" + "strconv" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/addr" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SuggestAddress", func() { + It("returns a free port and an address to bind to", func() { + port, host, err := addr.Suggest("") + + Expect(err).NotTo(HaveOccurred()) + Expect(host).To(Or(Equal("127.0.0.1"), Equal("::1"))) + Expect(port).NotTo(Equal(0)) + + addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, strconv.Itoa(port))) + Expect(err).NotTo(HaveOccurred()) + l, err := net.ListenTCP("tcp", addr) + defer func() { + Expect(l.Close()).To(Succeed()) + }() + Expect(err).NotTo(HaveOccurred()) + }) + + It("supports an explicit listenHost", func() { + port, host, err := addr.Suggest("localhost") + + Expect(err).NotTo(HaveOccurred()) + Expect(host).To(Or(Equal("127.0.0.1"), Equal("::1"))) + Expect(port).NotTo(Equal(0)) + + addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, strconv.Itoa(port))) + Expect(err).NotTo(HaveOccurred()) + l, err := net.ListenTCP("tcp", addr) + defer func() { + Expect(l.Close()).To(Succeed()) + }() + Expect(err).NotTo(HaveOccurred()) + }) + + It("supports a 0.0.0.0 listenHost", func() { + port, host, err := addr.Suggest("0.0.0.0") + + Expect(err).NotTo(HaveOccurred()) + Expect(host).To(Equal("0.0.0.0")) + Expect(port).NotTo(Equal(0)) + + addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, strconv.Itoa(port))) + Expect(err).NotTo(HaveOccurred()) + l, err := net.ListenTCP("tcp", addr) + defer func() { + Expect(l.Close()).To(Succeed()) + }() + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/envtest/internal/certs/certs_suite_test.go b/envtest/internal/certs/certs_suite_test.go new file mode 100644 index 0000000..3b3008c --- /dev/null +++ b/envtest/internal/certs/certs_suite_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certs_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestInternal(t *testing.T) { + t.Parallel() + RegisterFailHandler(Fail) + RunSpecs(t, "TinyCA (Internal Certs) Suite") +} diff --git a/envtest/internal/certs/tinyca.go b/envtest/internal/certs/tinyca.go new file mode 100644 index 0000000..b418823 --- /dev/null +++ b/envtest/internal/certs/tinyca.go @@ -0,0 +1,224 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certs + +// NB(directxman12): nothing has verified that this has good settings. In fact, +// the setting generated here are probably terrible, but they're fine for integration +// tests. These ABSOLUTELY SHOULD NOT ever be exposed in the public API. They're +// ONLY for use with envtest's ability to configure webhook testing. +// If I didn't otherwise not want to add a dependency on cfssl, I'd just use that. + +import ( + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + crand "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "time" + + certutil "k8s.io/client-go/util/cert" +) + +var ( + ellipticCurve = elliptic.P256() + bigOne = big.NewInt(1) +) + +// CertPair is a private key and certificate for use for client auth, as a CA, or serving. +type CertPair struct { + Key crypto.Signer + Cert *x509.Certificate +} + +// CertBytes returns the PEM-encoded version of the certificate for this pair. +func (k CertPair) CertBytes() []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: k.Cert.Raw, + }) +} + +// AsBytes encodes keypair in the appropriate formats for on-disk storage (PEM and +// PKCS8, respectively). +func (k CertPair) AsBytes() (cert []byte, key []byte, err error) { + cert = k.CertBytes() + + rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key) + if err != nil { + return nil, nil, fmt.Errorf("unable to encode private key: %w", err) + } + + key = pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Bytes: rawKeyData, + }) + + return cert, key, nil +} + +// TinyCA supports signing serving certs and client-certs, +// and can be used as an auth mechanism with envtest. +type TinyCA struct { + CA CertPair + orgName string + + nextSerial *big.Int +} + +// newPrivateKey generates a new private key of a relatively sane size (see +// rsaKeySize). +func newPrivateKey() (crypto.Signer, error) { + return ecdsa.GenerateKey(ellipticCurve, crand.Reader) +} + +// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY. +// Don't use this for anything else! +func NewTinyCA() (*TinyCA, error) { + caPrivateKey, err := newPrivateKey() + if err != nil { + return nil, fmt.Errorf("unable to generate private key for CA: %w", err) + } + caCfg := certutil.Config{CommonName: "envtest-environment", Organization: []string{"envtest"}} + caCert, err := certutil.NewSelfSignedCACert(caCfg, caPrivateKey) + if err != nil { + return nil, fmt.Errorf("unable to generate certificate for CA: %w", err) + } + + return &TinyCA{ + CA: CertPair{Key: caPrivateKey, Cert: caCert}, + orgName: "envtest", + nextSerial: big.NewInt(1), + }, nil +} + +func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) { + now := time.Now() + + key, err := newPrivateKey() + if err != nil { + return CertPair{}, fmt.Errorf("unable to create private key: %w", err) + } + + serial := new(big.Int).Set(c.nextSerial) + c.nextSerial.Add(c.nextSerial, bigOne) + + template := x509.Certificate{ + Subject: pkix.Name{CommonName: cfg.CommonName, Organization: cfg.Organization}, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + + // technically not necessary for testing, but let's set anyway just in case. + NotBefore: now.UTC(), + // 1 week -- the default for cfssl, and just long enough for a + // long-term test, but not too long that anyone would try to use this + // seriously. + NotAfter: now.Add(168 * time.Hour).UTC(), + } + + certRaw, err := x509.CreateCertificate(crand.Reader, &template, c.CA.Cert, key.Public(), c.CA.Key) + if err != nil { + return CertPair{}, fmt.Errorf("unable to create certificate: %w", err) + } + + cert, err := x509.ParseCertificate(certRaw) + if err != nil { + return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %w", err) + } + + return CertPair{ + Key: key, + Cert: cert, + }, nil +} + +// NewServingCert returns a new CertPair for a serving HTTPS on localhost (or other specified names). +func (c *TinyCA) NewServingCert(names ...string) (CertPair, error) { + if len(names) == 0 { + names = []string{"localhost"} + } + dnsNames, ips, err := resolveNames(names) + if err != nil { + return CertPair{}, err + } + + return c.makeCert(certutil.Config{ + CommonName: "localhost", + Organization: []string{c.orgName}, + AltNames: certutil.AltNames{ + DNSNames: dnsNames, + IPs: ips, + }, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + }) +} + +// ClientInfo describes some Kubernetes user for the purposes of creating +// client certificates. +type ClientInfo struct { + // Name is the user name (embedded as the cert's CommonName) + Name string + // Groups are the groups to which this user belongs (embedded as the cert's + // Organization) + Groups []string +} + +// NewClientCert produces a new CertPair suitable for use with Kubernetes +// client cert auth with an API server validating based on this CA. +func (c *TinyCA) NewClientCert(user ClientInfo) (CertPair, error) { + return c.makeCert(certutil.Config{ + CommonName: user.Name, + Organization: user.Groups, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }) +} + +func resolveNames(names []string) ([]string, []net.IP, error) { + dnsNames := []string{} + ips := []net.IP{} + for _, name := range names { + if name == "" { + continue + } + ip := net.ParseIP(name) + if ip == nil { + dnsNames = append(dnsNames, name) + // Also resolve to IPs. + nameIPs, err := net.LookupHost(name) + if err != nil { + return nil, nil, err + } + for _, nameIP := range nameIPs { + ip = net.ParseIP(nameIP) + if ip != nil { + ips = append(ips, ip) + } + } + } else { + ips = append(ips, ip) + } + } + return dnsNames, ips, nil +} diff --git a/envtest/internal/certs/tinyca_test.go b/envtest/internal/certs/tinyca_test.go new file mode 100644 index 0000000..5ee6b58 --- /dev/null +++ b/envtest/internal/certs/tinyca_test.go @@ -0,0 +1,254 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package certs_test + +import ( + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "sort" + "time" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/certs" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" +) + +var _ = Describe("TinyCA", func() { + var ca *certs.TinyCA + + BeforeEach(func() { + var err error + ca, err = certs.NewTinyCA() + Expect(err).NotTo(HaveOccurred(), "should be able to initialize the CA") + }) + + Describe("the CA certs themselves", func() { + It("should be retrievable as a cert pair", func() { + Expect(ca.CA.Key).NotTo(BeNil(), "should have a key") + Expect(ca.CA.Cert).NotTo(BeNil(), "should have a cert") + }) + + It("should be usable for signing & verifying", func() { + Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageCertSign).NotTo(BeEquivalentTo(0), "should be usable for cert signing") + Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") + }) + }) + + It("should produce unique serials among all generated certificates of all types", func() { + By("generating a few cert pairs for both serving and client auth") + firstCerts, err := ca.NewServingCert() + Expect(err).NotTo(HaveOccurred()) + secondCerts, err := ca.NewClientCert(certs.ClientInfo{Name: "user"}) + Expect(err).NotTo(HaveOccurred()) + thirdCerts, err := ca.NewServingCert() + Expect(err).NotTo(HaveOccurred()) + + By("checking that they have different serials") + serials := []*big.Int{ + firstCerts.Cert.SerialNumber, + secondCerts.Cert.SerialNumber, + thirdCerts.Cert.SerialNumber, + } + // quick uniqueness check of numbers: sort, then you only have to compare sequential entries + sort.Slice(serials, func(i, j int) bool { + return serials[i].Cmp(serials[j]) == -1 + }) + Expect(serials[1].Cmp(serials[0])).NotTo(Equal(0), "serials shouldn't be equal") + Expect(serials[2].Cmp(serials[1])).NotTo(Equal(0), "serials shouldn't be equal") + }) + + Describe("Generated serving certs", func() { + It("should be valid for short enough to avoid production usage, but long enough for long-running tests", func() { + cert, err := ca.NewServingCert() + Expect(err).NotTo(HaveOccurred(), "should be able to generate the serving certs") + + duration := time.Until(cert.Cert.NotAfter) + Expect(duration).To(BeNumerically("<=", 168*time.Hour), "not-after should be short-ish (<= 1 week)") + Expect(duration).To(BeNumerically(">=", 2*time.Hour), "not-after should be enough for long tests (couple of hours)") + }) + + Context("when encoding names", func() { + var cert certs.CertPair + BeforeEach(func() { + By("generating a serving cert with IPv4 & IPv6 addresses, and DNS names") + var err error + // IPs are in the "example & docs" blocks for IPv4 (TEST-NET-1) & IPv6 + cert, err = ca.NewServingCert("192.0.2.1", "localhost", "2001:db8::") + Expect(err).NotTo(HaveOccurred(), "should be able to create the serving certs") + }) + + It("should encode all non-IP names as DNS SANs", func() { + Expect(cert.Cert.DNSNames).To(ConsistOf("localhost")) + }) + + It("should encode all IP names as IP SANs", func() { + // NB(directxman12): this is non-exhaustive because we also + // convert DNS SANs to IPs too (see test below) + Expect(cert.Cert.IPAddresses).To(ContainElements( + // normalize the elements with To16 so we can compare them to the output of + // of ParseIP safely (the alternative is a custom matcher that calls Equal, + // but this is easier) + WithTransform(net.IP.To16, Equal(net.ParseIP("192.0.2.1"))), + WithTransform(net.IP.To16, Equal(net.ParseIP("2001:db8::"))), + )) + }) + + It("should add the corresponding IP address(es) (as IP SANs) for DNS names", func() { + // NB(directxman12): we currently fail if the lookup fails. + // I'm not certain this is the best idea (both the bailing on + // error and the actual idea), so if this causes issues, you + // might want to reconsider. + + localhostAddrs, err := net.LookupHost("localhost") + Expect(err).NotTo(HaveOccurred(), "should be able to find IPs for localhost") + localhostIPs := make([]interface{}, len(localhostAddrs)) + for i, addr := range localhostAddrs { + // normalize the elements with To16 so we can compare them to the output of + // of ParseIP safely (the alternative is a custom matcher that calls Equal, + // but this is easier) + localhostIPs[i] = WithTransform(net.IP.To16, Equal(net.ParseIP(addr))) + } + Expect(cert.Cert.IPAddresses).To(ContainElements(localhostIPs...)) + }) + }) + + It("should assume a name of localhost (DNS SAN) if no names are given", func() { + cert, err := ca.NewServingCert() + Expect(err).NotTo(HaveOccurred(), "should be able to generate a serving cert with the default name") + Expect(cert.Cert.DNSNames).To(ConsistOf("localhost"), "the default DNS name should be localhost") + + }) + + It("should be usable for server auth, verifying, and enciphering", func() { + cert, err := ca.NewServingCert() + Expect(err).NotTo(HaveOccurred(), "should be able to generate a serving cert") + + Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(BeEquivalentTo(0), "should be usable for key enciphering") + Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") + Expect(cert.Cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageServerAuth), "should be usable for server auth") + + }) + + It("should be signed by the CA", func() { + cert, err := ca.NewServingCert() + Expect(err).NotTo(HaveOccurred(), "should be able to generate a serving cert") + Expect(cert.Cert.CheckSignatureFrom(ca.CA.Cert)).To(Succeed()) + }) + }) + + Describe("Generated client certs", func() { + var cert certs.CertPair + BeforeEach(func() { + var err error + cert, err = ca.NewClientCert(certs.ClientInfo{ + Name: "user", + Groups: []string{"group1", "group2"}, + }) + Expect(err).NotTo(HaveOccurred(), "should be able to create a client cert") + }) + + It("should be valid for short enough to avoid production usage, but long enough for long-running tests", func() { + duration := time.Until(cert.Cert.NotAfter) + Expect(duration).To(BeNumerically("<=", 168*time.Hour), "not-after should be short-ish (<= 1 week)") + Expect(duration).To(BeNumerically(">=", 2*time.Hour), "not-after should be enough for long tests (couple of hours)") + }) + + It("should be usable for client auth, verifying, and enciphering", func() { + Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(BeEquivalentTo(0), "should be usable for key enciphering") + Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") + Expect(cert.Cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageClientAuth), "should be usable for client auth") + }) + + It("should encode the user name as the common name", func() { + Expect(cert.Cert.Subject.CommonName).To(Equal("user")) + }) + + It("should encode the groups as the organization values", func() { + Expect(cert.Cert.Subject.Organization).To(ConsistOf("group1", "group2")) + }) + + It("should be signed by the CA", func() { + Expect(cert.Cert.CheckSignatureFrom(ca.CA.Cert)).To(Succeed()) + }) + }) +}) + +var _ = Describe("Certificate Pairs", func() { + var pair certs.CertPair + BeforeEach(func() { + ca, err := certs.NewTinyCA() + Expect(err).NotTo(HaveOccurred(), "should be able to generate a cert pair") + + pair = ca.CA + }) + + Context("when serializing just the public key", func() { + It("should serialize into a CERTIFICATE PEM block", func() { + bytes := pair.CertBytes() + Expect(bytes).NotTo(BeEmpty(), "should produce some cert bytes") + + block, rest := pem.Decode(bytes) + Expect(rest).To(BeEmpty(), "shouldn't have any data besides the PEM block") + + Expect(block).To(PointTo(MatchAllFields(Fields{ + "Type": Equal("CERTIFICATE"), + "Headers": BeEmpty(), + "Bytes": Equal(pair.Cert.Raw), + }))) + }) + }) + + Context("when serializing both parts", func() { + var certBytes, keyBytes []byte + BeforeEach(func() { + var err error + certBytes, keyBytes, err = pair.AsBytes() + Expect(err).NotTo(HaveOccurred(), "should be able to serialize the pair") + }) + + It("should serialize the private key in PKCS8 form in a PRIVATE KEY PEM block", func() { + Expect(keyBytes).NotTo(BeEmpty(), "should produce some key bytes") + + By("decoding & checking the PEM block") + block, rest := pem.Decode(keyBytes) + Expect(rest).To(BeEmpty(), "shouldn't have any data besides the PEM block") + + Expect(block.Type).To(Equal("PRIVATE KEY")) + + By("decoding & checking the PKCS8 data") + Expect(x509.ParsePKCS8PrivateKey(block.Bytes)).NotTo(BeNil(), "should be able to parse back the private key") + }) + + It("should serialize the public key into a CERTIFICATE PEM block", func() { + Expect(certBytes).NotTo(BeEmpty(), "should produce some cert bytes") + + block, rest := pem.Decode(certBytes) + Expect(rest).To(BeEmpty(), "shouldn't have any data besides the PEM block") + + Expect(block).To(PointTo(MatchAllFields(Fields{ + "Type": Equal("CERTIFICATE"), + "Headers": BeEmpty(), + "Bytes": Equal(pair.Cert.Raw), + }))) + }) + + }) +}) diff --git a/envtest/internal/controlplane/auth.go b/envtest/internal/controlplane/auth.go new file mode 100644 index 0000000..693a85b --- /dev/null +++ b/envtest/internal/controlplane/auth.go @@ -0,0 +1,143 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/rest" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/certs" + "github.com/kcp-dev/multicluster-provider/envtest/internal/process" +) + +// User represents a Kubernetes user. +type User struct { + // Name is the user's Name. + Name string + // Groups are the groups to which the user belongs. + Groups []string +} + +// Authn knows how to configure kcp for a particular type of authentication, +// and provision users under that authentication scheme. +// +// The methods must be called in the following order (as presented below in the interface +// for a mnemonic): +// +// 1. Configure +// 2. Start +// 3. AddUsers (0+ calls) +// 4. Stop. +type Authn interface { + // Configure provides the working directory to this authenticator, + // and configures the given API server arguments to make use of this authenticator. + // + // Should be called first. + Configure(workDir string, args *process.Arguments) error + // Start runs this authenticator. Will be called just before API server start. + // + // Must be called after Configure. + Start() error + // AddUser provisions a user, returning a copy of the given base rest.Config + // configured to authenticate as that users. + // + // May only be called while the authenticator is "running". + AddUser(user User, baseCfg *rest.Config) (*rest.Config, error) + // Stop shuts down this authenticator. + Stop() error +} + +// CertAuthn is an authenticator (Authn) that makes use of client certificate authn. +type CertAuthn struct { + // ca is the CA used to sign the client certs + ca *certs.TinyCA + // certDir is the directory used to write the CA crt file + // so that the API server can read it. + certDir string +} + +// NewCertAuthn creates a new client-cert-based Authn with a new CA. +func NewCertAuthn() (*CertAuthn, error) { + ca, err := certs.NewTinyCA() + if err != nil { + return nil, fmt.Errorf("unable to provision client certificate auth CA: %w", err) + } + return &CertAuthn{ + ca: ca, + }, nil +} + +// AddUser provisions a new user that's authenticated via certificates, with +// the given uesrname and groups embedded in the certificate as expected by the +// API server. +func (c *CertAuthn) AddUser(user User, baseCfg *rest.Config) (*rest.Config, error) { + certs, err := c.ca.NewClientCert(certs.ClientInfo{ + Name: user.Name, + Groups: user.Groups, + }) + if err != nil { + return nil, fmt.Errorf("unable to create client certificates for %s: %w", user.Name, err) + } + + crt, key, err := certs.AsBytes() + if err != nil { + return nil, fmt.Errorf("unable to serialize client certificates for %s: %w", user.Name, err) + } + + cfg := rest.CopyConfig(baseCfg) + cfg.CertData = crt + cfg.KeyData = key + + return cfg, nil +} + +// caCrtPath returns the path to the on-disk client-cert CA crt file. +func (c *CertAuthn) caCrtPath() string { + return filepath.Join(c.certDir, "client-cert-auth-ca.crt") +} + +// Configure provides the working directory to this authenticator, +// and configures the given API server arguments to make use of this authenticator. +func (c *CertAuthn) Configure(workDir string, args *process.Arguments) error { + c.certDir = workDir + args.Set("client-ca-file", c.caCrtPath()) + return nil +} + +// Start runs this authenticator. Will be called just before API server start. +// +// Must be called after Configure. +func (c *CertAuthn) Start() error { + if len(c.certDir) == 0 { + return fmt.Errorf("start called before configure") + } + caCrt := c.ca.CA.CertBytes() + if err := os.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec + return fmt.Errorf("unable to save the client certificate CA to %s: %w", c.caCrtPath(), err) + } + + return nil +} + +// Stop shuts down this authenticator. +func (c *CertAuthn) Stop() error { + // no-op -- our workdir is cleaned up for us automatically + return nil +} diff --git a/envtest/internal/controlplane/auth_test.go b/envtest/internal/controlplane/auth_test.go new file mode 100644 index 0000000..9509406 --- /dev/null +++ b/envtest/internal/controlplane/auth_test.go @@ -0,0 +1,176 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane_test + +import ( + "crypto/tls" + "crypto/x509" + "os" + "path/filepath" + + "k8s.io/client-go/rest" + kcert "k8s.io/client-go/util/cert" + + cp "github.com/kcp-dev/multicluster-provider/envtest/internal/controlplane" + "github.com/kcp-dev/multicluster-provider/envtest/internal/process" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Cert Authentication", func() { + var authn *cp.CertAuthn + BeforeEach(func() { + var err error + authn, err = cp.NewCertAuthn() + Expect(err).NotTo(HaveOccurred(), "should be able to create the cert authn") + }) + Context("when starting", func() { + It("should write the verifying CA to the configured directory", func() { + By("setting up a temp dir") + dir, err := os.MkdirTemp("", "envtest_controlplane_*") + Expect(err).NotTo(HaveOccurred(), "should be able to provision a temp dir") + if dir != "" { + defer os.RemoveAll(dir) + } + + By("configuring to use that dir") + Expect(authn.Configure(dir, process.EmptyArguments())).To(Succeed()) + + By("starting and checking the dir") + Expect(authn.Start()).To(Succeed()) + defer func() { Expect(authn.Stop()).To(Succeed()) }() // not strictly necessary, but future-proof + + _, err = os.Stat(filepath.Join(dir, "client-cert-auth-ca.crt")) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should error out if we haven't been configured yet", func() { + // NB(directxman12): no configure here intentionally + Expect(authn.Start()).NotTo(Succeed()) + }) + }) + Context("when configuring", func() { + It("should have set up the API server to use the written file for client cert auth", func() { + args := process.EmptyArguments() + Expect(authn.Configure("/tmp/____doesnotexist", args)).To(Succeed()) + Expect(args.Get("client-ca-file").Get(nil)).To(ConsistOf("/tmp/____doesnotexist/client-cert-auth-ca.crt")) + }) + }) + + Describe("creating users", func() { + user := cp.User{Name: "someuser", Groups: []string{"group1", "group2"}} + + Context("before starting", func() { + It("should yield a REST config that contains certs valid for the to-be-written CA", func() { + cfg, err := authn.AddUser(user, &rest.Config{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + Expect(cfg.CertData).NotTo(BeEmpty()) + Expect(cfg.KeyData).NotTo(BeEmpty()) + + // double-check the cert (assume the key is fine if it's present + // and the cert is also present, cause it's more annoying to verify + // and we have separate tinyca & integration tests. + By("parsing the config's cert & key data") + certs, err := tls.X509KeyPair(cfg.CertData, cfg.KeyData) + Expect(err).NotTo(HaveOccurred(), "config cert/key data should be valid key pair") + cert, err := x509.ParseCertificate(certs.Certificate[0]) // re-parse cause .Leaf isn't saved + Expect(err).NotTo(HaveOccurred()) + + By("starting and loading the CA cert") + dir, err := os.MkdirTemp("", "envtest_controlplane_*") + Expect(err).NotTo(HaveOccurred(), "should be able to provision a temp dir") + if dir != "" { + defer os.RemoveAll(dir) + } + Expect(authn.Configure(dir, process.EmptyArguments())).To(Succeed()) + Expect(authn.Start()).To(Succeed()) + caCerts, err := kcert.CertsFromFile(filepath.Join(dir, "client-cert-auth-ca.crt")) + Expect(err).NotTo(HaveOccurred(), "should be able to read the CA cert file))))") + Expect(cert.CheckSignatureFrom(caCerts[0])).To(Succeed(), "the config's cert should be signed by the written CA") + }) + + It("should copy the configuration from the base CA without modifying it", func() { + By("creating a user and checking the output config") + base := &rest.Config{Burst: 30} + cfg, err := authn.AddUser(user, base) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + Expect(cfg.Burst).To(Equal(30)) + + By("mutating the base and verifying the cfg doesn't change") + base.Burst = 8675 + Expect(cfg.Burst).To(Equal(30)) + }) + }) + + Context("after starting", func() { + var dir string + BeforeEach(func() { + By("setting up a temp dir & starting with it") + var err error + dir, err = os.MkdirTemp("", "envtest_controlplane_*") + Expect(err).NotTo(HaveOccurred(), "should be able to provision a temp dir") + Expect(authn.Configure(dir, process.EmptyArguments())).To(Succeed()) + Expect(authn.Start()).To(Succeed()) + }) + AfterEach(func() { + if dir != "" { + defer os.RemoveAll(dir) + } + }) + + It("should yield a REST config that contains certs valid for the written CA", func() { + cfg, err := authn.AddUser(user, &rest.Config{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + Expect(cfg.CertData).NotTo(BeEmpty()) + Expect(cfg.KeyData).NotTo(BeEmpty()) + + // double-check the cert (assume the key is fine if it's present + // and the cert is also present, cause it's more annoying to verify + // and we have separate tinyca & integration tests. + By("parsing the config's cert & key data") + certs, err := tls.X509KeyPair(cfg.CertData, cfg.KeyData) + Expect(err).NotTo(HaveOccurred(), "config cert/key data should be valid key pair") + cert, err := x509.ParseCertificate(certs.Certificate[0]) // re-parse cause .Leaf isn't saved + Expect(err).NotTo(HaveOccurred()) + + By("loading the CA cert") + caCerts, err := kcert.CertsFromFile(filepath.Join(dir, "client-cert-auth-ca.crt")) + Expect(err).NotTo(HaveOccurred(), "should be able to read the CA cert file))))") + Expect(cert.CheckSignatureFrom(caCerts[0])).To(Succeed(), "the config's cert should be signed by the written CA") + }) + + It("should copy the configuration from the base CA without modifying it", func() { + By("creating a user and checking the output config") + base := &rest.Config{Burst: 30} + cfg, err := authn.AddUser(user, base) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + Expect(cfg.Burst).To(Equal(30)) + + By("mutating the base and verifying the cfg doesn't change") + base.Burst = 8675 + Expect(cfg.Burst).To(Equal(30)) + }) + }) + }) +}) diff --git a/envtest/internal/controlplane/controlplane_suite_test.go b/envtest/internal/controlplane/controlplane_suite_test.go new file mode 100644 index 0000000..393577f --- /dev/null +++ b/envtest/internal/controlplane/controlplane_suite_test.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestIntegration(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + t.Parallel() + RegisterFailHandler(Fail) + RunSpecs(t, "Kcp Standup Unit Tests") +} diff --git a/envtest/internal/controlplane/kubectl.go b/envtest/internal/controlplane/kubectl.go new file mode 100644 index 0000000..516074d --- /dev/null +++ b/envtest/internal/controlplane/kubectl.go @@ -0,0 +1,120 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + "bytes" + "fmt" + "io" + "net/url" + "os/exec" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + kcapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/process" +) + +const ( + envtestName = "envtest" +) + +// KubeConfigFromREST reverse-engineers a kubeconfig file from a rest.Config. +// The options are tailored towards the rest.Configs we generate, so they're +// not broadly applicable. +// +// This is not intended to be exposed beyond internal for the above reasons. +func KubeConfigFromREST(cfg *rest.Config) ([]byte, error) { + kubeConfig := kcapi.NewConfig() + protocol := "https" + if !rest.IsConfigTransportTLS(*cfg) { + protocol = "http" + } + + // cfg.Host is a URL, so we need to parse it so we can properly append the API path + baseURL, err := url.Parse(cfg.Host) + if err != nil { + return nil, fmt.Errorf("unable to interpret config's host value as a URL: %w", err) + } + + kubeConfig.Clusters[envtestName] = &kcapi.Cluster{ + // TODO(directxman12): if client-go ever decides to expose defaultServerUrlFor(config), + // we can just use that. Note that this is not the same as the public DefaultServerURL, + // which requires us to pass a bunch of stuff in manually. + Server: (&url.URL{Scheme: protocol, Host: baseURL.Host, Path: cfg.APIPath}).String(), + CertificateAuthorityData: cfg.CAData, + } + kubeConfig.AuthInfos[envtestName] = &kcapi.AuthInfo{ + // try to cover all auth strategies that aren't plugins + ClientCertificateData: cfg.CertData, + ClientKeyData: cfg.KeyData, + Token: cfg.BearerToken, + Username: cfg.Username, + Password: cfg.Password, + } + kcCtx := kcapi.NewContext() + kcCtx.Cluster = envtestName + kcCtx.AuthInfo = envtestName + kubeConfig.Contexts[envtestName] = kcCtx + kubeConfig.CurrentContext = envtestName + + contents, err := clientcmd.Write(*kubeConfig) + if err != nil { + return nil, fmt.Errorf("unable to serialize kubeconfig file: %w", err) + } + return contents, nil +} + +// KubeCtl is a wrapper around the kubectl binary. +type KubeCtl struct { + // Path where the kubectl binary can be found. + // + // If this is left empty, we will attempt to locate a binary, by checking for + // the TEST_ASSET_KUBECTL environment variable, and the default test assets + // directory. See the "Binaries" section above (in doc.go) for details. + Path string + + // Opts can be used to configure additional flags which will be used each + // time the wrapped binary is called. + // + // For example, you might want to use this to set the URL of the Shard to + // connect to. + Opts []string +} + +// Run executes the wrapped binary with some preconfigured options and the +// arguments given to this method. It returns Readers for the stdout and +// stderr. +func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) { + if k.Path == "" { + k.Path = process.BinPathFinder("kubectl", "") + } + + stdoutBuffer := &bytes.Buffer{} + stderrBuffer := &bytes.Buffer{} + allArgs := append(k.Opts, args...) + + cmd := exec.Command(k.Path, allArgs...) + cmd.Stdout = stdoutBuffer + cmd.Stderr = stderrBuffer + cmd.SysProcAttr = process.GetSysProcAttr() + + err = cmd.Run() + + return stdoutBuffer, stderrBuffer, err +} diff --git a/envtest/internal/controlplane/kubectl_test.go b/envtest/internal/controlplane/kubectl_test.go new file mode 100644 index 0000000..02421ad --- /dev/null +++ b/envtest/internal/controlplane/kubectl_test.go @@ -0,0 +1,138 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane_test + +import ( + "io" + + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + ccapi "k8s.io/client-go/tools/clientcmd/api" + + . "github.com/kcp-dev/multicluster-provider/envtest/internal/controlplane" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gstruct" +) + +var _ = Describe("Kubectl", func() { + It("runs kubectl", func() { + k := &KubeCtl{Path: "bash"} + args := []string{"-c", "echo 'something'"} + stdout, stderr, err := k.Run(args...) + Expect(err).NotTo(HaveOccurred()) + Expect(stdout).To(ContainSubstring("something")) + bytes, err := io.ReadAll(stderr) + Expect(err).NotTo(HaveOccurred()) + Expect(bytes).To(BeEmpty()) + }) + + Context("when the command returns a non-zero exit code", func() { + It("returns an error", func() { + k := &KubeCtl{Path: "bash"} + args := []string{ + "-c", "echo 'this is StdErr' >&2; echo 'but this is StdOut' >&1; exit 66", + } + + stdout, stderr, err := k.Run(args...) + + Expect(err).To(MatchError(ContainSubstring("exit status 66"))) + + Expect(stdout).To(ContainSubstring("but this is StdOut")) + Expect(stderr).To(ContainSubstring("this is StdErr")) + }) + }) +}) + +var _ = Describe("KubeConfigFromREST", func() { + var ( + restCfg *rest.Config + rawCfg []byte + cfg *ccapi.Config + ) + + BeforeEach(func() { + restCfg = &rest.Config{ + Host: "https://some-host:8675", + APIPath: "/some-prefix", + TLSClientConfig: rest.TLSClientConfig{ + CertData: []byte("cert"), + KeyData: []byte("key"), + CAData: []byte("ca-cert"), + }, + BearerToken: "some-tok", + Username: "some-user", + Password: "some-password", + } + }) + + JustBeforeEach(func() { + var err error + rawCfg, err = KubeConfigFromREST(restCfg) + Expect(err).NotTo(HaveOccurred(), "should be able to convert & serialize the kubeconfig") + + cfg, err = clientcmd.Load(rawCfg) + Expect(err).NotTo(HaveOccurred(), "should be able to deserialize the generated kubeconfig") + }) + + It("should set up a context, and set it as the current one", func() { + By("checking that the current context exists") + Expect(cfg.CurrentContext).NotTo(BeEmpty(), "should have a current context") + Expect(cfg.Contexts).To(HaveKeyWithValue(cfg.CurrentContext, Not(BeNil())), "the current context should exist as a context") + + By("checking that it points to valid info") + currCtx := cfg.Contexts[cfg.CurrentContext] + Expect(currCtx).To(PointTo(MatchFields(IgnoreExtras, Fields{ + "Cluster": Not(BeEmpty()), + "AuthInfo": Not(BeEmpty()), + }))) + + Expect(cfg.Clusters).To(HaveKeyWithValue(currCtx.Cluster, Not(BeNil())), "should point to a cluster") + Expect(cfg.AuthInfos).To(HaveKeyWithValue(currCtx.AuthInfo, Not(BeNil())), "should point to a user") + }) + + Context("when no TLS is enabled", func() { + BeforeEach(func() { + restCfg.Host = "http://some-host:8675" + restCfg.TLSClientConfig = rest.TLSClientConfig{} + }) + + It("should use http in the server url", func() { + cluster := cfg.Clusters[cfg.Contexts[cfg.CurrentContext].Cluster] + Expect(cluster.Server).To(HavePrefix("http://")) + }) + }) + + It("configure the current context to point to the given REST config's server, with CA data", func() { + cluster := cfg.Clusters[cfg.Contexts[cfg.CurrentContext].Cluster] + Expect(cluster).To(PointTo(MatchFields(IgnoreExtras, Fields{ + "Server": Equal("https://some-host:8675/some-prefix"), + "CertificateAuthorityData": Equal([]byte("ca-cert")), + }))) + }) + + It("should copy all non-plugin auth info over", func() { + user := cfg.AuthInfos[cfg.Contexts[cfg.CurrentContext].AuthInfo] + Expect(user).To(PointTo(MatchFields(IgnoreExtras, Fields{ + "ClientCertificateData": Equal([]byte("cert")), + "ClientKeyData": Equal([]byte("key")), + "Token": Equal("some-tok"), + "Username": Equal("some-user"), + "Password": Equal("some-password"), + }))) + }) +}) diff --git a/envtest/internal/controlplane/plane.go b/envtest/internal/controlplane/plane.go new file mode 100644 index 0000000..9247e14 --- /dev/null +++ b/envtest/internal/controlplane/plane.go @@ -0,0 +1,228 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + "fmt" + "os" + + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/client-go/rest" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/certs" +) + +// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY. +// Don't use this for anything else! +var NewTinyCA = certs.NewTinyCA + +// Kcp is a struct that knows how to start your test kcp. +// +// Right now, that means one kcp shard. This is likely to increase in +// future. +type Kcp struct { + RootShard *Shard + + // Kubectl will override the default asset search path for kubectl + KubectlPath string + + // for the deprecated methods (Kubectl, etc) + defaultUserCfg *rest.Config + defaultUserKubectl *KubeCtl +} + +// Start will start your kcp processes. To stop them, call Stop(). +func (f *Kcp) Start() (retErr error) { + if f.RootShard == nil { + f.RootShard = &Shard{} + } + if err := f.RootShard.Start(); err != nil { + return err + } + defer func() { + if retErr != nil { + _ = f.RootShard.Stop() + } + }() + + // provision the default user -- can be removed when the related + // methods are removed. The default user has admin permissions to + // mimic legacy no-authz setups. + user, err := f.AddUser(User{Name: "default", Groups: []string{"system:kcp:admin"}}, &rest.Config{}) + if err != nil { + return fmt.Errorf("unable to provision the default (legacy) user: %w", err) + } + kubectl, err := user.Kubectl() + if err != nil { + return fmt.Errorf("unable to provision the default (legacy) kubeconfig: %w", err) + } + f.defaultUserCfg = user.Config() + f.defaultUserKubectl = kubectl + return nil +} + +// Stop will stop your kcp processes, and clean up their data. +func (f *Kcp) Stop() error { + var errList []error + + if f.RootShard != nil { + if err := f.RootShard.Stop(); err != nil { + errList = append(errList, err) + } + } + + return kerrors.NewAggregate(errList) +} + +// KubeCtl returns a pre-configured KubeCtl, ready to connect to this +// Kcp. +// +// Deprecated: use AddUser & AuthenticatedUser.Kubectl instead. +func (f *Kcp) KubeCtl() *KubeCtl { + return f.defaultUserKubectl +} + +// RESTClientConfig returns a pre-configured restconfig, ready to connect to +// this Kcp. +// +// Deprecated: use AddUser & AuthenticatedUser.Config instead. +func (f *Kcp) RESTClientConfig() (*rest.Config, error) { + return f.defaultUserCfg, nil +} + +// AuthenticatedUser contains access information for an provisioned user, +// including REST config, kubeconfig contents, and access to a KubeCtl instance. +// +// It's not "safe" to use the methods on this till after the API server has been +// started (due to certificate initialization and such). The various methods will +// panic if this is done. +type AuthenticatedUser struct { + // cfg is the rest.Config for connecting to the API server. It's lazily initialized. + cfg *rest.Config + // cfgIsComplete indicates the cfg has had late-initialized fields (e.g. + // API server CA data) initialized. + cfgIsComplete bool + + // apiServer is a handle to the Shard that's used when finalizing cfg + // and producing the kubectl instance. + plane *Kcp + + // kubectl is our existing, provisioned kubectl. We don't provision one + // till someone actually asks for it. + kubectl *KubeCtl +} + +// Config returns the REST config that can be used to connect to the API server +// as this user. +// +// Will panic if used before the API server is started. +func (u *AuthenticatedUser) Config() *rest.Config { + // NB(directxman12): we choose to panic here for ergonomics sake, and because there's + // not really much you can do to "handle" this error. This machinery is intended to be + // used in tests anyway, so panicing is not a particularly big deal. + if u.cfgIsComplete { + return u.cfg + } + if len(u.plane.RootShard.SecureServing.CA) == 0 { + panic("the API server has not yet been started, please do that before accessing connection details") + } + + u.cfg.CAData = u.plane.RootShard.SecureServing.CA + u.cfg.Host = u.plane.RootShard.SecureServing.URL("https", "/").String() + u.cfgIsComplete = true + return u.cfg +} + +// KubeConfig returns a KubeConfig that's roughly equivalent to this user's REST config. +// +// Will panic if used before the API server is started. +func (u AuthenticatedUser) KubeConfig() ([]byte, error) { + // NB(directxman12): we don't return the actual API object to avoid yet another + // piece of kubernetes API in our public API, and also because generally the thing + // you want to do with this is just write it out to a file for external debugging + // purposes, etc. + return KubeConfigFromREST(u.Config()) +} + +// Kubectl returns a KubeCtl instance for talking to the API server as this user. It uses +// a kubeconfig equivalent to that returned by .KubeConfig. +// +// Will panic if used before the API server is started. +func (u *AuthenticatedUser) Kubectl() (*KubeCtl, error) { + if u.kubectl != nil { + return u.kubectl, nil + } + if len(u.plane.RootShard.RootDir) == 0 { + panic("the API server has not yet been started, please do that before accessing connection details") + } + + // cleaning this up is handled when our tmpDir is deleted + out, err := os.CreateTemp(u.plane.RootShard.RootDir, "*.kubecfg") + if err != nil { + return nil, fmt.Errorf("unable to create file for kubeconfig: %w", err) + } + defer out.Close() + contents, err := KubeConfigFromREST(u.Config()) + if err != nil { + return nil, err + } + if _, err := out.Write(contents); err != nil { + return nil, fmt.Errorf("unable to write kubeconfig to disk at %s: %w", out.Name(), err) + } + k := &KubeCtl{ + Path: u.plane.KubectlPath, + } + k.Opts = append(k.Opts, fmt.Sprintf("--kubeconfig=%s", out.Name())) + u.kubectl = k + return k, nil +} + +// AddUser provisions a new user in the cluster. It uses the Shard's authentication +// strategy -- see Shard.SecureServing.Authn. +// +// Unlike AddUser, it's safe to pass a nil rest.Config here if you have no +// particular opinions about the config. +// +// The default authentication strategy is not guaranteed to any specific strategy, but it is +// guaranteed to be callable both before and after Start has been called (but, as noted in the +// AuthenticatedUser docs, the given user objects are only valid after Start has been called). +func (f *Kcp) AddUser(user User, baseConfig *rest.Config) (*AuthenticatedUser, error) { + if f.GetRootShard().SecureServing.Authn == nil { + return nil, fmt.Errorf("no API server authentication is configured yet. The API server defaults one when Start is called, did you mean to use that?") + } + + if baseConfig == nil { + baseConfig = &rest.Config{} + } + cfg, err := f.GetRootShard().SecureServing.AddUser(user, baseConfig) + if err != nil { + return nil, err + } + + return &AuthenticatedUser{ + cfg: cfg, + plane: f, + }, nil +} + +// GetRootShard returns this Kcp's Shard, initializing it if necessary. +func (f *Kcp) GetRootShard() *Shard { + if f.RootShard == nil { + f.RootShard = &Shard{} + } + return f.RootShard +} diff --git a/envtest/internal/controlplane/plane_test.go b/envtest/internal/controlplane/plane_test.go new file mode 100644 index 0000000..13fc9c4 --- /dev/null +++ b/envtest/internal/controlplane/plane_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane_test + +import ( + "context" + + kauthn "k8s.io/api/authorization/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + . "github.com/kcp-dev/multicluster-provider/envtest/internal/controlplane" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Control Plane", func() { + It("should start and stop successfully with a default root shard", func() { + plane := &Kcp{} + Expect(plane.Start()).To(Succeed()) + Expect(plane.Stop()).To(Succeed()) + }) + + It("should use the given shard when starting, if present", func() { + rootShard := &Shard{} + plane := &Kcp{ + RootShard: rootShard, + } + Expect(plane.Start()).To(Succeed()) + defer func() { Expect(plane.Stop()).To(Succeed()) }() + + Expect(plane.RootShard).To(BeIdenticalTo(rootShard)) + }) + + It("should be able to restart", func() { + // NB(directxman12): currently restarting invalidates all current users + // when using CertAuthn. We need to support restarting as per our previous + // contract, but it's not clear how much else we actually need to handle, or + // whether or not this is a safe operation. + plane := &Kcp{} + Expect(plane.Start()).To(Succeed()) + Expect(plane.Stop()).To(Succeed()) + Expect(plane.Start()).To(Succeed()) + Expect(plane.Stop()).To(Succeed()) + }) + + Context("after having started", func() { + var plane *Kcp + BeforeEach(func() { + plane = &Kcp{} + Expect(plane.Start()).To(Succeed()) + }) + AfterEach(func() { + Expect(plane.Stop()).To(Succeed()) + }) + + It("should provision a working legacy user and legacy kubectl", func() { + By("grabbing the legacy kubectl") + Expect(plane.KubeCtl()).NotTo(BeNil()) + + By("grabbing the legacy REST config and testing it") + cfg, err := plane.RESTClientConfig() + Expect(err).NotTo(HaveOccurred(), "should be able to grab the legacy REST config") + cfg.Host += "/clusters/root" + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred(), "should be able to create a client") + + sar := &kauthn.SelfSubjectAccessReview{ + Spec: kauthn.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &kauthn.ResourceAttributes{ + Verb: "*", + Group: "*", + Version: "*", + Resource: "*", + }, + }, + } + Expect(cl.Create(context.Background(), sar)).To(Succeed(), "should be able to make a Self-SAR") + Expect(sar.Status.Allowed).To(BeTrue(), "admin user should be able to do everything") + }) + }) +}) + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) +}) diff --git a/envtest/internal/controlplane/shard.go b/envtest/internal/controlplane/shard.go new file mode 100644 index 0000000..05aca09 --- /dev/null +++ b/envtest/internal/controlplane/shard.go @@ -0,0 +1,351 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/addr" + "github.com/kcp-dev/multicluster-provider/envtest/internal/certs" + "github.com/kcp-dev/multicluster-provider/envtest/internal/process" +) + +var log = ctrllog.Log.WithName("controlplane") + +const ( + // saKeyFile is the name of the service account signing private key file. + saKeyFile = "sa-signer.key" + // saKeyFile is the name of the service account signing public key (cert) file. + saCertFile = "sa-signer.crt" +) + +// SecureServing provides/configures how the API server serves on the secure port. +type SecureServing struct { + // ListenAddr contains the host & port to serve on. + // + // Configurable. If unset, it will be defaulted. + process.ListenAddr + // CA contains the CA that signed the API server's serving certificates. + // + // Read-only. + CA []byte + // Authn can be used to provision users, and override what type of + // authentication is used to provision users. + // + // Configurable. If unset, it will be defaulted. + Authn +} + +// EmbeddedEtcd configures the embedded etcd. +type EmbeddedEtcd struct { + // PeerPort is the etcd peer port. + PeerPort string + // ClientPort is the etcd client port. + ClientPort string +} + +// Shard knows how to run a kcp shard. +type Shard struct { + // SecureServing indicates how the kcp shard will serve on the secure port. + // + // Some parts are configurable. Will be defaulted if unset. + SecureServing + + // EmbeddedEtcd configures the embedded etcd. + EmbeddedEtcd EmbeddedEtcd + + // Path is the path to the kcp binary. + // + // If this is left as the empty string, we will attempt to locate a binary, + // by checking for the TEST_ASSET_KCP environment variable, and + // the default test assets directory. See the "Binaries" section above (in + // doc.go) for details. + Path string + + // RootDir is a path to a directory containing certificates the + // Shard will need, the Shard's embedded etcd data and the admin kubeconfig. + // + // If left unspecified, then the Start() method will create a fresh temporary + // directory, and the Stop() method will clean it up. + RootDir string + + // StartTimeout, StopTimeout specify the time the Shard is allowed to + // take when starting and stoppping before an error is emitted. + // + // If not specified, these default to 1 min and 20 seconds respectively. + StartTimeout time.Duration + StopTimeout time.Duration + + // Out, Err specify where Shard should write its StdOut, StdErr to. + // + // If not specified, the output will be discarded. + Out io.Writer + Err io.Writer + + processState *process.State + + // args contains the structured arguments to use for running the kcp binary. + // Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs() + args *process.Arguments +} + +// Configure returns Arguments that may be used to customize the +// flags used to launch the kcp shard. A set of defaults will +// be applied underneath. +func (s *Shard) Configure() *process.Arguments { + if s.args == nil { + s.args = process.EmptyArguments() + } + return s.args +} + +// Start starts the kcp shard, waits for it to come up, and returns an error, +// if occurred. +func (s *Shard) Start() error { + if err := s.prepare(); err != nil { + return err + } + log.Info("starting kcp server", "command", quoteArgs(append([]string{s.processState.Path}, s.processState.Args...))) + return s.processState.Start(s.Out, s.Err) +} + +func (s *Shard) prepare() error { + if err := s.setProcessState(); err != nil { + return err + } + return s.Authn.Start() +} + +// configurePorts configures the serving ports for this API server. +// +// Most of this method currently deals with making the deprecated fields +// take precedence over the new fields. +func (s *Shard) configurePorts() error { + // prefer the old fields to the new fields if a user set one, + // otherwise, default the new fields and populate the old ones. + + // Secure: SecurePort, SecureServing + if s.SecureServing.Port == "" || s.SecureServing.Address == "" { + port, host, err := addr.Suggest("") + if err != nil { + return fmt.Errorf("unable to provision unused secure port: %w", err) + } + s.SecureServing.Port = strconv.Itoa(port) + s.SecureServing.Address = host + } + + if s.EmbeddedEtcd.PeerPort == "" { + port, _, err := addr.Suggest("") + if err != nil { + return fmt.Errorf("unable to provision unused etcd peer port: %w", err) + } + s.EmbeddedEtcd.PeerPort = strconv.Itoa(port) + } + if s.EmbeddedEtcd.ClientPort == "" { + port, _, err := addr.Suggest("") + if err != nil { + return fmt.Errorf("unable to provision unused etcd client port: %w", err) + } + s.EmbeddedEtcd.ClientPort = strconv.Itoa(port) + } + + return nil +} + +func (s *Shard) setProcessState() error { + // unconditionally re-set this so we can successfully restart + // TODO(directxman12): we supported this in the past, but do we actually + // want to support re-using an API server object to restart? The loss + // of provisioned users is surprising to say the least. + s.processState = &process.State{ + Dir: s.RootDir, + Path: s.Path, + StartTimeout: s.StartTimeout, + StopTimeout: s.StopTimeout, + } + if err := s.processState.Init("kcp"); err != nil { + return err + } + + if err := s.configurePorts(); err != nil { + return err + } + + // the secure port will always be on, so use that + s.processState.HealthCheck.URL = *s.SecureServing.URL("https", "/readyz") + + s.RootDir = s.processState.Dir + s.Path = s.processState.Path + s.StartTimeout = s.processState.StartTimeout + s.StopTimeout = s.processState.StopTimeout + + if err := s.populateServingCerts(); err != nil { + return err + } + + if s.SecureServing.Authn == nil { + authn, err := NewCertAuthn() + if err != nil { + return err + } + s.SecureServing.Authn = authn + } + + if err := s.Authn.Configure(s.RootDir, s.Configure()); err != nil { + return err + } + + // NB(directxman12): insecure port is a mess: + // - 1.19 and below have the `--insecure-port` flag, and require it to be set to zero to + // disable it, otherwise the default will be used and we'll conflict. + // - 1.20 requires the flag to be unset or set to zero, and yells at you if you configure it + // - 1.24 won't have the flag at all... + // + // In an effort to automatically do the right thing during this mess, we do feature discovery + // on the flags, and hope that we've "parsed" them properly. + // + // TODO(directxman12): once we support 1.20 as the min version (might be when 1.24 comes out, + // might be around 1.25 or 1.26), remove this logic and the corresponding line in API server's + // default args. + if err := s.discoverFlags(); err != nil { + return err + } + + s.processState.Args = append([]string{"start"}, s.Configure().AsStrings(s.defaultArgs())...) + + return nil +} + +// discoverFlags checks for certain flags that *must* be set in certain +// versions, and *must not* be set in others. +func (s *Shard) discoverFlags() error { + /* + present, err := s.processState.CheckFlag("insecure-port") + if err != nil { + return err + } + + if !present { + s.Configure().Disable("insecure-port") + } + */ + + return nil +} + +func (s *Shard) defaultArgs() map[string][]string { + args := map[string][]string{ + "root-directory": {s.RootDir}, + "secure-port": {s.SecureServing.Port}, + "bind-address": {s.SecureServing.Address}, + "embedded-etcd-peer-port": {s.EmbeddedEtcd.PeerPort}, + "embedded-etcd-client-port": {s.EmbeddedEtcd.ClientPort}, + } + return args +} + +func (s *Shard) populateServingCerts() error { + _, statErr := os.Stat(filepath.Join(s.RootDir, "apiserver.crt")) + if !os.IsNotExist(statErr) { + return statErr + } + + ca, err := certs.NewTinyCA() + if err != nil { + return err + } + + servingCerts, err := ca.NewServingCert() + if err != nil { + return err + } + + certData, keyData, err := servingCerts.AsBytes() + if err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(s.RootDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec + return err + } + if err := os.WriteFile(filepath.Join(s.RootDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec + return err + } + + s.SecureServing.CA = ca.CA.CertBytes() + + // service account signing files too + saCA, err := certs.NewTinyCA() + if err != nil { + return err + } + + saCert, saKey, err := saCA.CA.AsBytes() + if err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(s.RootDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec + return err + } + return os.WriteFile(filepath.Join(s.RootDir, saKeyFile), saKey, 0640) //nolint:gosec +} + +// Stop stops this process gracefully, waits for its termination, and cleans up +// the RootDir if necessary. +func (s *Shard) Stop() error { + if s.processState != nil { + if s.processState.DirNeedsCleaning { + s.RootDir = "" // reset the directory if it was randomly allocated, so that we can safely restart + } + if err := s.processState.Stop(); err != nil { + return err + } + } + return s.Authn.Stop() +} + +// PrepareShard is an internal-only (NEVER SHOULD BE EXPOSED) +// function that sets up the kcp shard just before starting it, +// without actually starting it. This saves time on tests. +// +// NB(directxman12): do not expose this outside of internal -- it's unsafe to +// use, because things like port allocation could race even more than they +// currently do if you later call start! +func PrepareShard(s *Shard) error { + return s.prepare() +} + +func quoteArgs(args []string) string { + quoted := make([]string, len(args)) + for i, arg := range args { + if strings.Contains(arg, " ") { + quoted[i] = strconv.Quote(arg) + } else { + quoted[i] = arg + } + } + return strings.Join(quoted, " ") +} diff --git a/envtest/internal/controlplane/shard_test.go b/envtest/internal/controlplane/shard_test.go new file mode 100644 index 0000000..e8c71a6 --- /dev/null +++ b/envtest/internal/controlplane/shard_test.go @@ -0,0 +1,155 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controlplane_test + +import ( + "errors" + + "k8s.io/client-go/rest" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/process" + + . "github.com/kcp-dev/multicluster-provider/envtest/internal/controlplane" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Shard", func() { + var server *Shard + BeforeEach(func() { + server = &Shard{} + }) + JustBeforeEach(func() { + Expect(PrepareShard(server)).To(Succeed()) + }) + Describe("setting up serving hosts & ports", func() { + Context("when SecureServing host & port are set", func() { + BeforeEach(func() { + server.Address = "localhost" + server.Port = "8675" + }) + + It("should leave SecureServing as-is", func() { + Expect(server.SecureServing.Address).To(Equal("localhost")) + Expect(server.SecureServing.Port).To(Equal("8675")) + }) + }) + + Context("when SecureServing is not set", func() { + It("should be defaulted with a random port", func() { + Expect(server.Port).NotTo(BeEquivalentTo(0)) + }) + }) + }) + + It("should default authn if not set", func() { + Expect(server.Authn).NotTo(BeNil()) + }) + + Describe("setting up auth", func() { + var auth *fakeAuthn + BeforeEach(func() { + auth = &fakeAuthn{ + setFlag: true, + } + server.Authn = auth + }) + It("should configure with the root dir", func() { + Expect(auth.workDir).To(Equal(server.RootDir)) + }) + It("should pass its args to be configured", func() { + Expect(server.Configure().Get("configure-called").Get(nil)).To(ConsistOf("true")) + }) + + Context("when configuring auth errors out", func() { + It("should fail to configure", func() { + server := &Shard{ + SecureServing: SecureServing{ + Authn: auth, + }, + } + auth.configureErr = errors.New("Oh no") + Expect(PrepareShard(server)).NotTo(Succeed()) + }) + }) + }) + + Describe("managing", func() { + // some of these tests are combined for speed reasons -- starting the apiserver + // takes a while, relatively speaking + + var ( + auth *fakeAuthn + ) + BeforeEach(func() { + auth = &fakeAuthn{} + server.Authn = auth + }) + + Context("after starting", func() { + BeforeEach(func() { + Expect(server.Start()).To(Succeed()) + }) + + It("should stop successfully, and stop auth", func() { + Expect(server.Stop()).To(Succeed()) + Expect(auth.stopCalled).To(BeTrue()) + }) + }) + + It("should fail to start when auth fails to start", func() { + auth.startErr = errors.New("Oh no") + Expect(server.Start()).NotTo(Succeed()) + }) + + It("should start successfully & start auth", func() { + Expect(server.Start()).To(Succeed()) + defer func() { Expect(server.Stop()).To(Succeed()) }() + Expect(auth.startCalled).To(BeTrue()) + }) + }) +}) + +type fakeAuthn struct { + workDir string + + startCalled bool + stopCalled bool + setFlag bool + + configureErr error + startErr error +} + +func (f *fakeAuthn) Configure(workDir string, args *process.Arguments) error { + f.workDir = workDir + if f.setFlag { + args.Set("configure-called", "true") + } + return f.configureErr +} +func (f *fakeAuthn) Start() error { + f.startCalled = true + return f.startErr +} +func (f *fakeAuthn) AddUser(user User, baseCfg *rest.Config) (*rest.Config, error) { + return nil, nil +} +func (f *fakeAuthn) Stop() error { + f.stopCalled = true + return nil +} diff --git a/envtest/internal/controlplane/testdata/fake-1.19-apiserver.sh b/envtest/internal/controlplane/testdata/fake-1.19-apiserver.sh new file mode 100755 index 0000000..8b71661 --- /dev/null +++ b/envtest/internal/controlplane/testdata/fake-1.19-apiserver.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env sh + +cat </=true|false for a specific API group and version (e.g. apps/v1=true) + api/all=true|false controls all API versions + api/ga=true|false controls all API versions of the form v[0-9]+ + api/beta=true|false controls all API versions of the form v[0-9]+beta[0-9]+ + api/alpha=true|false controls all API versions of the form v[0-9]+alpha[0-9]+ + api/legacy is deprecated, and will be removed in a future version + +Egress selector flags: + + --egress-selector-config-file string File with apiserver egress selector configuration. + +Admission flags: + + --admission-control strings Admission is divided into two phases. In the first phase, only mutating admission plugins run. In the second phase, only validating admission plugins run. The names in the below list may represent a validating plugin, a mutating plugin, or both. The order of plugins in which they are passed to this flag does not matter. Comma-delimited list of: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodPreset, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. (DEPRECATED: Use --enable-admission-plugins or --disable-admission-plugins instead. Will be removed in a future version.) + --admission-control-config-file string File with admission control configuration. + --disable-admission-plugins strings admission plugins that should be disabled although they are in the default enabled plugins list (NamespaceLifecycle, LimitRanger, ServiceAccount, TaintNodesByCondition, Priority, DefaultTolerationSeconds, DefaultStorageClass, StorageObjectInUseProtection, PersistentVolumeClaimResize, RuntimeClass, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, MutatingAdmissionWebhook, ValidatingAdmissionWebhook, ResourceQuota). Comma-delimited list of admission plugins: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodPreset, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. The order of plugins in this flag does not matter. + --enable-admission-plugins strings admission plugins that should be enabled in addition to default enabled ones (NamespaceLifecycle, LimitRanger, ServiceAccount, TaintNodesByCondition, Priority, DefaultTolerationSeconds, DefaultStorageClass, StorageObjectInUseProtection, PersistentVolumeClaimResize, RuntimeClass, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, MutatingAdmissionWebhook, ValidatingAdmissionWebhook, ResourceQuota). Comma-delimited list of admission plugins: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodPreset, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. The order of plugins in this flag does not matter. + +Metrics flags: + + --show-hidden-metrics-for-version string The previous version for which you want to show hidden metrics. Only the previous minor version is meaningful, other values will not be allowed. The format is ., e.g.: '1.16'. The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics, rather than being surprised when they are permanently removed in the release after that. + +Logs flags: + + --logging-format string Sets the log format. Permitted formats: "json", "text". + Non-default formats don't honor these flags: --add_dir_header, --alsologtostderr, --log_backtrace_at, --log_dir, --log_file, --log_file_max_size, --logtostderr, --skip_headers, --skip_log_headers, --stderrthreshold, --vmodule, --log-flush-frequency. + Non-default choices are currently alpha and subject to change without warning. (default "text") + +Misc flags: + + --allow-privileged If true, allow privileged containers. [default=false] + --apiserver-count int The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.) (default 1) + --enable-aggregator-routing Turns on aggregator routing requests to endpoints IP rather than cluster IP. + --endpoint-reconciler-type string Use an endpoint reconciler (master-count, lease, none) (default "lease") + --event-ttl duration Amount of time to retain events. (default 1h0m0s) + --kubelet-certificate-authority string Path to a cert file for the certificate authority. + --kubelet-client-certificate string Path to a client cert file for TLS. + --kubelet-client-key string Path to a client key file for TLS. + --kubelet-preferred-address-types strings List of the preferred NodeAddressTypes to use for kubelet connections. (default [Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP]) + --kubelet-timeout duration Timeout for kubelet operations. (default 5s) + --kubernetes-service-node-port int If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be of type NodePort, using this as the value of the port. If zero, the Kubernetes master service will be of type ClusterIP. + --max-connection-bytes-per-sec int If non-zero, throttle each user connection to this number of bytes/sec. Currently only applies to long-running requests. + --proxy-client-cert-file string Client certificate used to prove the identity of the aggregator or kube-apiserver when it must call out during a request. This includes proxying requests to a user api-server and calling out to webhook admission plugins. It is expected that this cert includes a signature from the CA in the --requestheader-client-ca-file flag. That CA is published in the 'extension-apiserver-authentication' configmap in the kube-system namespace. Components receiving calls from kube-aggregator should use that CA to perform their half of the mutual TLS verification. + --proxy-client-key-file string Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver when it must call out during a request. This includes proxying requests to a user api-server and calling out to webhook admission plugins. + --service-account-signing-key-file string Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key. (Requires the 'TokenRequest' feature gate.) + --service-cluster-ip-range string A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes or pods. + --service-node-port-range portRange A port range to reserve for services with NodePort visibility. Example: '30000-32767'. Inclusive at both ends of the range. (default 30000-32767) + +Global flags: + + --add-dir-header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -h, --help help for kube-apiserver + --log-backtrace-at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log-dir string If non-empty, write log files in this directory + --log-file string If non-empty, use this log file + --log-file-max-size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --logtostderr log to standard error instead of files (default true) + --skip-headers If true, avoid header prefixes in the log messages + --skip-log-headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level number for the log level verbosity + --version version[=true] Print version information and quit + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + +EOF diff --git a/envtest/internal/controlplane/testdata/fake-1.20-apiserver.sh b/envtest/internal/controlplane/testdata/fake-1.20-apiserver.sh new file mode 100755 index 0000000..112346c --- /dev/null +++ b/envtest/internal/controlplane/testdata/fake-1.20-apiserver.sh @@ -0,0 +1,318 @@ +#!/usr/bin/env sh + +cat </=true|false for a specific API group and version (e.g. apps/v1=true) + api/all=true|false controls all API versions + api/ga=true|false controls all API versions of the form v[0-9]+ + api/beta=true|false controls all API versions of the form v[0-9]+beta[0-9]+ + api/alpha=true|false controls all API versions of the form v[0-9]+alpha[0-9]+ + api/legacy is deprecated, and will be removed in a future version + +Egress selector flags: + + --egress-selector-config-file string File with apiserver egress selector configuration. + +Admission flags: + + --admission-control strings Admission is divided into two phases. In the first phase, only mutating admission plugins run. In the second phase, only validating admission plugins run. The names in the below list may represent a validating plugin, a mutating plugin, or both. The order of plugins in which they are passed to this flag does not matter. Comma-delimited list of: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. (DEPRECATED: Use --enable-admission-plugins or --disable-admission-plugins instead. Will be removed in a future version.) + --admission-control-config-file string File with admission control configuration. + --disable-admission-plugins strings admission plugins that should be disabled although they are in the default enabled plugins list (NamespaceLifecycle, LimitRanger, ServiceAccount, TaintNodesByCondition, Priority, DefaultTolerationSeconds, DefaultStorageClass, StorageObjectInUseProtection, PersistentVolumeClaimResize, RuntimeClass, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, MutatingAdmissionWebhook, ValidatingAdmissionWebhook, ResourceQuota). Comma-delimited list of admission plugins: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. The order of plugins in this flag does not matter. + --enable-admission-plugins strings admission plugins that should be enabled in addition to default enabled ones (NamespaceLifecycle, LimitRanger, ServiceAccount, TaintNodesByCondition, Priority, DefaultTolerationSeconds, DefaultStorageClass, StorageObjectInUseProtection, PersistentVolumeClaimResize, RuntimeClass, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, MutatingAdmissionWebhook, ValidatingAdmissionWebhook, ResourceQuota). Comma-delimited list of admission plugins: AlwaysAdmit, AlwaysDeny, AlwaysPullImages, CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, DenyEscalatingExec, DenyExecOnPrivileged, EventRateLimit, ExtendedResourceToleration, ImagePolicyWebhook, LimitPodHardAntiAffinityTopology, LimitRanger, MutatingAdmissionWebhook, NamespaceAutoProvision, NamespaceExists, NamespaceLifecycle, NodeRestriction, OwnerReferencesPermissionEnforcement, PersistentVolumeClaimResize, PersistentVolumeLabel, PodNodeSelector, PodSecurityPolicy, PodTolerationRestriction, Priority, ResourceQuota, RuntimeClass, SecurityContextDeny, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. The order of plugins in this flag does not matter. + +Metrics flags: + + --show-hidden-metrics-for-version string The previous version for which you want to show hidden metrics. Only the previous minor version is meaningful, other values will not be allowed. The format is ., e.g.: '1.16'. The purpose of this format is make sure you have the opportunity to notice if the next release hides additional metrics, rather than being surprised when they are permanently removed in the release after that. + +Logs flags: + + --experimental-logging-sanitization [Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens). + Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production. + --logging-format string Sets the log format. Permitted formats: "json", "text". + Non-default formats don't honor these flags: --add_dir_header, --alsologtostderr, --log_backtrace_at, --log_dir, --log_file, --log_file_max_size, --logtostderr, --one_output, --skip_headers, --skip_log_headers, --stderrthreshold, --vmodule, --log-flush-frequency. + Non-default choices are currently alpha and subject to change without warning. (default "text") + +Misc flags: + + --allow-privileged If true, allow privileged containers. [default=false] + --apiserver-count int The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.) (default 1) + --enable-aggregator-routing Turns on aggregator routing requests to endpoints IP rather than cluster IP. + --endpoint-reconciler-type string Use an endpoint reconciler (master-count, lease, none) (default "lease") + --event-ttl duration Amount of time to retain events. (default 1h0m0s) + --identity-lease-duration-seconds int The duration of kube-apiserver lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.) (default 3600) + --identity-lease-renew-interval-seconds int The interval of kube-apiserver renewing its lease in seconds, must be a positive number. (In use when the APIServerIdentity feature gate is enabled.) (default 10) + --kubelet-certificate-authority string Path to a cert file for the certificate authority. + --kubelet-client-certificate string Path to a client cert file for TLS. + --kubelet-client-key string Path to a client key file for TLS. + --kubelet-preferred-address-types strings List of the preferred NodeAddressTypes to use for kubelet connections. (default [Hostname,InternalDNS,InternalIP,ExternalDNS,ExternalIP]) + --kubelet-timeout duration Timeout for kubelet operations. (default 5s) + --kubernetes-service-node-port int If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be of type NodePort, using this as the value of the port. If zero, the Kubernetes master service will be of type ClusterIP. + --max-connection-bytes-per-sec int If non-zero, throttle each user connection to this number of bytes/sec. Currently only applies to long-running requests. + --proxy-client-cert-file string Client certificate used to prove the identity of the aggregator or kube-apiserver when it must call out during a request. This includes proxying requests to a user api-server and calling out to webhook admission plugins. It is expected that this cert includes a signature from the CA in the --requestheader-client-ca-file flag. That CA is published in the 'extension-apiserver-authentication' configmap in the kube-system namespace. Components receiving calls from kube-aggregator should use that CA to perform their half of the mutual TLS verification. + --proxy-client-key-file string Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver when it must call out during a request. This includes proxying requests to a user api-server and calling out to webhook admission plugins. + --service-account-signing-key-file string Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key. + --service-cluster-ip-range string A CIDR notation IP range from which to assign service cluster IPs. This must not overlap with any IP ranges assigned to nodes or pods. + --service-node-port-range portRange A port range to reserve for services with NodePort visibility. Example: '30000-32767'. Inclusive at both ends of the range. (default 30000-32767) + +Global flags: + + --add-dir-header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -h, --help help for kube-apiserver + --log-backtrace-at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log-dir string If non-empty, write log files in this directory + --log-file string If non-empty, use this log file + --log-file-max-size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --log-flush-frequency duration Maximum number of seconds between log flushes (default 5s) + --logtostderr log to standard error instead of files (default true) + --one-output If true, only write logs to their native severity level (vs also writing to each lower severity level + --skip-headers If true, avoid header prefixes in the log messages + --skip-log-headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + -v, --v Level number for the log level verbosity + --version version[=true] Print version information and quit + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +EOF diff --git a/envtest/internal/flock/doc.go b/envtest/internal/flock/doc.go new file mode 100644 index 0000000..11e3982 --- /dev/null +++ b/envtest/internal/flock/doc.go @@ -0,0 +1,21 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package flock is copied from k8s.io/kubernetes/pkg/util/flock to avoid +// importing k8s.io/kubernetes as a dependency. +// +// Provides file locking functionalities on unix systems. +package flock diff --git a/envtest/internal/flock/errors.go b/envtest/internal/flock/errors.go new file mode 100644 index 0000000..ee7a434 --- /dev/null +++ b/envtest/internal/flock/errors.go @@ -0,0 +1,24 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flock + +import "errors" + +var ( + // ErrAlreadyLocked is returned when the file is already locked. + ErrAlreadyLocked = errors.New("the file is already locked") +) diff --git a/envtest/internal/flock/flock_other.go b/envtest/internal/flock/flock_other.go new file mode 100644 index 0000000..069a5b3 --- /dev/null +++ b/envtest/internal/flock/flock_other.go @@ -0,0 +1,24 @@ +// +build !linux,!darwin,!freebsd,!openbsd,!netbsd,!dragonfly + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flock + +// Acquire is not implemented on non-unix systems. +func Acquire(path string) error { + return nil +} diff --git a/envtest/internal/flock/flock_unix.go b/envtest/internal/flock/flock_unix.go new file mode 100644 index 0000000..71ec576 --- /dev/null +++ b/envtest/internal/flock/flock_unix.go @@ -0,0 +1,48 @@ +//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly +// +build linux darwin freebsd openbsd netbsd dragonfly + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flock + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +// Acquire acquires a lock on a file for the duration of the process. This method +// is reentrant. +func Acquire(path string) error { + fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC, 0600) + if err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("cannot lock file %q: %w", path, ErrAlreadyLocked) + } + return err + } + + // We don't need to close the fd since we should hold + // it until the process exits. + err = unix.Flock(fd, unix.LOCK_NB|unix.LOCK_EX) + if errors.Is(err, unix.EWOULDBLOCK) { // This condition requires LOCK_NB. + return fmt.Errorf("cannot lock file %q: %w", path, ErrAlreadyLocked) + } + return err +} diff --git a/envtest/internal/process/arguments.go b/envtest/internal/process/arguments.go new file mode 100644 index 0000000..59f5b2d --- /dev/null +++ b/envtest/internal/process/arguments.go @@ -0,0 +1,234 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process + +import ( + "sort" +) + +// EmptyArguments constructs an empty set of flags with no defaults. +func EmptyArguments() *Arguments { + return &Arguments{ + values: make(map[string]Arg), + } +} + +// Arguments are structured, overridable arguments. +// Each Arguments object contains some set of default arguments, which may +// be appended to, or overridden. +// +// When ready, you can serialize them to pass to exec.Command and friends using +// AsStrings. +// +// All flag-setting methods return the *same* instance of Arguments so that you +// can chain calls. +type Arguments struct { + // values contains the user-set values for the arguments. + // `values[key] = dontPass` means "don't pass this flag" + // `values[key] = passAsName` means "pass this flag without args like --key` + // `values[key] = []string{a, b, c}` means "--key=a --key=b --key=c` + // any values not explicitly set here will be copied from defaults on final rendering. + values map[string]Arg +} + +// Arg is an argument that has one or more values, +// and optionally falls back to default values. +type Arg interface { + // Append adds new values to this argument, returning + // a new instance contain the new value. The intermediate + // argument should generally be assumed to be consumed. + Append(vals ...string) Arg + // Get returns the full set of values, optionally including + // the passed in defaults. If it returns nil, this will be + // skipped. If it returns a non-nil empty slice, it'll be + // assumed that the argument should be passed as name-only. + Get(defaults []string) []string +} + +type userArg []string + +func (a userArg) Append(vals ...string) Arg { + return userArg(append(a, vals...)) //nolint:unconvert +} +func (a userArg) Get(_ []string) []string { + return []string(a) +} + +type defaultedArg []string + +func (a defaultedArg) Append(vals ...string) Arg { + return defaultedArg(append(a, vals...)) //nolint:unconvert +} +func (a defaultedArg) Get(defaults []string) []string { + res := append([]string(nil), defaults...) + return append(res, a...) +} + +type dontPassArg struct{} + +func (a dontPassArg) Append(vals ...string) Arg { + return userArg(vals) +} +func (dontPassArg) Get(_ []string) []string { + return nil +} + +type passAsNameArg struct{} + +func (a passAsNameArg) Append(_ ...string) Arg { + return passAsNameArg{} +} +func (passAsNameArg) Get(_ []string) []string { + return []string{} +} + +var ( + // DontPass indicates that the given argument will not actually be + // rendered. + DontPass Arg = dontPassArg{} + // PassAsName indicates that the given flag will be passed as `--key` + // without any value. + PassAsName Arg = passAsNameArg{} +) + +// AsStrings serializes this set of arguments to a slice of strings appropriate +// for passing to exec.Command and friends, making use of the given defaults +// as indicated for each particular argument. +// +// - Any flag in defaults that's not in Arguments will be present in the output +// - Any flag that's present in Arguments will be passed the corresponding +// defaults to do with as it will (ignore, append-to, suppress, etc). +func (a *Arguments) AsStrings(defaults map[string][]string) []string { + // sort for deterministic ordering + keysInOrder := make([]string, 0, len(defaults)+len(a.values)) + for key := range defaults { + if _, userSet := a.values[key]; userSet { + continue + } + keysInOrder = append(keysInOrder, key) + } + for key := range a.values { + keysInOrder = append(keysInOrder, key) + } + sort.Strings(keysInOrder) + + var res []string + for _, key := range keysInOrder { + vals := a.Get(key).Get(defaults[key]) + switch { + case vals == nil: // don't pass + continue + case len(vals) == 0: // pass as name + res = append(res, "--"+key) + default: + for _, val := range vals { + res = append(res, "--"+key+"="+val) + } + } + } + + return res +} + +// Get returns the value of the given flag. If nil, +// it will not be passed in AsString, otherwise: +// +// len == 0 --> `--key`, len > 0 --> `--key=val1 --key=val2 ...`. +func (a *Arguments) Get(key string) Arg { + if vals, ok := a.values[key]; ok { + return vals + } + return defaultedArg(nil) +} + +// Enable configures the given key to be passed as a "name-only" flag, +// like, `--key`. +func (a *Arguments) Enable(key string) *Arguments { + a.values[key] = PassAsName + return a +} + +// Disable prevents this flag from be passed. +func (a *Arguments) Disable(key string) *Arguments { + a.values[key] = DontPass + return a +} + +// Append adds additional values to this flag. If this flag has +// yet to be set, initial values will include defaults. If you want +// to intentionally ignore defaults/start from scratch, call AppendNoDefaults. +// +// Multiple values will look like `--key=value1 --key=value2 ...`. +func (a *Arguments) Append(key string, values ...string) *Arguments { + vals, present := a.values[key] + if !present { + vals = defaultedArg{} + } + a.values[key] = vals.Append(values...) + return a +} + +// AppendNoDefaults adds additional values to this flag. However, +// unlike Append, it will *not* copy values from defaults. +func (a *Arguments) AppendNoDefaults(key string, values ...string) *Arguments { + vals, present := a.values[key] + if !present { + vals = userArg{} + } + a.values[key] = vals.Append(values...) + return a +} + +// Set resets the given flag to the specified values, ignoring any existing +// values or defaults. +func (a *Arguments) Set(key string, values ...string) *Arguments { + a.values[key] = userArg(values) + return a +} + +// SetRaw sets the given flag to the given Arg value directly. Use this if +// you need to do some complicated deferred logic or something. +// +// Otherwise behaves like Set. +func (a *Arguments) SetRaw(key string, val Arg) *Arguments { + a.values[key] = val + return a +} + +// FuncArg is a basic implementation of Arg that can be used for custom argument logic, +// like pulling values out of APIServer, or dynamically calculating values just before +// launch. +// +// The given function will be mapped directly to Arg#Get, and will generally be +// used in conjunction with SetRaw. For example, to set `--some-flag` to the +// API server's CertDir, you could do: +// +// server.Configure().SetRaw("--some-flag", FuncArg(func(defaults []string) []string { +// return []string{server.CertDir} +// })) +// +// FuncArg ignores Appends; if you need to support appending values too, consider implementing +// Arg directly. +type FuncArg func([]string) []string + +// Append is a no-op for FuncArg, and just returns itself. +func (a FuncArg) Append(vals ...string) Arg { return a } + +// Get delegates functionality to the FuncArg function itself. +func (a FuncArg) Get(defaults []string) []string { + return a(defaults) +} diff --git a/envtest/internal/process/arguments_test.go b/envtest/internal/process/arguments_test.go new file mode 100644 index 0000000..3ecc083 --- /dev/null +++ b/envtest/internal/process/arguments_test.go @@ -0,0 +1,145 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process_test + +import ( + "strings" + + . "github.com/kcp-dev/multicluster-provider/envtest/internal/process" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Arguments", func() { + Context("when appending", func() { + It("should copy from defaults when appending for the first time", func() { + args := EmptyArguments(). + Append("some-key", "val3") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"val1", "val2", "val3"})) + }) + + It("should not copy from defaults if the flag has been disabled previously", func() { + args := EmptyArguments(). + Disable("some-key"). + Append("some-key", "val3") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"val3"})) + }) + + It("should only copy defaults the first time", func() { + args := EmptyArguments(). + Append("some-key", "val3", "val4"). + Append("some-key", "val5") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"val1", "val2", "val3", "val4", "val5"})) + }) + + It("should not copy from defaults if the flag has been previously overridden", func() { + args := EmptyArguments(). + Set("some-key", "vala"). + Append("some-key", "valb", "valc") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"vala", "valb", "valc"})) + }) + + Context("when explicitly overriding defaults", func() { + It("should not copy from defaults, but should append to previous calls", func() { + args := EmptyArguments(). + AppendNoDefaults("some-key", "vala"). + AppendNoDefaults("some-key", "valb", "valc") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"vala", "valb", "valc"})) + }) + + It("should not copy from defaults, but should respect previous appends' copies", func() { + args := EmptyArguments(). + Append("some-key", "vala"). + AppendNoDefaults("some-key", "valb", "valc") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"val1", "val2", "vala", "valb", "valc"})) + }) + + It("should not copy from defaults if the flag has been previously appended to ignoring defaults", func() { + args := EmptyArguments(). + AppendNoDefaults("some-key", "vala"). + Append("some-key", "valb", "valc") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"vala", "valb", "valc"})) + }) + }) + }) + + It("should ignore defaults when overriding", func() { + args := EmptyArguments(). + Set("some-key", "vala") + Expect(args.Get("some-key").Get([]string{"val1", "val2"})).To(Equal([]string{"vala"})) + }) + + It("should allow directly setting the argument value for custom argument types", func() { + args := EmptyArguments(). + SetRaw("custom-key", commaArg{"val3"}). + Append("custom-key", "val4") + Expect(args.Get("custom-key").Get([]string{"val1", "val2"})).To(Equal([]string{"val1,val2,val3,val4"})) + }) + + Context("when rendering flags", func() { + It("should not render defaults for disabled flags", func() { + defs := map[string][]string{ + "some-key": {"val1", "val2"}, + "other-key": {"val"}, + } + args := EmptyArguments(). + Disable("some-key") + Expect(args.AsStrings(defs)).To(ConsistOf("--other-key=val")) + }) + + It("should render name-only flags as --key", func() { + args := EmptyArguments(). + Enable("some-key") + Expect(args.AsStrings(nil)).To(ConsistOf("--some-key")) + }) + + It("should render multiple values as --key=val1, --key=val2", func() { + args := EmptyArguments(). + Append("some-key", "val1", "val2"). + Append("other-key", "vala", "valb") + Expect(args.AsStrings(nil)).To(ConsistOf("--other-key=valb", "--other-key=vala", "--some-key=val1", "--some-key=val2")) + }) + + It("should read from defaults if the user hasn't set a value for a flag", func() { + defs := map[string][]string{ + "some-key": {"val1", "val2"}, + } + args := EmptyArguments(). + Append("other-key", "vala", "valb") + Expect(args.AsStrings(defs)).To(ConsistOf("--other-key=valb", "--other-key=vala", "--some-key=val1", "--some-key=val2")) + }) + + It("should not render defaults if the user has set a value for a flag", func() { + defs := map[string][]string{ + "some-key": {"val1", "val2"}, + } + args := EmptyArguments(). + Set("some-key", "vala") + Expect(args.AsStrings(defs)).To(ConsistOf("--some-key=vala")) + }) + }) +}) + +type commaArg []string + +func (a commaArg) Get(defs []string) []string { + // not quite, but close enough + return []string{strings.Join(defs, ",") + "," + strings.Join(a, ",")} +} +func (a commaArg) Append(vals ...string) Arg { + return commaArg(append(a, vals...)) //nolint:unconvert +} diff --git a/envtest/internal/process/bin_path_finder.go b/envtest/internal/process/bin_path_finder.go new file mode 100644 index 0000000..18be83b --- /dev/null +++ b/envtest/internal/process/bin_path_finder.go @@ -0,0 +1,70 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process + +import ( + "os" + "path/filepath" + "regexp" + "strings" +) + +const ( + // EnvAssetsPath is the environment variable that stores the global test + // binary location override. + EnvAssetsPath = "TEST_KCP_ASSETS" + // EnvAssetOverridePrefix is the environment variable prefix for per-binary + // location overrides. + EnvAssetOverridePrefix = "TEST_ASSET_" + // AssetsDefaultPath is the default location to look for test binaries in, + // if no override was provided. + AssetsDefaultPath = "/usr/local/kcp/bin" +) + +// BinPathFinder finds the path to the given named binary, using the following locations +// in order of precedence (highest first). Notice that the various env vars only need +// to be set -- the asset is not checked for existence on the filesystem. +// +// 1. TEST_ASSET_{tr/a-z-/A-Z_/} (if set; asset overrides -- EnvAssetOverridePrefix) +// 1. TEST_KCP_ASSETS (if set; global asset path -- EnvAssetsPath) +// 3. assetDirectory (if set; per-config asset directory) +// 4. /usr/local/kcp/bin (AssetsDefaultPath). +func BinPathFinder(symbolicName, assetDirectory string) (binPath string) { + punctuationPattern := regexp.MustCompile("[^A-Z0-9]+") + sanitizedName := punctuationPattern.ReplaceAllString(strings.ToUpper(symbolicName), "_") + leadingNumberPattern := regexp.MustCompile("^[0-9]+") + sanitizedName = leadingNumberPattern.ReplaceAllString(sanitizedName, "") + envVar := EnvAssetOverridePrefix + sanitizedName + + // TEST_ASSET_XYZ + if val, ok := os.LookupEnv(envVar); ok { + return val + } + + // TEST_KCP_ASSETS + if val, ok := os.LookupEnv(EnvAssetsPath); ok { + return filepath.Join(val, symbolicName) + } + + // assetDirectory + if assetDirectory != "" { + return filepath.Join(assetDirectory, symbolicName) + } + + // default path + return filepath.Join(AssetsDefaultPath, symbolicName) +} diff --git a/envtest/internal/process/bin_path_finder_test.go b/envtest/internal/process/bin_path_finder_test.go new file mode 100644 index 0000000..21ec821 --- /dev/null +++ b/envtest/internal/process/bin_path_finder_test.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process + +import ( + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("BinPathFinder", func() { + var prevAssetPath string + BeforeEach(func() { + prevAssetPath = os.Getenv(EnvAssetsPath) + Expect(os.Unsetenv(EnvAssetsPath)).To(Succeed()) + Expect(os.Unsetenv(EnvAssetOverridePrefix + "_SOME_FAKE")).To(Succeed()) + Expect(os.Unsetenv(EnvAssetOverridePrefix + "OTHERFAKE")).To(Succeed()) + }) + AfterEach(func() { + if prevAssetPath != "" { + Expect(os.Setenv(EnvAssetsPath, prevAssetPath)).To(Succeed()) + } + }) + Context("when individual overrides are present", func() { + BeforeEach(func() { + Expect(os.Setenv(EnvAssetOverridePrefix+"OTHERFAKE", "/other/path")).To(Succeed()) + Expect(os.Setenv(EnvAssetOverridePrefix+"_SOME_FAKE", "/some/path")).To(Succeed()) + // set the global path to make sure we don't prefer it + Expect(os.Setenv(EnvAssetsPath, "/global/path")).To(Succeed()) + }) + + It("should prefer individual overrides, using them unmodified", func() { + Expect(BinPathFinder("otherfake", "/hardcoded/path")).To(Equal("/other/path")) + }) + + It("should convert lowercase to uppercase, remove leading numbers, and replace punctuation with underscores when resolving the env var name", func() { + Expect(BinPathFinder("123.some-fake", "/hardcoded/path")).To(Equal("/some/path")) + }) + }) + + Context("when individual overrides are missing but the global override is present", func() { + BeforeEach(func() { + Expect(os.Setenv(EnvAssetsPath, "/global/path")).To(Succeed()) + }) + It("should prefer the global override, appending the name to that path", func() { + Expect(BinPathFinder("some-fake", "/hardcoded/path")).To(Equal("/global/path/some-fake")) + }) + }) + + Context("when an asset directory is given and no overrides are present", func() { + It("should use the asset directory, appending the name to that path", func() { + Expect(BinPathFinder("some-fake", "/hardcoded/path")).To(Equal("/hardcoded/path/some-fake")) + }) + }) + + Context("when no path configuration is given", func() { + It("should just use the default path", func() { + Expect(BinPathFinder("some-fake", "")).To(Equal("/usr/local/kcp/bin/some-fake")) + }) + }) +}) diff --git a/envtest/internal/process/procattr_other.go b/envtest/internal/process/procattr_other.go new file mode 100644 index 0000000..df13b34 --- /dev/null +++ b/envtest/internal/process/procattr_other.go @@ -0,0 +1,28 @@ +//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !zos +// +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!zos + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process + +import "syscall" + +// GetSysProcAttr returns the SysProcAttr to use for the process, +// for non-unix systems this returns nil. +func GetSysProcAttr() *syscall.SysProcAttr { + return nil +} diff --git a/envtest/internal/process/procattr_unix.go b/envtest/internal/process/procattr_unix.go new file mode 100644 index 0000000..83ad509 --- /dev/null +++ b/envtest/internal/process/procattr_unix.go @@ -0,0 +1,33 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris zos + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process + +import ( + "golang.org/x/sys/unix" +) + +// GetSysProcAttr returns the SysProcAttr to use for the process, +// for unix systems this returns a SysProcAttr with Setpgid set to true, +// which inherits the parent's process group id. +func GetSysProcAttr() *unix.SysProcAttr { + return &unix.SysProcAttr{ + Setpgid: true, + } +} diff --git a/envtest/internal/process/process.go b/envtest/internal/process/process.go new file mode 100644 index 0000000..74c1a9f --- /dev/null +++ b/envtest/internal/process/process.go @@ -0,0 +1,276 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path" + "regexp" + "sync" + "syscall" + "time" +) + +// ListenAddr represents some listening address and port. +type ListenAddr struct { + Address string + Port string +} + +// URL returns a URL for this address with the given scheme and subpath. +func (l *ListenAddr) URL(scheme string, path string) *url.URL { + return &url.URL{ + Scheme: scheme, + Host: l.HostPort(), + Path: path, + } +} + +// HostPort returns the joined host-port pair for this address. +func (l *ListenAddr) HostPort() string { + return net.JoinHostPort(l.Address, l.Port) +} + +// HealthCheck describes the information needed to health-check a process via +// some health-check URL. +type HealthCheck struct { + url.URL + + // HealthCheckPollInterval is the interval which will be used for polling the + // endpoint described by Host, Port, and Path. + // + // If left empty it will default to 100 Milliseconds. + PollInterval time.Duration +} + +// State define the state of the process. +type State struct { + Cmd *exec.Cmd + + // HealthCheck describes how to check if this process is up. If we get an http.StatusOK, + // we assume the process is ready to operate. + // + // For example, the /healthz endpoint of the k8s API server, or the /health endpoint of etcd. + HealthCheck HealthCheck + + Args []string + + StopTimeout time.Duration + StartTimeout time.Duration + + Dir string + DirNeedsCleaning bool + Path string + + // ready holds whether the process is currently in ready state (hit the ready condition) or not. + // It will be set to true on a successful `Start()` and set to false on a successful `Stop()` + ready bool + + // waitDone is closed when our call to wait finishes up, and indicates that + // our process has terminated. + waitDone chan struct{} + errMu sync.Mutex + exitErr error + exited bool +} + +// Init sets up this process, configuring binary paths if missing, initializing +// temporary directories, etc. +// +// This defaults all defaultable fields. +func (ps *State) Init(name string) error { + if ps.Path == "" { + if name == "" { + return fmt.Errorf("must have at least one of name or path") + } + ps.Path = BinPathFinder(name, "") + } + + if ps.Dir == "" { + newDir, err := os.MkdirTemp("", "k8s_test_framework_") + if err != nil { + return err + } + ps.Dir = newDir + ps.DirNeedsCleaning = true + } + + if ps.StartTimeout == 0 { + ps.StartTimeout = time.Minute + } + + if ps.StopTimeout == 0 { + ps.StopTimeout = 20 * time.Second + } + return nil +} + +type stopChannel chan struct{} + +// CheckFlag checks the help output of this command for the presence of the given flag, specified +// without the leading `--` (e.g. `CheckFlag("insecure-port")` checks for `--insecure-port`), +// returning true if the flag is present. +func (ps *State) CheckFlag(flag string) (bool, error) { + cmd := exec.Command(ps.Path, "start", "options") + outContents, err := cmd.CombinedOutput() + if err != nil { + return false, fmt.Errorf("unable to run command %q to check for flag %q: %w", ps.Path, flag, err) + } + pat := `(?m)^\s*--` + flag + `\b` // (m --> multi-line --> ^ matches start of line) + matched, err := regexp.Match(pat, outContents) + if err != nil { + return false, fmt.Errorf("unable to check command %q for flag %q in help output: %w", ps.Path, flag, err) + } + return matched, nil +} + +// Start starts the apiserver, waits for it to come up, and returns an error, +// if occurred. +func (ps *State) Start(stdout, stderr io.Writer) (err error) { + if ps.ready { + return nil + } + + ps.Cmd = exec.Command(ps.Path, ps.Args...) + ps.Cmd.Stdout = stdout + ps.Cmd.Stderr = stderr + ps.Cmd.SysProcAttr = GetSysProcAttr() + + ready := make(chan bool) + timedOut := time.After(ps.StartTimeout) + pollerStopCh := make(stopChannel) + go pollURLUntilOK(ps.HealthCheck.URL, ps.HealthCheck.PollInterval, ready, pollerStopCh) + + ps.waitDone = make(chan struct{}) + + if err := ps.Cmd.Start(); err != nil { + ps.errMu.Lock() + defer ps.errMu.Unlock() + ps.exited = true + return err + } + go func() { + defer close(ps.waitDone) + err := ps.Cmd.Wait() + + ps.errMu.Lock() + defer ps.errMu.Unlock() + ps.exitErr = err + ps.exited = true + }() + + select { + case <-ready: + ps.ready = true + return nil + case <-ps.waitDone: + close(pollerStopCh) + return fmt.Errorf("timeout waiting for process %s to start successfully "+ + "(it may have failed to start, or stopped unexpectedly before becoming ready)", + path.Base(ps.Path)) + case <-timedOut: + close(pollerStopCh) + if ps.Cmd != nil { + // intentionally ignore this -- we might've crashed, failed to start, etc + ps.Cmd.Process.Signal(syscall.SIGTERM) //nolint:errcheck + } + return fmt.Errorf("timeout waiting for process %s to start", path.Base(ps.Path)) + } +} + +// Exited returns true if the process exited, and may also +// return an error (as per Cmd.Wait) if the process did not +// exit with error code 0. +func (ps *State) Exited() (bool, error) { + ps.errMu.Lock() + defer ps.errMu.Unlock() + return ps.exited, ps.exitErr +} + +func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh stopChannel) { + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // there's probably certs *somewhere*, + // but it's fine to just skip validating + // them for health checks during testing + InsecureSkipVerify: true, //nolint:gosec + }, + }, + } + if interval <= 0 { + interval = 100 * time.Millisecond + } + for { + res, err := client.Get(url.String()) + if err == nil { + res.Body.Close() + if res.StatusCode == http.StatusOK { + ready <- true + return + } + } + + select { + case <-stopCh: + return + default: + time.Sleep(interval) + } + } +} + +// Stop stops this process gracefully, waits for its termination, and cleans up +// the CertDir if necessary. +func (ps *State) Stop() error { + // Always clear the directory if we need to. + defer func() { + if ps.DirNeedsCleaning { + _ = os.RemoveAll(ps.Dir) + } + }() + if ps.Cmd == nil { + return nil + } + if done, _ := ps.Exited(); done { + return nil + } + if err := ps.Cmd.Process.Signal(syscall.SIGTERM); err != nil { + return fmt.Errorf("unable to signal for process %s to stop: %w", ps.Path, err) + } + + timedOut := time.After(ps.StopTimeout) + + select { + case <-ps.waitDone: + break + case <-timedOut: + if err := ps.Cmd.Process.Signal(syscall.SIGKILL); err != nil { + return fmt.Errorf("unable to kill process %s: %w", ps.Path, err) + } + return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path)) + } + ps.ready = false + return nil +} diff --git a/envtest/internal/process/process_suite_test.go b/envtest/internal/process/process_suite_test.go new file mode 100644 index 0000000..5a64e9d --- /dev/null +++ b/envtest/internal/process/process_suite_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestInternal(t *testing.T) { + t.Parallel() + RegisterFailHandler(Fail) + RunSpecs(t, "Envtest Process Launcher Suite") +} diff --git a/envtest/internal/process/process_test.go b/envtest/internal/process/process_test.go new file mode 100644 index 0000000..e11dc89 --- /dev/null +++ b/envtest/internal/process/process_test.go @@ -0,0 +1,365 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process_test + +import ( + "bytes" + "net" + "net/http" + "net/url" + "os" + "strconv" + "time" + + "github.com/onsi/gomega/ghttp" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/addr" + + . "github.com/kcp-dev/multicluster-provider/envtest/internal/process" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + healthURLPath = "/healthz" +) + +var _ = Describe("Start method", func() { + var ( + processState *State + server *ghttp.Server + ) + BeforeEach(func() { + server = ghttp.NewServer() + + processState = &State{ + Path: "bash", + Args: simpleBashScript, + HealthCheck: HealthCheck{ + URL: getServerURL(server), + }, + } + processState.Path = "bash" + processState.Args = simpleBashScript + + }) + AfterEach(func() { + server.Close() + }) + + Context("when process takes too long to start", func() { + BeforeEach(func() { + server.RouteToHandler("GET", healthURLPath, func(resp http.ResponseWriter, _ *http.Request) { + time.Sleep(250 * time.Millisecond) + resp.WriteHeader(http.StatusOK) + }) + }) + It("returns a timeout error", func() { + processState.StartTimeout = 200 * time.Millisecond + + err := processState.Start(nil, nil) + Expect(err).To(MatchError(ContainSubstring("timeout"))) + + Eventually(func() bool { done, _ := processState.Exited(); return done }).Should(BeTrue()) + }) + }) + + Context("when the healthcheck returns ok", func() { + BeforeEach(func() { + + server.RouteToHandler("GET", healthURLPath, ghttp.RespondWith(http.StatusOK, "")) + }) + + It("can start a process", func() { + processState.StartTimeout = 10 * time.Second + + err := processState.Start(nil, nil) + Expect(err).NotTo(HaveOccurred()) + + Consistently(processState.Exited).Should(BeFalse()) + }) + + It("hits the endpoint, and successfully starts", func() { + processState.StartTimeout = 100 * time.Millisecond + + err := processState.Start(nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(server.ReceivedRequests()).To(HaveLen(1)) + Consistently(processState.Exited).Should(BeFalse()) + }) + + Context("when the command cannot be started", func() { + var err error + + BeforeEach(func() { + processState = &State{} + processState.Path = "/nonexistent" + + err = processState.Start(nil, nil) + }) + + It("propagates the error", func() { + Expect(os.IsNotExist(err)).To(BeTrue()) + }) + + Context("but Stop() is called on it", func() { + It("does not panic", func() { + stoppingFailedProcess := func() { + Expect(processState.Stop()).To(Succeed()) + } + + Expect(stoppingFailedProcess).NotTo(Panic()) + }) + }) + }) + + Context("when IO is configured", func() { + It("can inspect stdout & stderr", func() { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + + processState.Args = []string{ + "-c", + ` + echo 'this is stderr' >&2 + echo 'that is stdout' + echo 'i started' >&2 + `, + } + processState.StartTimeout = 5 * time.Second + + Expect(processState.Start(stdout, stderr)).To(Succeed()) + Eventually(processState.Exited).Should(BeTrue()) + + Expect(stdout.String()).To(Equal("that is stdout\n")) + Expect(stderr.String()).To(Equal("this is stderr\ni started\n")) + }) + }) + }) + + Context("when the healthcheck always returns failure", func() { + BeforeEach(func() { + server.RouteToHandler("GET", healthURLPath, ghttp.RespondWith(http.StatusInternalServerError, "")) + }) + It("returns a timeout error and stops health API checker", func() { + processState.HealthCheck.URL = getServerURL(server) + processState.HealthCheck.Path = healthURLPath + processState.StartTimeout = 500 * time.Millisecond + + err := processState.Start(nil, nil) + Expect(err).To(MatchError(ContainSubstring("timeout"))) + + nrReceivedRequests := len(server.ReceivedRequests()) + Expect(nrReceivedRequests).To(Equal(5)) + time.Sleep(200 * time.Millisecond) + Expect(nrReceivedRequests).To(Equal(5)) + }) + }) + + Context("when the healthcheck isn't even listening", func() { + BeforeEach(func() { + server.Close() + }) + + It("returns a timeout error", func() { + processState.HealthCheck.Path = healthURLPath + processState.StartTimeout = 500 * time.Millisecond + + port, host, err := addr.Suggest("") + Expect(err).NotTo(HaveOccurred()) + + processState.HealthCheck.URL = url.URL{ + Scheme: "http", + Host: net.JoinHostPort(host, strconv.Itoa(port)), + } + + err = processState.Start(nil, nil) + Expect(err).To(MatchError(ContainSubstring("timeout"))) + }) + }) + + Context("when the healthcheck fails initially but succeeds eventually", func() { + BeforeEach(func() { + server.AppendHandlers( + ghttp.RespondWith(http.StatusInternalServerError, ""), + ghttp.RespondWith(http.StatusInternalServerError, ""), + ghttp.RespondWith(http.StatusInternalServerError, ""), + ghttp.RespondWith(http.StatusOK, ""), + ) + }) + + It("hits the endpoint repeatedly, and successfully starts", func() { + processState.HealthCheck.URL = getServerURL(server) + processState.HealthCheck.Path = healthURLPath + processState.StartTimeout = 20 * time.Second + + err := processState.Start(nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(server.ReceivedRequests()).To(HaveLen(4)) + Consistently(processState.Exited).Should(BeFalse()) + }) + + Context("when the polling interval is not configured", func() { + It("uses the default interval for polling", func() { + processState.HealthCheck.URL = getServerURL(server) + processState.HealthCheck.Path = "/helathz" + processState.StartTimeout = 300 * time.Millisecond + + Expect(processState.Start(nil, nil)).To(MatchError(ContainSubstring("timeout"))) + Expect(server.ReceivedRequests()).To(HaveLen(3)) + }) + }) + + Context("when the polling interval is configured", func() { + BeforeEach(func() { + processState.HealthCheck.URL = getServerURL(server) + processState.HealthCheck.Path = healthURLPath + processState.HealthCheck.PollInterval = time.Millisecond * 150 + }) + + It("hits the endpoint in the configured interval", func() { + processState.StartTimeout = 3 * processState.HealthCheck.PollInterval + + Expect(processState.Start(nil, nil)).To(MatchError(ContainSubstring("timeout"))) + Expect(server.ReceivedRequests()).To(HaveLen(3)) + }) + }) + }) +}) + +var _ = Describe("Stop method", func() { + var ( + server *ghttp.Server + processState *State + ) + BeforeEach(func() { + server = ghttp.NewServer() + server.RouteToHandler("GET", healthURLPath, ghttp.RespondWith(http.StatusOK, "")) + processState = &State{ + Path: "bash", + Args: simpleBashScript, + HealthCheck: HealthCheck{ + URL: getServerURL(server), + }, + } + processState.StartTimeout = 10 * time.Second + }) + + AfterEach(func() { + server.Close() + }) + Context("when Stop() is called", func() { + BeforeEach(func() { + Expect(processState.Start(nil, nil)).To(Succeed()) + processState.StopTimeout = 10 * time.Second + }) + + It("stops the process", func() { + Expect(processState.Stop()).To(Succeed()) + }) + + Context("multiple times", func() { + It("does not error or panic on consecutive calls", func() { + stoppingTheProcess := func() { + Expect(processState.Stop()).To(Succeed()) + } + Expect(stoppingTheProcess).NotTo(Panic()) + Expect(stoppingTheProcess).NotTo(Panic()) + Expect(stoppingTheProcess).NotTo(Panic()) + }) + }) + }) + + Context("when the command cannot be stopped", func() { + It("returns a timeout error", func() { + Expect(processState.Start(nil, nil)).To(Succeed()) + processState.StopTimeout = 1 * time.Nanosecond // much shorter than the sleep in the script + + Expect(processState.Stop()).To(MatchError(ContainSubstring("timeout"))) + }) + }) + + Context("when the directory needs to be cleaned up", func() { + It("removes the directory", func() { + var err error + + Expect(processState.Start(nil, nil)).To(Succeed()) + processState.Dir, err = os.MkdirTemp("", "k8s_test_framework_") + Expect(err).NotTo(HaveOccurred()) + processState.DirNeedsCleaning = true + processState.StopTimeout = 400 * time.Millisecond + + Expect(processState.Stop()).To(Succeed()) + Expect(processState.Dir).NotTo(BeAnExistingFile()) + }) + }) +}) + +var _ = Describe("Init", func() { + Context("when all inputs are provided", func() { + It("passes them through", func() { + ps := &State{ + Dir: "/some/dir", + Path: "/some/path/to/some/bin", + StartTimeout: 20 * time.Hour, + StopTimeout: 65537 * time.Millisecond, + } + + Expect(ps.Init("some name")).To(Succeed()) + + Expect(ps.Dir).To(Equal("/some/dir")) + Expect(ps.DirNeedsCleaning).To(BeFalse()) + Expect(ps.Path).To(Equal("/some/path/to/some/bin")) + Expect(ps.StartTimeout).To(Equal(20 * time.Hour)) + Expect(ps.StopTimeout).To(Equal(65537 * time.Millisecond)) + }) + }) + + Context("when inputs are empty", func() { + It("ps them", func() { + ps := &State{} + Expect(ps.Init("some name")).To(Succeed()) + + Expect(ps.Dir).To(BeADirectory()) + Expect(os.RemoveAll(ps.Dir)).To(Succeed()) + Expect(ps.DirNeedsCleaning).To(BeTrue()) + + Expect(ps.Path).NotTo(BeEmpty()) + + Expect(ps.StartTimeout).NotTo(BeZero()) + Expect(ps.StopTimeout).NotTo(BeZero()) + }) + }) + + Context("when neither name nor path are provided", func() { + It("returns an error", func() { + ps := &State{} + Expect(ps.Init("")).To(MatchError("must have at least one of name or path")) + }) + }) +}) + +var simpleBashScript = []string{ + "-c", "tail -f /dev/null", +} + +func getServerURL(server *ghttp.Server) url.URL { + url, err := url.Parse(server.URL()) + Expect(err).NotTo(HaveOccurred()) + url.Path = healthURLPath + return *url +} diff --git a/envtest/scheme.go b/envtest/scheme.go new file mode 100644 index 0000000..138959d --- /dev/null +++ b/envtest/scheme.go @@ -0,0 +1,41 @@ +/* +Copyright 2016 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "k8s.io/client-go/kubernetes/scheme" + + apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + tenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + topologyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/topology/v1alpha1" +) + +func init() { + if err := apisv1alpha1.AddToScheme(scheme.Scheme); err != nil { + log.Info("WARNING: failed to add apis.kcp.io/v1alpha1 to scheme", "error", err) + } + if err := corev1alpha1.AddToScheme(scheme.Scheme); err != nil { + log.Info("WARNING: failed to add core.kcp.io/v1alpha1 to scheme", "error", err) + } + if err := tenancyv1alpha1.AddToScheme(scheme.Scheme); err != nil { + log.Info("WARNING: failed to add tenancy.kcp.io/v1alpha1 to scheme", "error", err) + } + if err := topologyv1alpha1.AddToScheme(scheme.Scheme); err != nil { + log.Info("WARNING: failed to add topology.kcp.io/v1alpha1 to scheme", "error", err) + } +} diff --git a/envtest/server.go b/envtest/server.go new file mode 100644 index 0000000..08c181d --- /dev/null +++ b/envtest/server.go @@ -0,0 +1,307 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/kcp-dev/multicluster-provider/envtest/internal/controlplane" + "github.com/kcp-dev/multicluster-provider/envtest/internal/process" +) + +var log = ctrllog.Log.WithName("envtest") + +/* +It's possible to override some defaults, by setting the following environment variables: +* USE_EXISTING_KCP (boolean): if set to true, envtest will use an existing kcp. +* TEST_ASSET_KCP (string): path to the kcp binary to use +* TEST_KCP_ASSETS (string): directory containing the binaries to use (kcp). Defaults to /usr/local/kcp/bin. +* TEST_KCP_START_TIMEOUT (string supported by time.ParseDuration): timeout for test kcp to start. Defaults to 1m. +* TEST_KCP_STOP_TIMEOUT (string supported by time.ParseDuration): timeout for test kcp to start. Defaults to 20s. +* TEST_ATTACH_KCP_OUTPUT (boolean): if set to true, the kcp's stdout and stderr are attached to os.Stdout and os.Stderr +*/ +const ( + envUseExistingCluster = "USE_EXISTING_KCP" + envAttachOutput = "TEST_ATTACH_KCP_OUTPUT" + envStartTimeout = "TEST_KCP_START_TIMEOUT" + envStopTimeout = "TEST_KCP_STOP_TIMEOUT" + + defaultKcpPlaneStartTimeout = time.Minute + defaultKcpStopTimeout = 20 * time.Second +) + +// internal types we expose as part of our public API. +type ( + // Kcp is the re-exported Kcp type from the internal testing package. + Kcp = controlplane.Kcp + + // Shard is the re-exported Shard from the internal testing package. + Shard = controlplane.Shard + + // User represents a Kubernetes user to provision for auth purposes. + User = controlplane.User + + // AuthenticatedUser represets a Kubernetes user that's been provisioned. + AuthenticatedUser = controlplane.AuthenticatedUser + + // ListenAddr indicates the address and port that the API server should listen on. + ListenAddr = process.ListenAddr + + // SecureServing contains details describing how the API server should serve + // its secure endpoint. + SecureServing = controlplane.SecureServing + + // Authn is an authentication method that can be used with the control plane to + // provision users. + Authn = controlplane.Authn + + // Arguments allows configuring a process's flags. + Arguments = process.Arguments + + // Arg is a single flag with one or more values. + Arg = process.Arg +) + +var ( + // EmptyArguments constructs a new set of flags with nothing set. + // + // This is mostly useful for testing helper methods -- you'll want to call + // Configure on the APIServer (or etcd) to configure their arguments. + EmptyArguments = process.EmptyArguments +) + +// Environment creates a Kubernetes test environment that will start / stop the Kubernetes control plane and +// install extension APIs. +type Environment struct { + // Kcp is the Kcp instance. + Kcp controlplane.Kcp + + // Scheme is used to determine if conversion webhooks should be enabled + // for a particular CRD / object. + // + // Conversion webhooks are going to be enabled if an object in the scheme + // implements Hub and Spoke conversions. + // + // If nil, scheme.Scheme is used. + Scheme *runtime.Scheme + + // Config can be used to talk to the apiserver. It's automatically + // populated if not set using the standard controller-runtime config + // loading. + Config *rest.Config + + // BinaryAssetsDirectory is the path where the binaries required for the envtest are + // located in the local environment. This field can be overridden by setting TEST_KCP_ASSETS. + BinaryAssetsDirectory string + + // UseExistingCluster indicates that this environments should use an + // existing kubeconfig, instead of trying to stand up a new kcp. + UseExistingKcp *bool + + // KcpStartTimeout is the maximum duration each kcp component + // may take to start. It defaults to the TEST_KCP_START_TIMEOUT + // environment variable or 20 seconds if unspecified + KcpStartTimeout time.Duration + + // KcpStopTimeout is the maximum duration each kcp component + // may take to stop. It defaults to the TEST_KCP_STOP_TIMEOUT + // environment variable or 20 seconds if unspecified + KcpStopTimeout time.Duration + + // AttachKcpOutput indicates if kcp output will be attached to os.Stdout and os.Stderr. + // Enable this to get more visibility of the testing kcp. + // It respect TEST_ATTACH_KCP_OUTPUT environment variable. + AttachKcpOutput bool +} + +// Stop stops a running server. +// Previously installed CRDs, as listed in CRDInstallOptions.CRDs, will be uninstalled +// if CRDInstallOptions.CleanUpAfterUse are set to true. +func (te *Environment) Stop() error { + if te.useExistingKcp() { + return nil + } + + return te.Kcp.Stop() +} + +// Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on. +func (te *Environment) Start() (*rest.Config, error) { + if te.useExistingKcp() { + log.V(1).Info("using existing cluster") + if te.Config == nil { + // we want to allow people to pass in their own config, so + // only load a config if it hasn't already been set. + log.V(1).Info("automatically acquiring client configuration") + + var err error + te.Config, err = config.GetConfig() + if err != nil { + return nil, fmt.Errorf("unable to get configuration for existing cluster: %w", err) + } + } + } else { + shard := te.Kcp.GetRootShard() + + if os.Getenv(envAttachOutput) == "true" { + te.AttachKcpOutput = true + } + if te.AttachKcpOutput { + if shard.Out == nil { + shard.Out = os.Stdout + } + if shard.Err == nil { + shard.Err = os.Stderr + } + } + + shard.Path = process.BinPathFinder("kcp", te.BinaryAssetsDirectory) + te.Kcp.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory) + + if err := te.defaultTimeouts(); err != nil { + return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err) + } + shard.StartTimeout = te.KcpStartTimeout + shard.StopTimeout = te.KcpStopTimeout + + log.V(1).Info("starting control plane") + if err := te.startControlPlane(); err != nil { + return nil, fmt.Errorf("unable to start control plane itself: %w", err) + } + + // Create the *rest.Config for creating new clients + baseConfig := &rest.Config{ + // gotta go fast during tests -- we don't really care about overwhelming our test API server + QPS: 1000.0, + Burst: 2000.0, + } + + adminInfo := User{Name: "admin", Groups: []string{"system:kcp:admin"}} + adminUser, err := te.Kcp.AddUser(adminInfo, baseConfig) + if err != nil { + return te.Config, fmt.Errorf("unable to provision admin user: %w", err) + } + te.Config = adminUser.Config() + } + + // Set the default scheme if nil. + if te.Scheme == nil { + te.Scheme = scheme.Scheme + } + + // If we are bringing etcd up for the first time, it can take some time for the + // default namespace to actually be created and seen as available to the apiserver + if err := te.waitForRootWorkspaceDefaultNamespace(te.Config); err != nil { + return nil, fmt.Errorf("default namespace didn't register within deadline: %w", err) + } + + return te.Config, nil +} + +// AddUser provisions a new user for connecting to this Environment. The user will +// have the specified name & belong to the specified groups. +// +// If you specify a "base" config, the returned REST Config will contain those +// settings as well as any required by the authentication method. You can use +// this to easily specify options like QPS. +// +// This is effectively a convinience alias for Kcp.AddUser -- see that +// for more low-level details. +func (te *Environment) AddUser(user User, baseConfig *rest.Config) (*AuthenticatedUser, error) { + return te.Kcp.AddUser(user, baseConfig) +} + +func (te *Environment) startControlPlane() error { + numTries, maxRetries := 0, 5 + var err error + for ; numTries < maxRetries; numTries++ { + // Start the control plane - retry if it fails + err = te.Kcp.Start() + if err == nil { + break + } + log.Error(err, "unable to start the controlplane", "tries", numTries) + } + if numTries == maxRetries { + return fmt.Errorf("failed to start the controlplane. retried %d times: %w", numTries, err) + } + return nil +} + +func (te *Environment) waitForRootWorkspaceDefaultNamespace(config *rest.Config) error { + cfg := rest.CopyConfig(config) + cfg.Host += "/clusters/root" + cs, err := client.New(cfg, client.Options{}) + if err != nil { + return fmt.Errorf("unable to create client: %w", err) + } + // It shouldn't take longer than 5s for the default namespace to be brought up in etcd + return wait.PollUntilContextTimeout(context.TODO(), time.Millisecond*50, time.Second*5, true, func(ctx context.Context) (bool, error) { + if err = cs.Get(ctx, types.NamespacedName{Name: "default"}, &corev1.Namespace{}); err != nil { + log.V(1).Info("Waiting for default namespace to be available", "error", err) + return false, nil //nolint:nilerr + } + return true, nil + }) +} + +func (te *Environment) defaultTimeouts() error { + var err error + if te.KcpStartTimeout == 0 { + if envVal := os.Getenv(envStartTimeout); envVal != "" { + te.KcpStartTimeout, err = time.ParseDuration(envVal) + if err != nil { + return err + } + } else { + te.KcpStartTimeout = defaultKcpPlaneStartTimeout + } + } + + if te.KcpStopTimeout == 0 { + if envVal := os.Getenv(envStopTimeout); envVal != "" { + te.KcpStopTimeout, err = time.ParseDuration(envVal) + if err != nil { + return err + } + } else { + te.KcpStopTimeout = defaultKcpStopTimeout + } + } + return nil +} + +func (te *Environment) useExistingKcp() bool { + if te.UseExistingKcp == nil { + return strings.ToLower(os.Getenv(envUseExistingCluster)) == "true" + } + return *te.UseExistingKcp +} diff --git a/envtest/testing.go b/envtest/testing.go new file mode 100644 index 0000000..136040e --- /dev/null +++ b/envtest/testing.go @@ -0,0 +1,33 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +// TestingT is implemented by *testing.T and potentially other test frameworks. +type TestingT interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + FailNow() + Failed() bool + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Log(args ...any) + Logf(format string, args ...any) + Name() string + TempDir() string +} diff --git a/envtest/workspaces.go b/envtest/workspaces.go new file mode 100644 index 0000000..2169496 --- /dev/null +++ b/envtest/workspaces.go @@ -0,0 +1,222 @@ +/* +Copyright 2022 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "strings" + "time" + + "github.com/martinlindhe/base36" + "github.com/stretchr/testify/require" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/kcp-dev/kcp/sdk/apis/core" + corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + tenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + "github.com/kcp-dev/logicalcluster/v3" + + kcpclient "github.com/kcp-dev/multicluster-provider/client" +) + +const ( + // workspaceInitTimeout is set to 60 seconds. wait.ForeverTestTimeout + // is 30 seconds and the optimism on that being forever is great, but workspace + // initialisation can take a while in CI. + workspaceInitTimeout = 60 * time.Second +) + +// WorkspaceOption is an option for creating a workspace. +type WorkspaceOption func(ws *tenancyv1alpha1.Workspace) + +// WithRootShard schedules the workspace on the root shard. +func WithRootShard() WorkspaceOption { + return WithShard(corev1alpha1.RootShard) +} + +// WithShard schedules the workspace on the given shard. +func WithShard(name string) WorkspaceOption { + return WithLocation(tenancyv1alpha1.WorkspaceLocation{Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": name, + }, + }}) +} + +// WithLocation sets the location of the workspace. +func WithLocation(w tenancyv1alpha1.WorkspaceLocation) WorkspaceOption { + return func(ws *tenancyv1alpha1.Workspace) { + ws.Spec.Location = &w + } +} + +// WithType sets the type of the workspace. +func WithType(path logicalcluster.Path, name tenancyv1alpha1.WorkspaceTypeName) WorkspaceOption { + return func(ws *tenancyv1alpha1.Workspace) { + ws.Spec.Type = tenancyv1alpha1.WorkspaceTypeReference{ + Name: name, + Path: path.String(), + } + } +} + +// WithName sets the name of the workspace. +func WithName(s string, formatArgs ...interface{}) WorkspaceOption { + return func(ws *tenancyv1alpha1.Workspace) { + ws.Name = fmt.Sprintf(s, formatArgs...) + ws.GenerateName = "" + } +} + +// WithNamePrefix make the workspace be named with the given prefix plus "-". +func WithNamePrefix(prefix string) WorkspaceOption { + return func(ws *tenancyv1alpha1.Workspace) { + ws.GenerateName += prefix + "-" + } +} + +// NewWorkspaceFixture creates a new workspace under the given parent +// using the given client. +func NewWorkspaceFixture(t TestingT, clusterClient kcpclient.ClusterClient, parent logicalcluster.Path, options ...WorkspaceOption) (*tenancyv1alpha1.Workspace, logicalcluster.Path) { + t.Helper() + + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) + + ws := &tenancyv1alpha1.Workspace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-workspace-", + }, + Spec: tenancyv1alpha1.WorkspaceSpec{ + Type: tenancyv1alpha1.WorkspaceTypeReference{ + Name: tenancyv1alpha1.WorkspaceTypeName("universal"), + Path: "root", + }, + }, + } + for _, opt := range options { + opt(ws) + } + + // we are referring here to a WorkspaceType that may have just been created; if the admission controller + // does not have a fresh enough cache, our request will be denied as the admission controller does not know the + // type exists. Therefore, we can require.Eventually our way out of this problem. We expect users to create new + // types very infrequently, so we do not think this will be a serious UX issue in the product. + Eventually(t, func() (bool, string) { + err := clusterClient.Cluster(parent).Create(ctx, ws) + return err == nil, fmt.Sprintf("error creating workspace under %s: %v", parent, err) + }, wait.ForeverTestTimeout, time.Millisecond*100, "failed to create %s workspace under %s", ws.Spec.Type.Name, parent) + + wsName := ws.Name + t.Cleanup(func() { + if os.Getenv("PRESERVE") != "" { + return + } + + ctx, cancelFn := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) + defer cancelFn() + + err := clusterClient.Cluster(parent).Delete(ctx, ws) + if apierrors.IsNotFound(err) || apierrors.IsForbidden(err) { + return // ignore not found and forbidden because this probably means the parent has been deleted + } + require.NoErrorf(t, err, "failed to delete workspace %s", wsName) + }) + + Eventually(t, func() (bool, string) { + err := clusterClient.Cluster(parent).Get(ctx, client.ObjectKey{Name: ws.Name}, ws) + require.Falsef(t, apierrors.IsNotFound(err), "workspace %s was deleted", parent.Join(ws.Name)) + require.NoError(t, err, "failed to get workspace %s", parent.Join(ws.Name)) + if actual, expected := ws.Status.Phase, corev1alpha1.LogicalClusterPhaseReady; actual != expected { + return false, fmt.Sprintf("workspace phase is %s, not %s\n\n%s", actual, expected, toYaml(t, ws)) + } + return true, "" + }, workspaceInitTimeout, time.Millisecond*100, "failed to wait for %s workspace %s to become ready", ws.Spec.Type, parent.Join(ws.Name)) + + Eventually(t, func() (bool, string) { + lc := &corev1alpha1.LogicalCluster{} + if err := clusterClient.Cluster(logicalcluster.NewPath(ws.Spec.Cluster)).Get(ctx, client.ObjectKey{Name: corev1alpha1.LogicalClusterName}, lc); err != nil { + return false, fmt.Sprintf("failed to get LogicalCluster %s by cluster name %s: %v", parent.Join(ws.Name), ws.Spec.Cluster, err) + } + if err := clusterClient.Cluster(parent.Join(ws.Name)).Get(ctx, client.ObjectKey{Name: corev1alpha1.LogicalClusterName}, lc); err != nil { + return false, fmt.Sprintf("failed to get LogicalCluster %s via path: %v", parent.Join(ws.Name), err) + } + return true, "" + }, wait.ForeverTestTimeout, time.Millisecond*100, "failed to wait for %s workspace %s to become accessible", ws.Spec.Type, parent.Join(ws.Name)) + + t.Logf("Created %s workspace %s as /clusters/%s on shard %q", ws.Spec.Type, parent.Join(ws.Name), ws.Spec.Cluster, WorkspaceShardOrDie(t, clusterClient, ws).Name) + return ws, parent.Join(ws.Name) +} + +// WorkspaceShard returns the shard that a workspace is scheduled on. +func WorkspaceShard(ctx context.Context, kcpClient kcpclient.ClusterClient, ws *tenancyv1alpha1.Workspace) (*corev1alpha1.Shard, error) { + shards := &corev1alpha1.ShardList{} + err := kcpClient.Cluster(core.RootCluster.Path()).List(ctx, shards) + if err != nil { + return nil, err + } + + // best effort to get a shard name from the hash in the annotation + hash := ws.Annotations["internal.tenancy.kcp.io/shard"] + if hash == "" { + return nil, fmt.Errorf("workspace %s does not have a shard hash annotation", logicalcluster.From(ws).Path().Join(ws.Name)) + } + + for i := range shards.Items { + if name := shards.Items[i].Name; base36Sha224NameValue(name) == hash { + return &shards.Items[i], nil + } + } + + return nil, fmt.Errorf("failed to determine shard for workspace %s", ws.Name) +} + +// WorkspaceShardOrDie returns the shard that a workspace is scheduled on, or +// fails the test on error. +func WorkspaceShardOrDie(t TestingT, kcpClient kcpclient.ClusterClient, ws *tenancyv1alpha1.Workspace) *corev1alpha1.Shard { + t.Helper() + + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(cancelFunc) + + shard, err := WorkspaceShard(ctx, kcpClient, ws) + require.NoError(t, err, "failed to determine shard for workspace %s", ws.Name) + return shard +} + +func base36Sha224NameValue(name string) string { + hash := sha256.Sum224([]byte(name)) + base36hash := strings.ToLower(base36.EncodeBytes(hash[:])) + + return base36hash[:8] +} + +func toYaml(t TestingT, obj any) string { + t.Helper() + + yml, err := yaml.Marshal(obj) + require.NoError(t, err) + return string(yml) +} diff --git a/go.mod b/go.mod index c4365ab..c8402b4 100644 --- a/go.mod +++ b/go.mod @@ -4,18 +4,25 @@ go 1.23.5 require ( github.com/go-logr/logr v1.4.2 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/kcp-dev/apimachinery/v2 v2.0.1-0.20240817110845-a9eb9752bfeb github.com/kcp-dev/kcp/sdk v0.26.1 github.com/kcp-dev/logicalcluster/v3 v3.0.5 + github.com/martinlindhe/base36 v1.1.1 github.com/multicluster-runtime/multicluster-runtime v0.20.0-alpha.5 + github.com/onsi/ginkgo/v2 v2.21.0 + github.com/onsi/gomega v1.35.1 github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace - github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.10.0 - k8s.io/api v0.32.1 - k8s.io/apimachinery v0.32.1 - k8s.io/client-go v0.32.1 + github.com/stretchr/testify v1.10.0 + golang.org/x/sync v0.11.0 + golang.org/x/sys v0.30.0 + k8s.io/api v0.32.3 + k8s.io/apimachinery v0.32.3 + k8s.io/client-go v0.32.3 k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 sigs.k8s.io/controller-runtime v0.20.1 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -30,12 +37,14 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/uuid v1.6.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -43,7 +52,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/gomega v1.35.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.19.1 // indirect @@ -53,12 +61,13 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.7.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect + golang.org/x/net v0.36.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect + golang.org/x/term v0.29.0 // indirect + golang.org/x/text v0.22.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.29.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect @@ -66,8 +75,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.32.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 20b63e7..8b3ba67 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -82,6 +84,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/martinlindhe/base36 v1.1.1 h1:1F1MZ5MGghBXDZ2KJ3QfxmiydlWOGB8HCEtkap5NkVg= +github.com/martinlindhe/base36 v1.1.1/go.mod h1:vMS8PaZ5e/jV9LwFKlm0YLnXl/hpOihiBxKkIoc3g08= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -120,11 +124,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -142,42 +147,42 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= -golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -201,16 +206,16 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= -k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= +k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls= +k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k= k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= -k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= -k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= +k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= -k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= -k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= +k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= +k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/hack/boilerplate/kubernetes/boilerplate.go.txt b/hack/boilerplate/kubernetes/boilerplate.go.txt new file mode 100644 index 0000000..4b76f1f --- /dev/null +++ b/hack/boilerplate/kubernetes/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright YEAR The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/hack/run-tests.sh b/hack/run-tests.sh index 6fb48cc..d9c231c 100755 --- a/hack/run-tests.sh +++ b/hack/run-tests.sh @@ -19,5 +19,6 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CGO_ENABLED=1 go_test unit_tests \ - -tags "unit" -timeout 20m -race -v ./... +TEST_KCP_ASSETS="${PWD}/_tools" \ +CGO_ENABLED=1 \ + go_test unit_tests -short -tags "unit" -timeout 20m -race -v ./... diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh index 094f3c8..f01c711 100755 --- a/hack/verify-boilerplate.sh +++ b/hack/verify-boilerplate.sh @@ -25,4 +25,19 @@ echo "Checking file boilerplates…" _tools/boilerplate \ -boilerplates hack/boilerplate \ -exclude .github \ - -exclude virtualworkspace/forked_cache_reader.go + -exclude virtualworkspace/forked_cache_reader.go \ + -exclude envtest +_tools/boilerplate -boilerplates hack/boilerplate/kubernetes \ + -exclude envtest/doc.go \ + -exclude envtest/eventually.go \ + -exclude envtest/scheme.go \ + -exclude envtest/testing.go \ + -exclude envtest/workspaces.go \ + envtest virtualworkspace/forked_cache_reader.go +_tools/boilerplate \ + -boilerplates hack/boilerplate \ + envtest/doc.go \ + envtest/eventually.go \ + envtest/scheme.go \ + envtest/testing.go \ + envtest/workspaces.go \ No newline at end of file diff --git a/test/e2e/client_test.go b/test/e2e/client_test.go new file mode 100644 index 0000000..0be7fc7 --- /dev/null +++ b/test/e2e/client_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kcp-dev/kcp/sdk/apis/core" + + clusterclient "github.com/kcp-dev/multicluster-provider/client" + "github.com/kcp-dev/multicluster-provider/envtest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Cluster Client", Ordered, func() { + It("can access the root cluster", func(ctx context.Context) { + cli, err := clusterclient.New(kcpConfig, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + + ns := &corev1.Namespace{} + err = cli.Cluster(core.RootCluster.Path()).Get(ctx, client.ObjectKey{Name: "default"}, ns) + Expect(err).NotTo(HaveOccurred()) + }) + + It("can create a workspace and access it", func(ctx context.Context) { + cli, err := clusterclient.New(kcpConfig, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + + _, ws := envtest.NewWorkspaceFixture(GinkgoT(), cli, core.RootCluster.Path()) + + ns := &corev1.Namespace{} + err = cli.Cluster(ws).Get(ctx, client.ObjectKey{Name: "default"}, ns) + Expect(err).NotTo(HaveOccurred()) + }) +}) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go new file mode 100644 index 0000000..7900dae --- /dev/null +++ b/test/e2e/suite_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/client-go/rest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "github.com/kcp-dev/multicluster-provider/envtest" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var ( + env *envtest.Environment + kcpConfig *rest.Config +) + +func TestE2e(t *testing.T) { + RegisterFailHandler(Fail) + + // Start a shared kcp instance. + var err error + env = &envtest.Environment{AttachKcpOutput: testing.Verbose()} + kcpConfig, err = env.Start() + require.NoError(t, err, "failed to start envtest environment") + defer env.Stop() //nolint:errcheck // we don't care about the error here. + + RunSpecs(t, "Provider Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + // Prevent the metrics listener being created + metricsserver.DefaultBindAddress = "0" +}) + +var _ = AfterSuite(func() { + // Put the DefaultBindAddress back + metricsserver.DefaultBindAddress = ":8080" +})