From 8b8115769e1d78affc6685087453d5b0cd37f91f Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Mon, 15 Jul 2024 12:09:52 +0200 Subject: [PATCH 01/10] Switch the default operand image from 'quay.io/janus-idp/backstage-showcase:latest' to 'quay.io/rhdh/rhdh-hub-rhel9:latest' 'quay.io/rhdh/rhdh-hub-rhel9' is the downstream image of the Showcase image --- bundle/manifests/backstage-default-config_v1_configmap.yaml | 4 ++-- .../manifests/backstage-operator.clusterserviceversion.yaml | 6 +++--- config/manager/default-config/deployment.yaml | 4 ++-- config/manager/manager.yaml | 2 +- tests/e2e/testdata/backstage-operator-0.1.3.yaml | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bundle/manifests/backstage-default-config_v1_configmap.yaml b/bundle/manifests/backstage-default-config_v1_configmap.yaml index 5c27dc25..d4dec69d 100644 --- a/bundle/manifests/backstage-default-config_v1_configmap.yaml +++ b/bundle/manifests/backstage-default-config_v1_configmap.yaml @@ -191,7 +191,7 @@ data: command: - ./install-dynamic-plugins.sh - /dynamic-plugins-root - image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next + image: quay.io/rhdh/rhdh-hub-rhel9:latest # will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set imagePullPolicy: IfNotPresent securityContext: runAsNonRoot: true @@ -218,7 +218,7 @@ data: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/rhdh/rhdh-hub-rhel9:latest imagePullPolicy: IfNotPresent args: - "--config" diff --git a/bundle/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/manifests/backstage-operator.clusterserviceversion.yaml index b6396b6a..4e73011d 100644 --- a/bundle/manifests/backstage-operator.clusterserviceversion.yaml +++ b/bundle/manifests/backstage-operator.clusterserviceversion.yaml @@ -21,7 +21,7 @@ metadata: } ] capabilities: Seamless Upgrades - createdAt: "2024-07-08T12:45:16Z" + createdAt: "2024-07-12T14:27:27Z" operatorframework.io/suggested-namespace: backstage-system operators.operatorframework.io/builder: operator-sdk-v1.33.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 @@ -213,7 +213,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/janus-idp/backstage-showcase:latest + value: quay.io/rhdh/rhdh-hub-rhel9:latest image: quay.io/janus-idp/operator:0.3.0 livenessProbe: httpGet: @@ -317,6 +317,6 @@ spec: relatedImages: - image: quay.io/fedora/postgresql-15:latest name: postgresql - - image: quay.io/janus-idp/backstage-showcase:latest + - image: quay.io/rhdh/rhdh-hub-rhel9:latest name: backstage version: 0.3.0 diff --git a/config/manager/default-config/deployment.yaml b/config/manager/default-config/deployment.yaml index 20d25f96..9bb9d86b 100644 --- a/config/manager/default-config/deployment.yaml +++ b/config/manager/default-config/deployment.yaml @@ -38,7 +38,7 @@ spec: command: - ./install-dynamic-plugins.sh - /dynamic-plugins-root - image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next + image: quay.io/rhdh/rhdh-hub-rhel9:latest # will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set imagePullPolicy: IfNotPresent securityContext: runAsNonRoot: true @@ -65,7 +65,7 @@ spec: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/rhdh/rhdh-hub-rhel9:latest imagePullPolicy: IfNotPresent args: - "--config" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 05a0ad2d..6853eb87 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -76,7 +76,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/janus-idp/backstage-showcase:latest + value: quay.io/rhdh/rhdh-hub-rhel9:latest image: controller:latest name: manager securityContext: diff --git a/tests/e2e/testdata/backstage-operator-0.1.3.yaml b/tests/e2e/testdata/backstage-operator-0.1.3.yaml index 76094911..2fe7a832 100644 --- a/tests/e2e/testdata/backstage-operator-0.1.3.yaml +++ b/tests/e2e/testdata/backstage-operator-0.1.3.yaml @@ -739,7 +739,7 @@ data: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/rhdh/rhdh-hub-rhel9:1.1 imagePullPolicy: IfNotPresent name: install-dynamic-plugins volumeMounts: @@ -761,7 +761,7 @@ data: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/rhdh/rhdh-hub-rhel9:1.1 imagePullPolicy: IfNotPresent args: - "--config" @@ -949,7 +949,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/janus-idp/backstage-showcase:latest + value: quay.io/rhdh/rhdh-hub-rhel9:1.1 # TODO(asoro): Default image is 'quay.io/janus-idp/operator:0.1.3' on 1.1.x, # but replaced by the one from RHDH, because the Janus-IDP image expires after 14d if not updated. image: quay.io/rhdh/rhdh-rhel9-operator:1.1 From 247a2bcfb6c385c8bfd83962781089fbe5a0ba07 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Mon, 15 Jul 2024 12:12:45 +0200 Subject: [PATCH 02/10] Fix the E2E tests using the latest stable image of the operand User authentication is now needed to query the '/api/dynamic-plugins-info/loaded-plugins' API endpoint --- examples/rhdh-cr-with-app-configs.yaml | 16 +++++-- tests/e2e/e2e_test.go | 43 ++++++++++++++++- tests/e2e/e2e_upgrade_test.go | 2 +- tests/helper/helper_backstage.go | 65 ++++++++++++++++++++------ 4 files changed, 103 insertions(+), 23 deletions(-) diff --git a/examples/rhdh-cr-with-app-configs.yaml b/examples/rhdh-cr-with-app-configs.yaml index ece40664..a76de415 100644 --- a/examples/rhdh-cr-with-app-configs.yaml +++ b/examples/rhdh-cr-with-app-configs.yaml @@ -47,10 +47,16 @@ data: backend: auth: externalAccess: - - type: legacy - options: - subject: legacy-default-config - secret: "${BACKEND_SECRET}" + - type: legacy + options: + subject: legacy-default-config + secret: "${BACKEND_SECRET}" + auth: + environment: development + providers: + guest: + # using the guest user to query the '/api/dynamic-plugins-info/loaded-plugins' endpoint. + dangerouslyAllowOutsideDevelopment: true --- apiVersion: v1 kind: Secret @@ -128,7 +134,7 @@ data: initialDelay: { seconds: 15} - package: ./dynamic-plugins/dist/backstage-plugin-techdocs-backend-dynamic pluginConfig: - # Reference documentation http://backstage.io/docs/features/techdocs/configuration + # Reference documentation https://backstage.io/docs/features/techdocs/configuration # Note: After experimenting with basic setup, use CI/CD to generate docs # and an external cloud storage when deploying TechDocs for production use-case. # https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-basic-to-recommended-deployment-approach diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index b2db9d4f..c587484b 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -15,7 +15,11 @@ package e2e import ( + "crypto/tls" + "encoding/json" "fmt" + "io" + "net/http" "os/exec" "path/filepath" "strconv" @@ -78,9 +82,44 @@ var _ = Describe("Backstage Operator E2E", func() { crName: "bs-app-config", additionalApiEndpointTests: []helper.ApiEndpointTest{ { - Endpoint: "/api/dynamic-plugins-info/loaded-plugins", + Endpoint: "/api/dynamic-plugins-info/loaded-plugins", + BearerTokenRetrievalFn: func(baseUrl string) (string, error) { // Authenticated endpoint that does not accept service tokens + url := fmt.Sprintf("%s/api/auth/guest/refresh", baseUrl) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, // #nosec G402 -- test code only, not used in production + }, + } + httpClient := &http.Client{Transport: tr} + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", fmt.Errorf("error while building request to GET %q: %w", url, err) + } + req.Header.Add("Accept", "application/json") + resp, err := httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("error while trying to GET %q: %w", url, err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("error while trying to read response body from 'GET %q': %w", url, err) + } + if resp.StatusCode != 200 { + return "", fmt.Errorf("expected status code 200, but got %d in response to 'GET %q', body: %s", resp.StatusCode, url, string(body)) + } + var authResponse helper.BackstageAuthRefreshResponse + err = json.Unmarshal(body, &authResponse) + if err != nil { + return "", fmt.Errorf("error while trying to decode response body from 'GET %q': %w", url, err) + } + return authResponse.BackstageIdentity.Token, nil + }, ExpectedHttpStatusCode: 200, BodyMatcher: SatisfyAll( + ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-quay-dynamic"), + ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-regex-dynamic"), + ContainSubstring("roadiehq-scaffolder-backend-module-utils-dynamic"), ContainSubstring("backstage-plugin-catalog-backend-module-github-dynamic"), ContainSubstring("backstage-plugin-techdocs-backend-dynamic"), ContainSubstring("backstage-plugin-catalog-backend-module-gitlab-dynamic")), @@ -187,7 +226,7 @@ var _ = Describe("Backstage Operator E2E", func() { }) func ensureRouteIsReachable(ns string, crName string, additionalApiEndpointTests []helper.ApiEndpointTest) { - Eventually(helper.VerifyBackstageRoute, time.Minute, time.Second). + Eventually(helper.VerifyBackstageRoute, 5*time.Minute, time.Second). WithArguments(ns, crName, additionalApiEndpointTests). Should(Succeed()) } diff --git a/tests/e2e/e2e_upgrade_test.go b/tests/e2e/e2e_upgrade_test.go index 9a488c5d..e6acabca 100644 --- a/tests/e2e/e2e_upgrade_test.go +++ b/tests/e2e/e2e_upgrade_test.go @@ -93,7 +93,7 @@ var _ = Describe("Operator upgrade with existing instances", func() { }) By("checking the status of the existing CR") - Eventually(helper.VerifyBackstageCRStatus, time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`).Should(Succeed()) + Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`).Should(Succeed()) By("checking the Backstage operand pod") Eventually(func(g Gomega) { diff --git a/tests/helper/helper_backstage.go b/tests/helper/helper_backstage.go index 94532e11..55d65bcf 100644 --- a/tests/helper/helper_backstage.go +++ b/tests/helper/helper_backstage.go @@ -20,20 +20,48 @@ import ( "io" "net/http" "os/exec" - "redhat-developer/red-hat-developer-hub-operator/pkg/model" "strings" + "redhat-developer/red-hat-developer-hub-operator/pkg/model" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" ) type ApiEndpointTest struct { + BearerTokenRetrievalFn func(baseUrl string) (string, error) Endpoint string ExpectedHttpStatusCode int BodyMatcher types.GomegaMatcher } +// BackstageAuthRefreshResponse is the struct of the response returned by the '/api/auth/:user/refresh' API endpoint. +// +// Example: +// +// { +// "backstageIdentity": { +// "expiresInSeconds": 3600, +// "identity": { +// "ownershipEntityRefs": [ +// "user:development/guest" +// ], +// "type": "user", +// "userEntityRef": "user:development/guest" +// }, +// "token": "eyJ0..." +// }, +// "profile": {} +// } +type BackstageAuthRefreshResponse struct { + BackstageIdentity BackstageIdentity `json:"backstageIdentity,omitempty"` +} + +type BackstageIdentity struct { + Token string `json:"token,omitempty"` +} + func VerifyBackstagePodStatus(g Gomega, ns string, crName string, expectedStatus string) { cmd := exec.Command("kubectl", "get", "pods", "-l", "rhdh.redhat.com/app=backstage-"+crName, @@ -111,21 +139,13 @@ func GetBackstageRouteHost(ns string, crName string) (string, error) { return fmt.Sprintf("%s.%s", subDomain, ingressDomain), err } +// unauthenticated endpoints var defaultApiEndpointTests = []ApiEndpointTest{ { Endpoint: "/", ExpectedHttpStatusCode: 200, BodyMatcher: ContainSubstring("You need to enable JavaScript to run this app"), }, - { - Endpoint: "/api/dynamic-plugins-info/loaded-plugins", - ExpectedHttpStatusCode: 200, - BodyMatcher: SatisfyAll( - ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-quay-dynamic"), - ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-regex-dynamic"), - ContainSubstring("roadiehq-scaffolder-backend-module-utils-dynamic"), - ), - }, } func VerifyBackstageRoute(g Gomega, ns string, crName string, tests []ApiEndpointTest) { @@ -133,6 +153,7 @@ func VerifyBackstageRoute(g Gomega, ns string, crName string, tests []ApiEndpoin fmt.Fprintln(GinkgoWriter, host) g.Expect(err).ShouldNot(HaveOccurred()) g.Expect(host).ShouldNot(BeEmpty()) + baseUrl := fmt.Sprintf("https://%s", host) tr := &http.Transport{ TLSClientConfig: &tls.Config{ @@ -142,17 +163,31 @@ func VerifyBackstageRoute(g Gomega, ns string, crName string, tests []ApiEndpoin httpClient := &http.Client{Transport: tr} performTest := func(tt ApiEndpointTest) { - url := fmt.Sprintf("https://%s/%s", host, strings.TrimPrefix(tt.Endpoint, "/")) + url := fmt.Sprintf("%s/%s", baseUrl, strings.TrimPrefix(tt.Endpoint, "/")) + + req, reqErr := http.NewRequest("GET", url, nil) + g.Expect(reqErr).ShouldNot(HaveOccurred(), fmt.Sprintf("error while building request to GET %q", url)) + + req.Header.Add("Accept", "application/json") + + if tt.BearerTokenRetrievalFn != nil { + bearerToken, tErr := tt.BearerTokenRetrievalFn(baseUrl) + g.Expect(tErr).ShouldNot(HaveOccurred(), fmt.Sprintf("error while retrieving bearer token, context: %q", tt.Endpoint)) + if bearerToken != "" { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", bearerToken)) + } + } + fmt.Fprintf(GinkgoWriter, "--> GET %q\n", url) - resp, rErr := httpClient.Get(url) + resp, rErr := httpClient.Do(req) g.Expect(rErr).ShouldNot(HaveOccurred(), fmt.Sprintf("error while trying to GET %q", url)) defer resp.Body.Close() - - g.Expect(resp.StatusCode).Should(Equal(tt.ExpectedHttpStatusCode), "context: "+tt.Endpoint) body, rErr := io.ReadAll(resp.Body) g.Expect(rErr).ShouldNot(HaveOccurred(), fmt.Sprintf("error while trying to read response body from 'GET %q'", url)) + bodyStr := string(body) + g.Expect(resp.StatusCode).Should(Equal(tt.ExpectedHttpStatusCode), fmt.Sprintf("context: %s\n===Response body===\n%s", tt.Endpoint, bodyStr)) if tt.BodyMatcher != nil { - g.Expect(string(body)).Should(tt.BodyMatcher, "context: "+tt.Endpoint) + g.Expect(bodyStr).Should(tt.BodyMatcher, "context: "+tt.Endpoint) } } allTests := append(defaultApiEndpointTests, tests...) From 0f56bfe8df97b06f9dc6097aac91e03c4d646c3e Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Mon, 15 Jul 2024 18:23:41 +0200 Subject: [PATCH 03/10] wip: show controller logs in case CR status is not updated --- tests/e2e/e2e_suite_test.go | 10 ++++++++++ tests/e2e/e2e_upgrade_test.go | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index bbe60b5f..52282ed4 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -223,6 +223,16 @@ func verifyControllerUp(g Gomega, managerPodLabel string) { g.Expect(string(status)).Should(Equal("Running"), fmt.Sprintf("controller pod in %s status", status)) } +func getControllerLogs(managerPodLabel string) string { + // Get pod name + cmd := exec.Command(helper.GetPlatformTool(), "logs", + "-l", managerPodLabel, + "-n", _namespace, + ) + output, _ := helper.Run(cmd) + return string(output) +} + func uninstallOperator() { switch testMode { case rhdhLatestTestMode, rhdhNextTestMode, rhdhAirgapTestMode: diff --git a/tests/e2e/e2e_upgrade_test.go b/tests/e2e/e2e_upgrade_test.go index e6acabca..85bc7210 100644 --- a/tests/e2e/e2e_upgrade_test.go +++ b/tests/e2e/e2e_upgrade_test.go @@ -93,7 +93,8 @@ var _ = Describe("Operator upgrade with existing instances", func() { }) By("checking the status of the existing CR") - Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`).Should(Succeed()) + Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`). + Should(Succeed(), fmt.Sprintf("=== Operator logs ===\n%s\n", getControllerLogs(managerPodLabel))) By("checking the Backstage operand pod") Eventually(func(g Gomega) { From cae2b5e6422091d280ac319959b9fed0d73318c8 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Tue, 16 Jul 2024 10:08:44 +0200 Subject: [PATCH 04/10] Fix the upgrade path E2E tests by ensuring the CR on 1.1 is still using v1alpha1 as version Otherwise, the 1.x operator is not aware of the v1alpha2 version --- examples/bs1.yaml | 3 --- tests/e2e/e2e_upgrade_test.go | 21 ++++++++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/examples/bs1.yaml b/examples/bs1.yaml index 458933da..9d5747d4 100644 --- a/examples/bs1.yaml +++ b/examples/bs1.yaml @@ -2,6 +2,3 @@ apiVersion: rhdh.redhat.com/v1alpha2 kind: Backstage metadata: name: bs1 - - - diff --git a/tests/e2e/e2e_upgrade_test.go b/tests/e2e/e2e_upgrade_test.go index 85bc7210..f89c008c 100644 --- a/tests/e2e/e2e_upgrade_test.go +++ b/tests/e2e/e2e_upgrade_test.go @@ -16,6 +16,7 @@ package e2e import ( "fmt" + "io" "os/exec" "path/filepath" "time" @@ -49,13 +50,10 @@ var _ = Describe("Operator upgrade with existing instances", func() { When("Previous version of operator is installed and CR is created", func() { const managerPodLabel = "control-plane=controller-manager" + const crName = "my-backstage-app" // 0.1.3 is the version of the operator in the 1.1.x branch var fromDeploymentManifest = filepath.Join(projectDir, "tests", "e2e", "testdata", "backstage-operator-0.1.3.yaml") - var ( - crName = "bs1" - crPath = filepath.Join(projectDir, "examples", "bs1.yaml") - ) BeforeEach(func() { if testMode != defaultDeployTestMode { @@ -71,9 +69,22 @@ var _ = Describe("Operator upgrade with existing instances", func() { Expect(err).ShouldNot(HaveOccurred()) EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, time.Second).WithArguments(managerPodLabel).Should(Succeed()) - cmd = exec.Command(helper.GetPlatformTool(), "apply", "-f", crPath, "-n", ns) + cmd = exec.Command(helper.GetPlatformTool(), "-n", ns, "create", "-f", "-") + stdin, err := cmd.StdinPipe() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + go func() { + defer stdin.Close() + _, _ = io.WriteString(stdin, fmt.Sprintf(` +apiVersion: rhdh.redhat.com/v1alpha1 +kind: Backstage +metadata: + name: my-backstage-app + namespace: %s +`, ns)) + }() _, err = helper.Run(cmd) Expect(err).ShouldNot(HaveOccurred()) + // Reason is DeployOK in 1.1.x, but was renamed to Deployed in 1.2 Eventually(helper.VerifyBackstageCRStatus, time.Minute, time.Second).WithArguments(ns, crName, `"reason":"DeployOK"`).Should(Succeed()) }) From a9e0e64fce1aca4f23622d92aca1a62e3009feb9 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Tue, 16 Jul 2024 10:27:20 +0200 Subject: [PATCH 05/10] Also display operand logs in case there is an assertion failure --- tests/e2e/e2e_suite_test.go | 7 +++---- tests/e2e/e2e_upgrade_test.go | 11 ++++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index 52282ed4..4ce06edf 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -223,11 +223,10 @@ func verifyControllerUp(g Gomega, managerPodLabel string) { g.Expect(string(status)).Should(Equal("Running"), fmt.Sprintf("controller pod in %s status", status)) } -func getControllerLogs(managerPodLabel string) string { - // Get pod name +func getPodLogs(ns string, label string) string { cmd := exec.Command(helper.GetPlatformTool(), "logs", - "-l", managerPodLabel, - "-n", _namespace, + "-l", label, + "-n", ns, ) output, _ := helper.Run(cmd) return string(output) diff --git a/tests/e2e/e2e_upgrade_test.go b/tests/e2e/e2e_upgrade_test.go index f89c008c..8cda7548 100644 --- a/tests/e2e/e2e_upgrade_test.go +++ b/tests/e2e/e2e_upgrade_test.go @@ -105,13 +105,16 @@ metadata: By("checking the status of the existing CR") Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`). - Should(Succeed(), fmt.Sprintf("=== Operator logs ===\n%s\n", getControllerLogs(managerPodLabel))) + Should(Succeed(), func() string { + return fmt.Sprintf("=== Operator logs ===\n%s\n", getPodLogs(_namespace, managerPodLabel)) + }) By("checking the Backstage operand pod") + crLabel := fmt.Sprintf("rhdh.redhat.com/app=backstage-%s", crName) Eventually(func(g Gomega) { // Get pod name cmd := exec.Command(helper.GetPlatformTool(), "get", - "pods", "-l", fmt.Sprintf("rhdh.redhat.com/app=backstage-%s", crName), + "pods", "-l", crLabel, "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+ "{{ \"\\n\" }}{{ end }}{{ end }}", "-n", ns, @@ -120,7 +123,9 @@ metadata: g.Expect(err).ShouldNot(HaveOccurred()) podNames := helper.GetNonEmptyLines(string(podOutput)) g.Expect(podNames).Should(HaveLen(1), fmt.Sprintf("expected 1 Backstage operand pod(s) running, but got %d", len(podNames))) - }, 5*time.Minute, time.Second).Should(Succeed()) + }, 5*time.Minute, time.Second).Should(Succeed(), func() string { + return fmt.Sprintf("=== Operand logs ===\n%s\n", getPodLogs(ns, crLabel)) + }) }) }) From 507160b7cf3e702df4c0a87bbbee5110aca36f36 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Tue, 16 Jul 2024 10:41:24 +0200 Subject: [PATCH 06/10] Adjust timeouts because the operand image takes a bit longer to start --- tests/e2e/e2e_upgrade_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/e2e_upgrade_test.go b/tests/e2e/e2e_upgrade_test.go index 8cda7548..1fd31d0c 100644 --- a/tests/e2e/e2e_upgrade_test.go +++ b/tests/e2e/e2e_upgrade_test.go @@ -100,11 +100,11 @@ metadata: It("should successfully reconcile existing CR when upgrading the operator", func() { By("Upgrading the operator", func() { installOperatorWithMakeDeploy(false) - EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, time.Second).WithArguments(managerPodLabel).Should(Succeed()) + EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, 3*time.Second).WithArguments(managerPodLabel).Should(Succeed()) }) By("checking the status of the existing CR") - Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`). + Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, 3*time.Second).WithArguments(ns, crName, `"reason":"Deployed"`). Should(Succeed(), func() string { return fmt.Sprintf("=== Operator logs ===\n%s\n", getPodLogs(_namespace, managerPodLabel)) }) @@ -123,7 +123,7 @@ metadata: g.Expect(err).ShouldNot(HaveOccurred()) podNames := helper.GetNonEmptyLines(string(podOutput)) g.Expect(podNames).Should(HaveLen(1), fmt.Sprintf("expected 1 Backstage operand pod(s) running, but got %d", len(podNames))) - }, 5*time.Minute, time.Second).Should(Succeed(), func() string { + }, 10*time.Minute, 5*time.Second).Should(Succeed(), func() string { return fmt.Sprintf("=== Operand logs ===\n%s\n", getPodLogs(ns, crLabel)) }) }) From 89b0750b167ae0f1f7d009d7d37354cf1ee110b6 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Tue, 16 Jul 2024 11:24:49 +0200 Subject: [PATCH 07/10] Bump setup-minikube action to the latest It should hopefully fix the random network connection issues --- .github/workflows/nightly.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index ca29663f..c40ce9fb 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -49,9 +49,7 @@ jobs: - name: Start Minikube if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }} - uses: medyagh/setup-minikube@317d92317e473a10540357f1f4b2878b80ee7b95 # v0.0.16 - with: - addons: ingress + uses: medyagh/setup-minikube@d8c0eb871f6f455542491d86a574477bd3894533 # v0.0.18 - name: Run E2E tests (Operator Upgrade path) # Testing upgrade from 1.1.x From 45d36308a9d023949d89b77b906afe0925910177 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Tue, 16 Jul 2024 12:04:51 +0200 Subject: [PATCH 08/10] Separate upgrade and non-upgrade tests as different jobs This will reduce the likeliness of random connectivity failures --- .github/workflows/nightly.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index c40ce9fb..b130a0ce 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -13,9 +13,13 @@ jobs: fail-fast: false matrix: branch: [ main, 1.2.x, 1.1.x ] - name: E2E Tests - ${{ matrix.branch }} + test_upgrade: [ 'true', 'false' ] + exclude: + - branch: 1.1.x # Testing upgrade from 1.1.x + test_upgrade: 'true' + name: 'E2E Tests - ${{ matrix.branch }} - upgrade=${{ matrix.test_upgrade }}' concurrency: - group: ${{ github.workflow }}-${{ matrix.branch }} + group: '${{ github.workflow }}-${{ matrix.branch }}-${{ matrix.test_upgrade }}' cancel-in-progress: true env: CONTAINER_ENGINE: podman @@ -52,15 +56,14 @@ jobs: uses: medyagh/setup-minikube@d8c0eb871f6f455542491d86a574477bd3894533 # v0.0.18 - name: Run E2E tests (Operator Upgrade path) - # Testing upgrade from 1.1.x - if: ${{ matrix.branch != '1.1.x' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }} + if: ${{ matrix.test_upgrade == 'true' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }} env: BACKSTAGE_OPERATOR_TESTS_PLATFORM: minikube IMG: ${{ env.OPERATOR_IMAGE }} run: make test-e2e-upgrade - name: Run E2E tests - if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }} + if: ${{ matrix.test_upgrade == 'false' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }} env: BACKSTAGE_OPERATOR_TESTS_PLATFORM: minikube IMG: ${{ env.OPERATOR_IMAGE }} From cc27ccbaae40b7df99548d36c8ddcd6f80a47892 Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Tue, 16 Jul 2024 17:57:26 +0200 Subject: [PATCH 09/10] Revert "Switch the default operand image from 'quay.io/janus-idp/backstage-showcase:latest' to 'quay.io/rhdh/rhdh-hub-rhel9:latest'" This reverts commit 8b8115769e1d78affc6685087453d5b0cd37f91f. --- bundle/manifests/backstage-default-config_v1_configmap.yaml | 4 ++-- .../manifests/backstage-operator.clusterserviceversion.yaml | 6 +++--- config/manager/default-config/deployment.yaml | 4 ++-- config/manager/manager.yaml | 2 +- tests/e2e/testdata/backstage-operator-0.1.3.yaml | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bundle/manifests/backstage-default-config_v1_configmap.yaml b/bundle/manifests/backstage-default-config_v1_configmap.yaml index d4dec69d..5c27dc25 100644 --- a/bundle/manifests/backstage-default-config_v1_configmap.yaml +++ b/bundle/manifests/backstage-default-config_v1_configmap.yaml @@ -191,7 +191,7 @@ data: command: - ./install-dynamic-plugins.sh - /dynamic-plugins-root - image: quay.io/rhdh/rhdh-hub-rhel9:latest # will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set + image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent securityContext: runAsNonRoot: true @@ -218,7 +218,7 @@ data: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/rhdh/rhdh-hub-rhel9:latest + image: quay.io/janus-idp/backstage-showcase:latest imagePullPolicy: IfNotPresent args: - "--config" diff --git a/bundle/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/manifests/backstage-operator.clusterserviceversion.yaml index 4e73011d..b6396b6a 100644 --- a/bundle/manifests/backstage-operator.clusterserviceversion.yaml +++ b/bundle/manifests/backstage-operator.clusterserviceversion.yaml @@ -21,7 +21,7 @@ metadata: } ] capabilities: Seamless Upgrades - createdAt: "2024-07-12T14:27:27Z" + createdAt: "2024-07-08T12:45:16Z" operatorframework.io/suggested-namespace: backstage-system operators.operatorframework.io/builder: operator-sdk-v1.33.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 @@ -213,7 +213,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/rhdh/rhdh-hub-rhel9:latest + value: quay.io/janus-idp/backstage-showcase:latest image: quay.io/janus-idp/operator:0.3.0 livenessProbe: httpGet: @@ -317,6 +317,6 @@ spec: relatedImages: - image: quay.io/fedora/postgresql-15:latest name: postgresql - - image: quay.io/rhdh/rhdh-hub-rhel9:latest + - image: quay.io/janus-idp/backstage-showcase:latest name: backstage version: 0.3.0 diff --git a/config/manager/default-config/deployment.yaml b/config/manager/default-config/deployment.yaml index 9bb9d86b..20d25f96 100644 --- a/config/manager/default-config/deployment.yaml +++ b/config/manager/default-config/deployment.yaml @@ -38,7 +38,7 @@ spec: command: - ./install-dynamic-plugins.sh - /dynamic-plugins-root - image: quay.io/rhdh/rhdh-hub-rhel9:latest # will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set + image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent securityContext: runAsNonRoot: true @@ -65,7 +65,7 @@ spec: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/rhdh/rhdh-hub-rhel9:latest + image: quay.io/janus-idp/backstage-showcase:latest imagePullPolicy: IfNotPresent args: - "--config" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 6853eb87..05a0ad2d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -76,7 +76,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/rhdh/rhdh-hub-rhel9:latest + value: quay.io/janus-idp/backstage-showcase:latest image: controller:latest name: manager securityContext: diff --git a/tests/e2e/testdata/backstage-operator-0.1.3.yaml b/tests/e2e/testdata/backstage-operator-0.1.3.yaml index 2fe7a832..76094911 100644 --- a/tests/e2e/testdata/backstage-operator-0.1.3.yaml +++ b/tests/e2e/testdata/backstage-operator-0.1.3.yaml @@ -739,7 +739,7 @@ data: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/rhdh/rhdh-hub-rhel9:1.1 + image: quay.io/janus-idp/backstage-showcase:latest imagePullPolicy: IfNotPresent name: install-dynamic-plugins volumeMounts: @@ -761,7 +761,7 @@ data: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/rhdh/rhdh-hub-rhel9:1.1 + image: quay.io/janus-idp/backstage-showcase:latest imagePullPolicy: IfNotPresent args: - "--config" @@ -949,7 +949,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/rhdh/rhdh-hub-rhel9:1.1 + value: quay.io/janus-idp/backstage-showcase:latest # TODO(asoro): Default image is 'quay.io/janus-idp/operator:0.1.3' on 1.1.x, # but replaced by the one from RHDH, because the Janus-IDP image expires after 14d if not updated. image: quay.io/rhdh/rhdh-rhel9-operator:1.1 From 66eb66807ad5095495ca0a807c406bd0a3c192af Mon Sep 17 00:00:00 2001 From: Armel Soro Date: Tue, 16 Jul 2024 18:05:33 +0200 Subject: [PATCH 10/10] Switch the default operand image tag from 'latest' to 'next' 'next' is the development tag of the upstream Showcase image, and should hopefully be stable. The goal with this is to monitor how our tests behave against this 'next' tag. If it proves too unstable, we might want to switch back to using the downstream RHDH image (`quay.io/rhdh/rhdh-hub-rhel9:latest`). --- bundle/manifests/backstage-default-config_v1_configmap.yaml | 5 +++-- .../manifests/backstage-operator.clusterserviceversion.yaml | 6 +++--- config/manager/default-config/deployment.yaml | 5 +++-- config/manager/manager.yaml | 2 +- tests/e2e/testdata/backstage-operator-0.1.3.yaml | 6 +++--- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/bundle/manifests/backstage-default-config_v1_configmap.yaml b/bundle/manifests/backstage-default-config_v1_configmap.yaml index 5c27dc25..e4c09416 100644 --- a/bundle/manifests/backstage-default-config_v1_configmap.yaml +++ b/bundle/manifests/backstage-default-config_v1_configmap.yaml @@ -191,7 +191,8 @@ data: command: - ./install-dynamic-plugins.sh - /dynamic-plugins-root - image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next + # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set + image: quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent securityContext: runAsNonRoot: true @@ -218,7 +219,7 @@ data: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent args: - "--config" diff --git a/bundle/manifests/backstage-operator.clusterserviceversion.yaml b/bundle/manifests/backstage-operator.clusterserviceversion.yaml index b6396b6a..168cebc0 100644 --- a/bundle/manifests/backstage-operator.clusterserviceversion.yaml +++ b/bundle/manifests/backstage-operator.clusterserviceversion.yaml @@ -21,7 +21,7 @@ metadata: } ] capabilities: Seamless Upgrades - createdAt: "2024-07-08T12:45:16Z" + createdAt: "2024-07-16T16:00:50Z" operatorframework.io/suggested-namespace: backstage-system operators.operatorframework.io/builder: operator-sdk-v1.33.0 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 @@ -213,7 +213,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/janus-idp/backstage-showcase:latest + value: quay.io/janus-idp/backstage-showcase:next image: quay.io/janus-idp/operator:0.3.0 livenessProbe: httpGet: @@ -317,6 +317,6 @@ spec: relatedImages: - image: quay.io/fedora/postgresql-15:latest name: postgresql - - image: quay.io/janus-idp/backstage-showcase:latest + - image: quay.io/janus-idp/backstage-showcase:next name: backstage version: 0.3.0 diff --git a/config/manager/default-config/deployment.yaml b/config/manager/default-config/deployment.yaml index 20d25f96..404c68bd 100644 --- a/config/manager/default-config/deployment.yaml +++ b/config/manager/default-config/deployment.yaml @@ -38,7 +38,8 @@ spec: command: - ./install-dynamic-plugins.sh - /dynamic-plugins-root - image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next + # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set + image: quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent securityContext: runAsNonRoot: true @@ -65,7 +66,7 @@ spec: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent args: - "--config" diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 05a0ad2d..e3c4286a 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -76,7 +76,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/janus-idp/backstage-showcase:latest + value: quay.io/janus-idp/backstage-showcase:next image: controller:latest name: manager securityContext: diff --git a/tests/e2e/testdata/backstage-operator-0.1.3.yaml b/tests/e2e/testdata/backstage-operator-0.1.3.yaml index 76094911..9ffdba40 100644 --- a/tests/e2e/testdata/backstage-operator-0.1.3.yaml +++ b/tests/e2e/testdata/backstage-operator-0.1.3.yaml @@ -739,7 +739,7 @@ data: - name: NPM_CONFIG_USERCONFIG value: /opt/app-root/src/.npmrc.dynamic-plugins # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent name: install-dynamic-plugins volumeMounts: @@ -761,7 +761,7 @@ data: containers: - name: backstage-backend # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set - image: quay.io/janus-idp/backstage-showcase:latest + image: quay.io/janus-idp/backstage-showcase:next imagePullPolicy: IfNotPresent args: - "--config" @@ -949,7 +949,7 @@ spec: - name: RELATED_IMAGE_postgresql value: quay.io/fedora/postgresql-15:latest - name: RELATED_IMAGE_backstage - value: quay.io/janus-idp/backstage-showcase:latest + value: quay.io/janus-idp/backstage-showcase:next # TODO(asoro): Default image is 'quay.io/janus-idp/operator:0.1.3' on 1.1.x, # but replaced by the one from RHDH, because the Janus-IDP image expires after 14d if not updated. image: quay.io/rhdh/rhdh-rhel9-operator:1.1