chore: Switch the default operand image tag from latest to next and fix CI nightly job [RHIDP-3157]

.github/workflows/nightly.yaml

@@ -13,9 +13,13 @@ jobs:
       fail-fast: false
       matrix:
         branch: [ main, 1.2.x, 1.1.x ]
-    name: E2E Tests - ${{ matrix.branch }}
+        test_upgrade: [ 'true', 'false' ]
+        exclude:
+          - branch: 1.1.x # Testing upgrade from 1.1.x
+            test_upgrade: 'true'
+    name: 'E2E Tests - ${{ matrix.branch }} - upgrade=${{ matrix.test_upgrade }}'
     concurrency:
-      group: ${{ github.workflow }}-${{ matrix.branch }}
+      group: '${{ github.workflow }}-${{ matrix.branch }}-${{ matrix.test_upgrade }}'
       cancel-in-progress: true
     env:
       CONTAINER_ENGINE: podman
@@ -49,20 +53,17 @@ jobs:
 
       - name: Start Minikube
         if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
-        uses: medyagh/setup-minikube@317d92317e473a10540357f1f4b2878b80ee7b95 # v0.0.16
-        with:
-          addons: ingress
+        uses: medyagh/setup-minikube@d8c0eb871f6f455542491d86a574477bd3894533 # v0.0.18
 
       - name: Run E2E tests (Operator Upgrade path)
-        # Testing upgrade from 1.1.x
-        if: ${{ matrix.branch != '1.1.x' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
+        if: ${{ matrix.test_upgrade == 'true' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
         env:
           BACKSTAGE_OPERATOR_TESTS_PLATFORM: minikube
           IMG: ${{ env.OPERATOR_IMAGE }}
         run: make test-e2e-upgrade
 
       - name: Run E2E tests
-        if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
+        if: ${{ matrix.test_upgrade == 'false' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
         env:
           BACKSTAGE_OPERATOR_TESTS_PLATFORM: minikube
           IMG: ${{ env.OPERATOR_IMAGE }}

bundle/manifests/backstage-default-config_v1_configmap.yaml

@@ -191,7 +191,8 @@ data:
               command:
                 - ./install-dynamic-plugins.sh
                 - /dynamic-plugins-root
-              image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next
+              # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               securityContext:
                 runAsNonRoot: true
@@ -218,7 +219,7 @@ data:
           containers:
             - name: backstage-backend
               # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-              image: quay.io/janus-idp/backstage-showcase:latest
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               args:
                 - "--config"

bundle/manifests/backstage-operator.clusterserviceversion.yaml

@@ -21,7 +21,7 @@ metadata:
         }
       ]
     capabilities: Seamless Upgrades
-    createdAt: "2024-07-08T12:45:16Z"
+    createdAt: "2024-07-16T16:00:50Z"
     operatorframework.io/suggested-namespace: backstage-system
     operators.operatorframework.io/builder: operator-sdk-v1.33.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -213,7 +213,7 @@ spec:
                 - name: RELATED_IMAGE_postgresql
                   value: quay.io/fedora/postgresql-15:latest
                 - name: RELATED_IMAGE_backstage
-                  value: quay.io/janus-idp/backstage-showcase:latest
+                  value: quay.io/janus-idp/backstage-showcase:next
                 image: quay.io/janus-idp/operator:0.3.0
                 livenessProbe:
                   httpGet:
@@ -317,6 +317,6 @@ spec:
   relatedImages:
   - image: quay.io/fedora/postgresql-15:latest
     name: postgresql
-  - image: quay.io/janus-idp/backstage-showcase:latest
+  - image: quay.io/janus-idp/backstage-showcase:next
     name: backstage
   version: 0.3.0

config/manager/default-config/deployment.yaml

@@ -38,7 +38,8 @@ spec:
           command:
             - ./install-dynamic-plugins.sh
             - /dynamic-plugins-root
-          image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next
+          # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
+          image: quay.io/janus-idp/backstage-showcase:next
           imagePullPolicy: IfNotPresent
           securityContext:
             runAsNonRoot: true
@@ -65,7 +66,7 @@ spec:
       containers:
         - name: backstage-backend
           # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-          image: quay.io/janus-idp/backstage-showcase:latest
+          image: quay.io/janus-idp/backstage-showcase:next
           imagePullPolicy: IfNotPresent
           args:
             - "--config"

config/manager/manager.yaml

@@ -76,7 +76,7 @@ spec:
         - name: RELATED_IMAGE_postgresql
           value: quay.io/fedora/postgresql-15:latest
         - name: RELATED_IMAGE_backstage
-          value: quay.io/janus-idp/backstage-showcase:latest
+          value: quay.io/janus-idp/backstage-showcase:next
         image: controller:latest
         name: manager
         securityContext:

examples/bs1.yaml

@@ -2,6 +2,3 @@ apiVersion: rhdh.redhat.com/v1alpha2
 kind: Backstage
 metadata:
   name: bs1
-
-
-

examples/rhdh-cr-with-app-configs.yaml

@@ -47,10 +47,16 @@ data:
     backend:
       auth:
         externalAccess:
-            - type: legacy
-              options:
-                subject: legacy-default-config
-                secret: "${BACKEND_SECRET}"
+          - type: legacy
+            options:
+              subject: legacy-default-config
+              secret: "${BACKEND_SECRET}"
+    auth:
+      environment: development
+      providers:
+        guest:
+          # using the guest user to query the '/api/dynamic-plugins-info/loaded-plugins' endpoint.
+          dangerouslyAllowOutsideDevelopment: true
 ---
 apiVersion: v1
 kind: Secret
@@ -128,7 +134,7 @@ data:
                     initialDelay: { seconds: 15}
       - package: ./dynamic-plugins/dist/backstage-plugin-techdocs-backend-dynamic
         pluginConfig:
-          # Reference documentation http://backstage.io/docs/features/techdocs/configuration
+          # Reference documentation https://backstage.io/docs/features/techdocs/configuration
           # Note: After experimenting with basic setup, use CI/CD to generate docs
           # and an external cloud storage when deploying TechDocs for production use-case.
           # https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-basic-to-recommended-deployment-approach

tests/e2e/e2e_suite_test.go

@@ -223,6 +223,15 @@ func verifyControllerUp(g Gomega, managerPodLabel string) {
 	g.Expect(string(status)).Should(Equal("Running"), fmt.Sprintf("controller pod in %s status", status))
 }
 
+func getPodLogs(ns string, label string) string {
+	cmd := exec.Command(helper.GetPlatformTool(), "logs",
+		"-l", label,
+		"-n", ns,
+	)
+	output, _ := helper.Run(cmd)
+	return string(output)
+}
+
 func uninstallOperator() {
 	switch testMode {
 	case rhdhLatestTestMode, rhdhNextTestMode, rhdhAirgapTestMode:

tests/e2e/e2e_test.go

@@ -15,7 +15,11 @@
 package e2e
 
 import (
+	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
 	"os/exec"
 	"path/filepath"
 	"strconv"
@@ -78,9 +82,44 @@ var _ = Describe("Backstage Operator E2E", func() {
 				crName:     "bs-app-config",
 				additionalApiEndpointTests: []helper.ApiEndpointTest{
 					{
-						Endpoint:               "/api/dynamic-plugins-info/loaded-plugins",
+						Endpoint: "/api/dynamic-plugins-info/loaded-plugins",
+						BearerTokenRetrievalFn: func(baseUrl string) (string, error) { // Authenticated endpoint that does not accept service tokens
+							url := fmt.Sprintf("%s/api/auth/guest/refresh", baseUrl)
+							tr := &http.Transport{
+								TLSClientConfig: &tls.Config{
+									InsecureSkipVerify: true, // #nosec G402 -- test code only, not used in production
+								},
+							}
+							httpClient := &http.Client{Transport: tr}
+							req, err := http.NewRequest("GET", url, nil)
+							if err != nil {
+								return "", fmt.Errorf("error while building request to GET %q: %w", url, err)
+							}
+							req.Header.Add("Accept", "application/json")
+							resp, err := httpClient.Do(req)
+							if err != nil {
+								return "", fmt.Errorf("error while trying to GET %q: %w", url, err)
+							}
+							defer resp.Body.Close()
+							body, err := io.ReadAll(resp.Body)
+							if err != nil {
+								return "", fmt.Errorf("error while trying to read response body from 'GET %q': %w", url, err)
+							}
+							if resp.StatusCode != 200 {
+								return "", fmt.Errorf("expected status code 200, but got %d in response to 'GET %q', body: %s", resp.StatusCode, url, string(body))
+							}
+							var authResponse helper.BackstageAuthRefreshResponse
+							err = json.Unmarshal(body, &authResponse)
+							if err != nil {
+								return "", fmt.Errorf("error while trying to decode response body from 'GET %q': %w", url, err)
+							}
+							return authResponse.BackstageIdentity.Token, nil
+						},
 						ExpectedHttpStatusCode: 200,
 						BodyMatcher: SatisfyAll(
+							ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-quay-dynamic"),
+							ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-regex-dynamic"),
+							ContainSubstring("roadiehq-scaffolder-backend-module-utils-dynamic"),
 							ContainSubstring("backstage-plugin-catalog-backend-module-github-dynamic"),
 							ContainSubstring("backstage-plugin-techdocs-backend-dynamic"),
 							ContainSubstring("backstage-plugin-catalog-backend-module-gitlab-dynamic")),
@@ -187,7 +226,7 @@ var _ = Describe("Backstage Operator E2E", func() {
 })
 
 func ensureRouteIsReachable(ns string, crName string, additionalApiEndpointTests []helper.ApiEndpointTest) {
-	Eventually(helper.VerifyBackstageRoute, time.Minute, time.Second).
+	Eventually(helper.VerifyBackstageRoute, 5*time.Minute, time.Second).
 		WithArguments(ns, crName, additionalApiEndpointTests).
 		Should(Succeed())
 }

tests/e2e/e2e_upgrade_test.go

@@ -16,6 +16,7 @@ package e2e
 
 import (
 	"fmt"
+	"io"
 	"os/exec"
 	"path/filepath"
 	"time"
@@ -49,13 +50,10 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 	When("Previous version of operator is installed and CR is created", func() {
 
 		const managerPodLabel = "control-plane=controller-manager"
+		const crName = "my-backstage-app"
 
 		// 0.1.3 is the version of the operator in the 1.1.x branch
 		var fromDeploymentManifest = filepath.Join(projectDir, "tests", "e2e", "testdata", "backstage-operator-0.1.3.yaml")
-		var (
-			crName = "bs1"
-			crPath = filepath.Join(projectDir, "examples", "bs1.yaml")
-		)
 
 		BeforeEach(func() {
 			if testMode != defaultDeployTestMode {
@@ -71,9 +69,22 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 			Expect(err).ShouldNot(HaveOccurred())
 			EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, time.Second).WithArguments(managerPodLabel).Should(Succeed())
 
-			cmd = exec.Command(helper.GetPlatformTool(), "apply", "-f", crPath, "-n", ns)
+			cmd = exec.Command(helper.GetPlatformTool(), "-n", ns, "create", "-f", "-")
+			stdin, err := cmd.StdinPipe()
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			go func() {
+				defer stdin.Close()
+				_, _ = io.WriteString(stdin, fmt.Sprintf(`
+apiVersion: rhdh.redhat.com/v1alpha1
+kind: Backstage
+metadata:
+  name: my-backstage-app
+  namespace: %s
+`, ns))
+			}()
 			_, err = helper.Run(cmd)
 			Expect(err).ShouldNot(HaveOccurred())
+
 			// Reason is DeployOK in 1.1.x, but was renamed to Deployed in 1.2
 			Eventually(helper.VerifyBackstageCRStatus, time.Minute, time.Second).WithArguments(ns, crName, `"reason":"DeployOK"`).Should(Succeed())
 		})
@@ -89,17 +100,21 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 		It("should successfully reconcile existing CR when upgrading the operator", func() {
 			By("Upgrading the operator", func() {
 				installOperatorWithMakeDeploy(false)
-				EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, time.Second).WithArguments(managerPodLabel).Should(Succeed())
+				EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, 3*time.Second).WithArguments(managerPodLabel).Should(Succeed())
 			})
 
 			By("checking the status of the existing CR")
-			Eventually(helper.VerifyBackstageCRStatus, time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`).Should(Succeed())
+			Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, 3*time.Second).WithArguments(ns, crName, `"reason":"Deployed"`).
+				Should(Succeed(), func() string {
+					return fmt.Sprintf("=== Operator logs ===\n%s\n", getPodLogs(_namespace, managerPodLabel))
+				})
 
 			By("checking the Backstage operand pod")
+			crLabel := fmt.Sprintf("rhdh.redhat.com/app=backstage-%s", crName)
 			Eventually(func(g Gomega) {
 				// Get pod name
 				cmd := exec.Command(helper.GetPlatformTool(), "get",
-					"pods", "-l", fmt.Sprintf("rhdh.redhat.com/app=backstage-%s", crName),
+					"pods", "-l", crLabel,
 					"-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+
 						"{{ \"\\n\" }}{{ end }}{{ end }}",
 					"-n", ns,
@@ -108,7 +123,9 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 				g.Expect(err).ShouldNot(HaveOccurred())
 				podNames := helper.GetNonEmptyLines(string(podOutput))
 				g.Expect(podNames).Should(HaveLen(1), fmt.Sprintf("expected 1 Backstage operand pod(s) running, but got %d", len(podNames)))
-			}, 5*time.Minute, time.Second).Should(Succeed())
+			}, 10*time.Minute, 5*time.Second).Should(Succeed(), func() string {
+				return fmt.Sprintf("=== Operand logs ===\n%s\n", getPodLogs(ns, crLabel))
+			})
 		})
 	})
 

tests/e2e/testdata/backstage-operator-0.1.3.yaml

@@ -739,7 +739,7 @@ data:
                 - name: NPM_CONFIG_USERCONFIG
                   value: /opt/app-root/src/.npmrc.dynamic-plugins
               # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-              image: quay.io/janus-idp/backstage-showcase:latest
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               name: install-dynamic-plugins
               volumeMounts:
@@ -761,7 +761,7 @@ data:
           containers:
             - name: backstage-backend
               # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-              image: quay.io/janus-idp/backstage-showcase:latest
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               args:
                 - "--config"
@@ -949,7 +949,7 @@ spec:
             - name: RELATED_IMAGE_postgresql
               value: quay.io/fedora/postgresql-15:latest
             - name: RELATED_IMAGE_backstage
-              value: quay.io/janus-idp/backstage-showcase:latest
+              value: quay.io/janus-idp/backstage-showcase:next
           # TODO(asoro): Default image is 'quay.io/janus-idp/operator:0.1.3' on 1.1.x,
           # but replaced by the one from RHDH, because the Janus-IDP image expires after 14d if not updated.
           image: quay.io/rhdh/rhdh-rhel9-operator:1.1