ci: Add viewable Matrix test Playwright reports (#1169)

Author: backspace

.github/workflows/ci.yaml

@@ -176,12 +176,14 @@ jobs:
         with:
           junit_files: junit/host.xml
 
-  matrix-client-test-1:
-    name: Matrix Client Tests - Group 1
+  matrix-client-test:
+    name: Matrix Client Tests
     runs-on: ubuntu-latest
-    concurrency:
-      group: matrix-client-test-1-${{ github.head_ref || github.run_id }}
-      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4, 5, 6]
+        shardTotal: [6]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/init
@@ -203,50 +205,90 @@ jobs:
         run: pnpm start:without-matrix &
         working-directory: packages/realm-server
       - name: Run Playwright tests
-        run: pnpm test:group group1
+        run: pnpm test:group ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
         working-directory: packages/matrix
-      - uses: actions/upload-artifact@v3
-        if: always()
+
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
         with:
-          name: playwright-report
-          path: packages/matrix/playwright-report/
-          retention-days: 30
+          name: blob-report-${{ matrix.shardIndex }}
+          path: packages/matrix/blob-report
+          retention-days: 1
 
-  matrix-client-test-2:
-    name: Matrix Client Tests - Group 2
+  matrix-client-merge-reports-and-publish:
+    name: Merge Matrix reports and publish
+    if: always()
+    needs: matrix-client-test
     runs-on: ubuntu-latest
-    concurrency:
-      group: matrix-client-test-2-${{ github.head_ref || github.run_id }}
-      cancel-in-progress: true
+
+    permissions:
+      id-token: write
+      contents: write
+      checks: write
+      statuses: write
+
+    outputs:
+      timestamp: ${{ steps.timestampid.outputs.timestamp }}
+
     steps:
+      - name: Create a timestamp as a directory to store reports in
+        id: timestampid
+        run: echo "timestamp=$(date --utc +%Y%m%d_%H%M%SZ)" >> "$GITHUB_OUTPUT"
       - uses: actions/checkout@v3
       - uses: ./.github/actions/init
-      - name: Install Playwright Browsers
-        run: pnpm exec playwright install --with-deps
-        working-directory: packages/matrix
-      - name: Build boxel-ui
-        run: pnpm build
-        working-directory: packages/boxel-ui/addon
-      - name: Build boxel-motion
-        run: pnpm build
-        working-directory: packages/boxel-motion/addon
-      - name: Build host dist/ for fastboot
-        run: pnpm build
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge blobs into one single report
+        run: pnpm exec playwright merge-reports --reporter html ./all-blob-reports
+
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report--attempt-${{ github.run_attempt }}
+          path: playwright-report
+          retention-days: 14
+
+      - name: Set up env
         env:
-          NODE_OPTIONS: --max_old_space_size=4096
-        working-directory: packages/host
-      - name: Start realm servers
-        run: pnpm start:without-matrix &
-        working-directory: packages/realm-server
-      - name: Run Playwright tests
-        run: pnpm test:group group2
-        working-directory: packages/matrix
-      - uses: actions/upload-artifact@v3
-        if: always()
+          INPUT_ENVIRONMENT: ${{ inputs.environment }}
+        run: |
+          echo "AWS_REGION=us-east-1" >> $GITHUB_ENV
+          echo "AWS_ROLE_ARN=arn:aws:iam::680542703984:role/boxel-matrix-playwright-reports" >> $GITHUB_ENV
+          echo "AWS_S3_BUCKET=cardstack-boxel-matrix-playwright-reports-staging" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
         with:
-          name: playwright-report
-          path: packages/matrix/playwright-report/
-          retention-days: 30
+          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          aws-region: us-east-1
+
+      - name: Publish consolidated report to S3
+        run: aws s3 sync ./playwright-report s3://cardstack-boxel-matrix-playwright-reports-staging/${{ github.head_ref || github.ref_name }}/${{ steps.timestampid.outputs.timestamp }}
+
+      - name: Store Playwright report URL
+        shell: bash
+        run: echo "PLAYWRIGHT_REPORT_URL=https://boxel-matrix-playwright-reports.stack.cards/${{ github.head_ref || github.ref_name }}/${{ steps.timestampid.outputs.timestamp }}/index.html" >> $GITHUB_ENV
+
+      - name: Add status with link to Playwright report
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          REPOSITORY: ${{ github.repository }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          curl \
+            -X POST \
+            -H "Authorization: token $GITHUB_TOKEN" \
+            -H "Accept: application/vnd.github.v3+json" \
+            https://api.github.com/repos/$REPOSITORY/statuses/$HEAD_SHA \
+            -d '{"context":"Matrix Playwright tests report","description":"","target_url":"'"$PLAYWRIGHT_REPORT_URL"'","state":"success"}'
 
   realm-server-test:
     name: Realm Server Tests

package.json

@@ -50,6 +50,7 @@
     "@glint/core": "1.3.0",
     "@glint/environment-ember-loose": "1.3.0",
     "@glint/environment-ember-template-imports": "1.3.0",
+    "@playwright/test": "^1.43.1",
     "@typescript-eslint/eslint-plugin": "^5.17.0",
     "@typescript-eslint/parser": "^5.17.0",
     "ember-cli-htmlbars": "^6.3.0",

packages/matrix/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "license": "MIT",
   "devDependencies": {
-    "@playwright/test": "^1.35.0",
+    "@playwright/test": "^1.43.1",
     "@types/fs-extra": "^9.0.13",
     "@types/jsonwebtoken": "^9.0.5",
     "@types/node": "^18.18.5",
@@ -29,11 +29,11 @@
     "register-test-admin": "MATRIX_IS_ADMIN=TRUE MATRIX_USERNAME=admin MATRIX_PASSWORD=password ts-node --transpileOnly ./scripts/register-test-user.ts",
     "register-test-admin-and-token": "pnpm register-test-admin && ts-node --transpileOnly ./scripts/register-test-token.ts",
     "register-all": "pnpm register-test-admin-and-token && pnpm register-realm-users && pnpm register-bot-user",
-    "test": "./scripts/test.sh all",
+    "test": "./scripts/test.sh",
     "test:group": "./scripts/test.sh",
     "wait": "sleep 10000000",
     "lint": "glint",
-    "clean": "rm -rf ./synapse-data" 
+    "clean": "rm -rf ./synapse-data"
   },
   "volta": {
     "extends": "../../package.json"

packages/matrix/playwright.config.ts

@@ -1,10 +1,4 @@
-import { defineConfig, devices } from '@playwright/test';
-import { readdirSync } from 'fs';
-
-let tests = readdirSync('./tests');
-let middle = Math.floor(tests.length / 2);
-let group1 = tests.slice(0, middle);
-let group2 = tests.slice(middle);
+import { defineConfig } from '@playwright/test';
 
 /**
  * See https://playwright.dev/docs/test-configuration.
@@ -15,35 +9,14 @@ export default defineConfig({
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
   workers: 1,
-  reporter: 'html',
+  reporter: process.env.CI ? 'blob' : 'html',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     baseURL: 'http://localhost:4202/test',
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
     trace: 'on-first-retry',
   },
-
-  projects: [
-    {
-      name: 'all',
-      use: { ...devices['Desktop Chrome'] },
-    },
-    {
-      name: 'group1',
-      testMatch: new RegExp(
-        `.*(${group1.map((i) => i.replace(/\./g, '\\.')).join('|')})`,
-      ),
-      use: { ...devices['Desktop Chrome'] },
-    },
-    {
-      name: 'group2',
-      testMatch: new RegExp(
-        `.*(${group2.map((i) => i.replace(/\./g, '\\.')).join('|')})`,
-      ),
-      use: { ...devices['Desktop Chrome'] },
-    },
-  ],
   // General timeout per test
   timeout: 30000,
 

packages/matrix/scripts/test.sh

@@ -1,8 +1,9 @@
 #! /bin/sh
+shard_flag=${1:+--shard}
 echo "running tests: ${1}"
 start-server-and-test \
   'pnpm run wait' \
   'http-get://localhost:4201/base/fields/boolean-field?acceptHeader=application%2Fvnd.card%2Bjson|http-get://localhost:4202/test/hassan?acceptHeader=application%2Fvnd.card%2Bjson' \
   'pnpm run start:host-pre-built' \
   'http://127.0.0.1:4200' \
-  "pnpm playwright test --project=${1}"
+  "pnpm playwright test ${shard_flag} ${1}"