Cypress visual regression testing

.github/workflows/cypress.yml

@@ -0,0 +1,38 @@
+name: Cypress visual regression tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  cypress-run:
+    runs-on: ubuntu-22.04
+    container:
+      # This must stay in sync with the image used by the test-{site}-visual scripts in package.json.
+      image: cypress/browsers:node-20.14.0-chrome-125.0.6422.141-1-ff-126.0.1-edge-125.0.2535.85-1
+      options: --user 1001
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Cypress run (Ada)
+        uses: cypress-io/github-action@v6
+        with:
+          component: true
+          browser: chrome
+        env:
+          CYPRESS_SITE: ada
+      - name: Cypress run (Physics)
+        uses: cypress-io/github-action@v6
+        with:
+          component: true
+          browser: chrome
+        env:
+          CYPRESS_SITE: phy
+      - name: Upload artifacts
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-diffs
+          path: src/test/**/*.diff.png

.gitignore

@@ -8,6 +8,10 @@ bundle-stats-*.json
 
 # testing
 /coverage
+/cypress/screenshots
+/cypress/downloads
+*.actual.png
+*.diff.png
 
 # production
 /build

config/webpack.config.ada.cypress.js

@@ -0,0 +1,11 @@
+/* eslint-disable */
+const configAda = require('./webpack.config.ada');
+const {merge} = require('webpack-merge');
+
+module.exports = env => {
+    let configAdaCypress = {
+        cache: {type: 'filesystem', name: 'cs'},
+    };
+
+    return merge(configAda({...env, isRenderer: false, prod: false}), configAdaCypress);
+};

config/webpack.config.phy.cypress.js

@@ -0,0 +1,11 @@
+/* eslint-disable */
+const configPhy = require('./webpack.config.physics');
+const {merge} = require('webpack-merge');
+
+module.exports = env => {
+    let configPhyCypress = {
+        cache: {type: 'filesystem', name: 'phy'}
+    };
+
+    return merge(configPhy({...env, isRenderer: false, prod: false}), configPhyCypress);
+};

cypress.config.ts

@@ -0,0 +1,25 @@
+import { defineConfig } from "cypress";
+import { initPlugin } from "@frsource/cypress-plugin-visual-regression-diff/plugins";
+
+const SITE_STRING = process.env.CYPRESS_SITE == 'ada' ? 'ada' : 'phy';
+const UPDATE_BASELINE = process.env.CYPRESS_UPDATE_BASELINE == 'true';
+
+export default defineConfig({
+  component: {
+    devServer: {
+      framework: "react",
+      bundler: "webpack",
+      webpackConfig: require(`./config/webpack.config.${SITE_STRING}.cypress.js`)
+    },
+    indexHtmlFile: `cypress/support/component-index-${SITE_STRING}.html`,
+    supportFile: `cypress/support/component-${SITE_STRING}.tsx`,
+    setupNodeEvents(on, config) {
+      initPlugin(on, config);
+    }
+  },
+  env: {
+    pluginVisualRegressionImagesPath : `{spec_path}/__image_snapshots__/${SITE_STRING}`,
+    pluginVisualRegressionMaxDiffThreshold: 0,
+    pluginVisualRegressionUpdateImages: UPDATE_BASELINE,
+  }
+});

cypress/support/commands.tsx

@@ -0,0 +1,78 @@
+/// <reference types="cypress" />
+// ***********************************************
+// This example commands.ts shows you how to
+// create various custom commands and overwrite
+// existing commands.
+//
+// For more comprehensive examples of custom
+// commands please read more here:
+// https://on.cypress.io/custom-commands
+// ***********************************************
+//
+//
+// -- This is a parent command --
+// Cypress.Commands.add('login', (email, password) => { ... })
+//
+//
+// -- This is a child command --
+// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This will overwrite an existing command --
+// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })
+//
+// declare global {
+//   namespace Cypress {
+//     interface Chainable {
+//       login(email: string, password: string): Chainable<void>
+//       drag(subject: string, options?: Partial<TypeOptions>): Chainable<Element>
+//       dismiss(subject: string, options?: Partial<TypeOptions>): Chainable<Element>
+//       visit(originalFn: CommandOriginalFn, url: string, options: Partial<VisitOptions>): Chainable<Element>
+//     }
+//   }
+// }
+
+import {mount, MountOptions} from 'cypress/react';
+
+// Augment the Cypress namespace to include type definitions for
+// your custom command.
+// Alternatively, can be defined in cypress/support/component.d.ts
+// with a <reference path="./component" /> at the top of your spec.
+declare global {
+    // eslint-disable-next-line @typescript-eslint/no-namespace
+    namespace Cypress {
+        interface Chainable {
+            mountWithStoreAndRouter(component: ReactNode, routes: string[], options: MountOptions): Chainable<Element>;
+        }
+    }
+}
+
+import React, {ReactNode} from "react";
+import {Provider} from "react-redux";
+import {store} from "../../src/app/state";
+import {MemoryRouter} from "react-router";
+
+Cypress.Commands.add('mountWithStoreAndRouter', (component, routes, options) => {
+    mount(
+        <Provider store={store}>
+            <MemoryRouter initialEntries={routes}>
+                {component}
+            </MemoryRouter>
+        </Provider>
+        , options
+    );
+});
+
+import "@frsource/cypress-plugin-visual-regression-diff/dist/support";
+
+// Skip visual regression tests in interactive mode - the results are not consistent with headless.
+// It may be useful to comment this out when debugging tests locally, but don't commit the snapshots.
+if (Cypress.config('isInteractive')) {
+    Cypress.Commands.add('matchImage', () => {
+        cy.log('Skipping snapshot 👀');
+    });
+}

cypress/support/component-ada.tsx

@@ -0,0 +1,26 @@
+// ***********************************************************
+// This example support/component-ada.ts is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands';
+
+// Import styles
+import '../../src/scss/cs/isaac.scss';
+
+// Start Mock Service Worker - we use this instead of Cypress API mocking
+import { worker } from '../../src/mocks/browser';
+Cypress.on('test:before:run:async', async () => {
+  await worker.start();
+});

cypress/support/component-index-ada.html

@@ -0,0 +1,34 @@
+<!DOCTYPE html>
+<!-- Note: this needs to remain synchronised with index-ada.html -->
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="/assets/cs/favicon/favicon-196x196.png" sizes="196x196" />
+    <link rel="icon" href="/assets/cs/favicon/favicon-96x96.png"   sizes="96x96" />
+    <link rel="icon" href="/assets/cs/favicon/favicon-32x32.png"   sizes="32x32" />
+    <link rel="icon" href="/assets/cs/favicon/favicon-16x16.png"   sizes="16x16" />
+    <link rel="shortcut icon" href="/assets/cs/favicon/favicon.ico" />
+    <link rel="apple-touch-icon" href="/assets/cs/favicon/apple-touch-icon-180x180.png" sizes="180x180">
+    <link rel="apple-touch-icon" href="/assets/cs/favicon/apple-touch-icon-76x76.png"   sizes="76x76">
+    <link rel="apple-touch-icon" href="/assets/cs/favicon/apple-touch-icon-precomposed.png">
+    <link rel="preload" href="/assets/cs/fonts/poppins-regular.woff2" as="font" crossorigin="anonymous" />
+    <link rel="preload" href="/assets/cs/fonts/poppins-semibold.woff2" as="font" crossorigin="anonymous" />
+    <link rel="preload" as="image" href="/assets/common/logos/ada_logo_3-stack_aqua_white_text.svg">
+    <link rel="preload" as="image" href="/assets/cs/decor/ada_pie_pink.png">
+    <link rel="preload" as="image" href="/assets/cs/decor/ada_pie_turquoise.png">
+    <link rel="manifest" href="/manifest-ada.json" />
+    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no, shrink-to-fit=no" />
+    <meta name="theme-color" content="#000000" />
+    <meta name="application-name" content="Ada Computer Science" />
+    <meta name="description" content="Join Ada Computer Science, the free, online computer science programme for students and teachers. Learn with our computer science resources and questions."  data-react-helmet="true" />
+    <meta property="og:site_name" content="Ada Computer Science" />
+    <meta property="og:title" content="Ada Computer Science" data-react-helmet="true" />
+    <meta property="og:type" content="website" />
+    <meta property="og:description" content="Join Ada Computer Science, the free, online computer science programme for students and teachers. Learn with our computer science resources and questions." data-react-helmet="true" />
+    <meta property="og:image" content="https://cdn.adacomputerscience.org/ada/logos/ada-logo-aqua-500px.png">
+    <title>Ada Computer Science</title>
+  </head>
+  <body>
+    <div data-cy-root></div>
+  </body>
+</html>

cypress/support/component-index-phy.html

@@ -0,0 +1,34 @@
+<!DOCTYPE html>
+<!-- Note: this needs to remain synchronised with index-phy.html -->
+<html>
+<head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="/assets/phy/favicon/favicon-196x196.png" sizes="196x196" />
+    <link rel="icon" href="/assets/phy/favicon/favicon-96x96.png"   sizes="96x96" />
+    <link rel="icon" href="/assets/phy/favicon/favicon-32x32.png"   sizes="32x32" />
+    <link rel="apple-touch-icon" href="/assets/phy/favicon/apple-touch-icon-180x180.png" sizes="180x180">
+    <link rel="apple-touch-icon" href="/assets/phy/favicon/apple-touch-icon-76x76.png"   sizes="76x76">
+    <link rel="apple-touch-icon" href="/assets/phy/favicon/apple-touch-icon-precomposed.png">
+    <link rel="manifest" href="/manifest-phy.json" />
+    <link rel="preload" href="/assets/phy/fonts/exo2-semibold-webfont.woff2" as="font" crossorigin="anonymous" />
+    <link rel="preload" href="/assets/phy/fonts/exo2-regular-webfont.woff2" as="font" crossorigin="anonymous" />
+    <link rel="preload" href="/assets/phy/fonts/exo2-bold-webfont.woff2" as="font" crossorigin="anonymous" />
+    <link rel="preload" href="/assets/phy/fonts/exo2-medium-webfont.woff2" as="font" crossorigin="anonymous" />
+    <link rel="preload" as="image" href="/assets/phy/logo.svg">
+    <link rel="preload" as="image" href="/assets/phy/line.svg">
+    <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no, shrink-to-fit=no" />
+    <meta name="theme-color" content="#000000" />
+    <meta name="application-name" content="Isaac Physics" />
+    <meta name="description" content="Isaac Physics is a project designed to offer support and activities in physics problem solving to teachers and students from GCSE level through to university." data-react-helmet="true" />
+    <meta property="og:site_name" content="Isaac Physics" />
+    <meta property="og:title" content="Isaac Physics" data-react-helmet="true" />
+    <meta property="og:type" content="website" />
+    <meta property="og:description" content="Isaac Physics is a project designed to offer support and activities in physics problem solving to teachers and students from GCSE level through to university." data-react-helmet="true" />
+    <meta property="og:image" content="https://cdn.isaacphysics.org/isaac/logos/isaacphysics-favicon-500px.png" />
+    <meta property="fb:app_id" content="760382960667256" />
+    <title>Isaac Physics</title>
+</head>
+<body>
+<div data-cy-root></div>
+</body>
+</html>

cypress/support/component-phy.tsx

@@ -0,0 +1,26 @@
+// ***********************************************************
+// This example support/component-ada.ts is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands';
+
+// Import styles
+import '../../src/scss/phy/isaac.scss';
+
+// Start Mock Service Worker - we use this instead of Cypress API mocking
+import { worker } from '../../src/mocks/browser';
+Cypress.on('test:before:run:async', async () => {
+  await worker.start();
+});

docker-entrypoint-vrt.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+yarn install --frozen-lockfile
+yarn cypress run --component --browser chrome

package.json

@@ -59,8 +59,14 @@
     "start-phy": "webpack-dev-server --hot --port 8004 --history-api-fallback --config config/webpack.config.physics.js --allowed-hosts=true",
     "start-ada": "webpack-dev-server --hot --port 8003 --history-api-fallback --config config/webpack.config.ada.js --allowed-hosts=true",
     "test": "yarn run test-phy && yarn run test-ada",
+    "test-visual": "yarn run test-phy-visual && yarn run test-ada-visual",
+    "test-visual-update-baselines": "yarn run test-phy-visual-update-baseline && yarn run test-ada-visual-update-baseline",
     "test-phy": "jest --config config/jest/jest.config.physics.js",
+    "test-phy-visual": "python3 vrt-in-docker.py phy",
+    "test-phy-visual-update-baseline": "python3 vrt-in-docker.py phy --update-baselines",
     "test-ada": "jest --config config/jest/jest.config.ada.js",
+    "test-ada-visual": "python3 vrt-in-docker.py ada",
+    "test-ada-visual-update-baseline": "python3 vrt-in-docker.py ada --update-baselines",
     "lint": "eslint --ext .ts,.tsx,.js,.jsx src/",
     "build-stats-phy": "webpack --env prod --config config/webpack.config.physics.js --profile --json > bundle-stats-phy.json",
     "analyse-bundle-phy": "webpack-bundle-analyzer bundle-stats-phy.json build-physics",
@@ -125,6 +131,7 @@
     "@babel/preset-env": "^7.24.3",
     "@babel/preset-react": "^7.24.1",
     "@babel/preset-typescript": "^7.24.1",
+    "@frsource/cypress-plugin-visual-regression-diff": "^3.3.10",
     "@testing-library/dom": "^9.3.4",
     "@testing-library/jest-dom": "^6.4.2",
     "@testing-library/react": "^12.1.5",
@@ -137,6 +144,7 @@
     "@types/katex": "^0.16.7",
     "@types/leaflet": "^1.7.9",
     "@types/lodash": "^4.17.0",
+    "@types/node": "^20.12.12",
     "@types/object-hash": "^3.0.6",
     "@types/qrcode": "^1.5.5",
     "@types/react-beautiful-dnd": "^13.1.8",
@@ -159,6 +167,7 @@
     "clean-webpack-plugin": "^4.0.0",
     "copy-webpack-plugin": "^10.2.4",
     "css-loader": "^6.10.0",
+    "cypress": "13.6.4",
     "dotenv": "^10.0.0",
     "eslint": "^8.57.0",
     "eslint-plugin-jsx-a11y": "^6.8.0",

src/app/components/handlers/IsaacSpinner.tsx

@@ -14,7 +14,7 @@ export interface IsaacSpinnerProps {
 export const IsaacSpinner = ({size = "md", className, color = "primary", inline = false, displayText = "Loading..."} : IsaacSpinnerProps) => {
     const contents = <>
         <img style={siteSpecific({width: "auto", height: "5.5rem"}, {})} className={classNames(`isaac-spinner-${size}`, className)} alt="" src={siteSpecific("/assets/phy/isaac-phy-apple-grow.svg", "/assets/cs/icons/loading-spinner-placeholder.svg")}/>
-        <span className="sr-only">{displayText}</span>
+        <span data-testid={"loading"} className="sr-only">{displayText}</span>
     </>;
     return inline
         ? <span role="status">{contents}</span>