Skip to content

Commit 054f867

Browse files
mwtrewmwtrew
authored
Cypress visual regression testing (#998)
* Partial cypress configuration * Add basic Cypress component testing config * Configure visual regression testing * Support site-specific visual testing * Add dockerised cypress run * Run Cypress visual regression tests on CI * Refactor utils to resolve Node dependency issue * Remove -it from Docker commands * Use Cypress workflow definition * Remove unused import * Add artifact upload step to Cypress action * Run artifact upload if VRT fails * Force Cypress to wait for page load * Update baselines * Downgrade Cypress and regenerate baselines Latest Cypress doesn't play well with Docker on M1, see cypress-io/cypress#29095 * Fix Ada run configuration * Fix Ada run configuration * Fix Ada run configuration * Use Chrome for visual regression tests * Add new command to mount with store and router * Revert changes to RTK renderTestEnvironment * Cache dependencies when running locally * Add groups VRT, use fixed future date for test data * Remove unused imports * Remove sample Cypress fixtures file * Add Set Assignments VRTs * Add Cypress downloads to gitignore * Cache webpack output to speed up local dev and VRTs Also fixes line endings of webpack.config.common.js * Revert common webpack config, create derived Cypress configs * Remove unused imports * Add Windows support for Cypress VRTs * Fix site and update baseline vars, remove unused imports * Rename docker entrypoint * Use python3 in package.json call to suit MacOS & WSL * Restore common webpack config * Update VRT baselines
1 parent 9245b32 commit 054f867

38 files changed

+1442
-139
lines changed

‎.github/workflows/cypress.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Cypress visual regression tests
2+
3+
on:
4+
push:
5+
branches: [ master ]
6+
pull_request:
7+
branches: [ master ]
8+
9+
jobs:
10+
cypress-run:
11+
runs-on: ubuntu-22.04
12+
container:
13+
# This must stay in sync with the image used by the test-{site}-visual scripts in package.json.
14+
image: cypress/browsers:node-20.14.0-chrome-125.0.6422.141-1-ff-126.0.1-edge-125.0.2535.85-1
15+
options: --user 1001
16+
steps:
17+
- name: Checkout
18+
uses: actions/checkout@v4
19+
- name: Cypress run (Ada)
20+
uses: cypress-io/github-action@v6
21+
with:
22+
component: true
23+
browser: chrome
24+
env:
25+
CYPRESS_SITE: ada
26+
- name: Cypress run (Physics)
27+
uses: cypress-io/github-action@v6
28+
with:
29+
component: true
30+
browser: chrome
31+
env:
32+
CYPRESS_SITE: phy
33+
- name: Upload artifacts
34+
if: ${{ failure() }}
35+
uses: actions/upload-artifact@v4
36+
with:
37+
name: visual-diffs
38+
path: src/test/**/*.diff.png

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ bundle-stats-*.json
88

99
# testing
1010
/coverage
11+
/cypress/screenshots
12+
/cypress/downloads
13+
*.actual.png
14+
*.diff.png
1115

1216
# production
1317
/build

‎config/webpack.config.ada.cypress.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/* eslint-disable */
2+
const configAda = require('./webpack.config.ada');
3+
const {merge} = require('webpack-merge');
4+
5+
module.exports = env => {
6+
let configAdaCypress = {
7+
cache: {type: 'filesystem', name: 'cs'},
8+
};
9+
10+
return merge(configAda({...env, isRenderer: false, prod: false}), configAdaCypress);
11+
};

‎config/webpack.config.phy.cypress.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/* eslint-disable */
2+
const configPhy = require('./webpack.config.physics');
3+
const {merge} = require('webpack-merge');
4+
5+
module.exports = env => {
6+
let configPhyCypress = {
7+
cache: {type: 'filesystem', name: 'phy'}
8+
};
9+
10+
return merge(configPhy({...env, isRenderer: false, prod: false}), configPhyCypress);
11+
};

‎cypress.config.ts

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import { defineConfig } from "cypress";
2+
import { initPlugin } from "@frsource/cypress-plugin-visual-regression-diff/plugins";
3+
4+
const SITE_STRING = process.env.CYPRESS_SITE == 'ada' ? 'ada' : 'phy';
5+
const UPDATE_BASELINE = process.env.CYPRESS_UPDATE_BASELINE == 'true';
6+
7+
export default defineConfig({
8+
component: {
9+
devServer: {
10+
framework: "react",
11+
bundler: "webpack",
12+
webpackConfig: require(`./config/webpack.config.${SITE_STRING}.cypress.js`)
13+
},
14+
indexHtmlFile: `cypress/support/component-index-${SITE_STRING}.html`,
15+
supportFile: `cypress/support/component-${SITE_STRING}.tsx`,
16+
setupNodeEvents(on, config) {
17+
initPlugin(on, config);
18+
}
19+
},
20+
env: {
21+
pluginVisualRegressionImagesPath : `{spec_path}/__image_snapshots__/${SITE_STRING}`,
22+
pluginVisualRegressionMaxDiffThreshold: 0,
23+
pluginVisualRegressionUpdateImages: UPDATE_BASELINE,
24+
}
25+
});

‎cypress/support/commands.tsx

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/// <reference types="cypress" />
2+
// ***********************************************
3+
// This example commands.ts shows you how to
4+
// create various custom commands and overwrite
5+
// existing commands.
6+
//
7+
// For more comprehensive examples of custom
8+
// commands please read more here:
9+
// https://on.cypress.io/custom-commands
10+
// ***********************************************
11+
//
12+
//
13+
// -- This is a parent command --
14+
// Cypress.Commands.add('login', (email, password) => { ... })
15+
//
16+
//
17+
// -- This is a child command --
18+
// Cypress.Commands.add('drag', { prevSubject: 'element'}, (subject, options) => { ... })
19+
//
20+
//
21+
// -- This is a dual command --
22+
// Cypress.Commands.add('dismiss', { prevSubject: 'optional'}, (subject, options) => { ... })
23+
//
24+
//
25+
// -- This will overwrite an existing command --
26+
// Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... })
27+
//
28+
// declare global {
29+
// namespace Cypress {
30+
// interface Chainable {
31+
// login(email: string, password: string): Chainable<void>
32+
// drag(subject: string, options?: Partial<TypeOptions>): Chainable<Element>
33+
// dismiss(subject: string, options?: Partial<TypeOptions>): Chainable<Element>
34+
// visit(originalFn: CommandOriginalFn, url: string, options: Partial<VisitOptions>): Chainable<Element>
35+
// }
36+
// }
37+
// }
38+
39+
import {mount, MountOptions} from 'cypress/react';
40+
41+
// Augment the Cypress namespace to include type definitions for
42+
// your custom command.
43+
// Alternatively, can be defined in cypress/support/component.d.ts
44+
// with a <reference path="./component" /> at the top of your spec.
45+
declare global {
46+
// eslint-disable-next-line @typescript-eslint/no-namespace
47+
namespace Cypress {
48+
interface Chainable {
49+
mountWithStoreAndRouter(component: ReactNode, routes: string[], options: MountOptions): Chainable<Element>;
50+
}
51+
}
52+
}
53+
54+
import React, {ReactNode} from "react";
55+
import {Provider} from "react-redux";
56+
import {store} from "../../src/app/state";
57+
import {MemoryRouter} from "react-router";
58+
59+
Cypress.Commands.add('mountWithStoreAndRouter', (component, routes, options) => {
60+
mount(
61+
<Provider store={store}>
62+
<MemoryRouter initialEntries={routes}>
63+
{component}
64+
</MemoryRouter>
65+
</Provider>
66+
, options
67+
);
68+
});
69+
70+
import "@frsource/cypress-plugin-visual-regression-diff/dist/support";
71+
72+
// Skip visual regression tests in interactive mode - the results are not consistent with headless.
73+
// It may be useful to comment this out when debugging tests locally, but don't commit the snapshots.
74+
if (Cypress.config('isInteractive')) {
75+
Cypress.Commands.add('matchImage', () => {
76+
cy.log('Skipping snapshot 👀');
77+
});
78+
}

‎cypress/support/component-ada.tsx

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// ***********************************************************
2+
// This example support/component-ada.ts is processed and
3+
// loaded automatically before your test files.
4+
//
5+
// This is a great place to put global configuration and
6+
// behavior that modifies Cypress.
7+
//
8+
// You can change the location of this file or turn off
9+
// automatically serving support files with the
10+
// 'supportFile' configuration option.
11+
//
12+
// You can read more here:
13+
// https://on.cypress.io/configuration
14+
// ***********************************************************
15+
16+
// Import commands.js using ES2015 syntax:
17+
import './commands';
18+
19+
// Import styles
20+
import '../../src/scss/cs/isaac.scss';
21+
22+
// Start Mock Service Worker - we use this instead of Cypress API mocking
23+
import { worker } from '../../src/mocks/browser';
24+
Cypress.on('test:before:run:async', async () => {
25+
await worker.start();
26+
});

‎cypress/support/component-index-ada.html

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<!DOCTYPE html>
2+
<!-- Note: this needs to remain synchronised with index-ada.html -->
3+
<html>
4+
<head>
5+
<meta charset="utf-8" />
6+
<link rel="icon" href="/assets/cs/favicon/favicon-196x196.png" sizes="196x196" />
7+
<link rel="icon" href="/assets/cs/favicon/favicon-96x96.png" sizes="96x96" />
8+
<link rel="icon" href="/assets/cs/favicon/favicon-32x32.png" sizes="32x32" />
9+
<link rel="icon" href="/assets/cs/favicon/favicon-16x16.png" sizes="16x16" />
10+
<link rel="shortcut icon" href="/assets/cs/favicon/favicon.ico" />
11+
<link rel="apple-touch-icon" href="/assets/cs/favicon/apple-touch-icon-180x180.png" sizes="180x180">
12+
<link rel="apple-touch-icon" href="/assets/cs/favicon/apple-touch-icon-76x76.png" sizes="76x76">
13+
<link rel="apple-touch-icon" href="/assets/cs/favicon/apple-touch-icon-precomposed.png">
14+
<link rel="preload" href="/assets/cs/fonts/poppins-regular.woff2" as="font" crossorigin="anonymous" />
15+
<link rel="preload" href="/assets/cs/fonts/poppins-semibold.woff2" as="font" crossorigin="anonymous" />
16+
<link rel="preload" as="image" href="/assets/common/logos/ada_logo_3-stack_aqua_white_text.svg">
17+
<link rel="preload" as="image" href="/assets/cs/decor/ada_pie_pink.png">
18+
<link rel="preload" as="image" href="/assets/cs/decor/ada_pie_turquoise.png">
19+
<link rel="manifest" href="/manifest-ada.json" />
20+
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no, shrink-to-fit=no" />
21+
<meta name="theme-color" content="#000000" />
22+
<meta name="application-name" content="Ada Computer Science" />
23+
<meta name="description" content="Join Ada Computer Science, the free, online computer science programme for students and teachers. Learn with our computer science resources and questions." data-react-helmet="true" />
24+
<meta property="og:site_name" content="Ada Computer Science" />
25+
<meta property="og:title" content="Ada Computer Science" data-react-helmet="true" />
26+
<meta property="og:type" content="website" />
27+
<meta property="og:description" content="Join Ada Computer Science, the free, online computer science programme for students and teachers. Learn with our computer science resources and questions." data-react-helmet="true" />
28+
<meta property="og:image" content="https://cdn.adacomputerscience.org/ada/logos/ada-logo-aqua-500px.png">
29+
<title>Ada Computer Science</title>
30+
</head>
31+
<body>
32+
<div data-cy-root></div>
33+
</body>
34+
</html>

‎cypress/support/component-index-phy.html

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<!DOCTYPE html>
2+
<!-- Note: this needs to remain synchronised with index-phy.html -->
3+
<html>
4+
<head>
5+
<meta charset="utf-8" />
6+
<link rel="icon" href="/assets/phy/favicon/favicon-196x196.png" sizes="196x196" />
7+
<link rel="icon" href="/assets/phy/favicon/favicon-96x96.png" sizes="96x96" />
8+
<link rel="icon" href="/assets/phy/favicon/favicon-32x32.png" sizes="32x32" />
9+
<link rel="apple-touch-icon" href="/assets/phy/favicon/apple-touch-icon-180x180.png" sizes="180x180">
10+
<link rel="apple-touch-icon" href="/assets/phy/favicon/apple-touch-icon-76x76.png" sizes="76x76">
11+
<link rel="apple-touch-icon" href="/assets/phy/favicon/apple-touch-icon-precomposed.png">
12+
<link rel="manifest" href="/manifest-phy.json" />
13+
<link rel="preload" href="/assets/phy/fonts/exo2-semibold-webfont.woff2" as="font" crossorigin="anonymous" />
14+
<link rel="preload" href="/assets/phy/fonts/exo2-regular-webfont.woff2" as="font" crossorigin="anonymous" />
15+
<link rel="preload" href="/assets/phy/fonts/exo2-bold-webfont.woff2" as="font" crossorigin="anonymous" />
16+
<link rel="preload" href="/assets/phy/fonts/exo2-medium-webfont.woff2" as="font" crossorigin="anonymous" />
17+
<link rel="preload" as="image" href="/assets/phy/logo.svg">
18+
<link rel="preload" as="image" href="/assets/phy/line.svg">
19+
<meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no, shrink-to-fit=no" />
20+
<meta name="theme-color" content="#000000" />
21+
<meta name="application-name" content="Isaac Physics" />
22+
<meta name="description" content="Isaac Physics is a project designed to offer support and activities in physics problem solving to teachers and students from GCSE level through to university." data-react-helmet="true" />
23+
<meta property="og:site_name" content="Isaac Physics" />
24+
<meta property="og:title" content="Isaac Physics" data-react-helmet="true" />
25+
<meta property="og:type" content="website" />
26+
<meta property="og:description" content="Isaac Physics is a project designed to offer support and activities in physics problem solving to teachers and students from GCSE level through to university." data-react-helmet="true" />
27+
<meta property="og:image" content="https://cdn.isaacphysics.org/isaac/logos/isaacphysics-favicon-500px.png" />
28+
<meta property="fb:app_id" content="760382960667256" />
29+
<title>Isaac Physics</title>
30+
</head>
31+
<body>
32+
<div data-cy-root></div>
33+
</body>
34+
</html>

‎cypress/support/component-phy.tsx

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// ***********************************************************
2+
// This example support/component-ada.ts is processed and
3+
// loaded automatically before your test files.
4+
//
5+
// This is a great place to put global configuration and
6+
// behavior that modifies Cypress.
7+
//
8+
// You can change the location of this file or turn off
9+
// automatically serving support files with the
10+
// 'supportFile' configuration option.
11+
//
12+
// You can read more here:
13+
// https://on.cypress.io/configuration
14+
// ***********************************************************
15+
16+
// Import commands.js using ES2015 syntax:
17+
import './commands';
18+
19+
// Import styles
20+
import '../../src/scss/phy/isaac.scss';
21+
22+
// Start Mock Service Worker - we use this instead of Cypress API mocking
23+
import { worker } from '../../src/mocks/browser';
24+
Cypress.on('test:before:run:async', async () => {
25+
await worker.start();
26+
});

‎docker-entrypoint-vrt.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
yarn install --frozen-lockfile
4+
yarn cypress run --component --browser chrome

‎package.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,14 @@
5959
"start-phy": "webpack-dev-server --hot --port 8004 --history-api-fallback --config config/webpack.config.physics.js --allowed-hosts=true",
6060
"start-ada": "webpack-dev-server --hot --port 8003 --history-api-fallback --config config/webpack.config.ada.js --allowed-hosts=true",
6161
"test": "yarn run test-phy && yarn run test-ada",
62+
"test-visual": "yarn run test-phy-visual && yarn run test-ada-visual",
63+
"test-visual-update-baselines": "yarn run test-phy-visual-update-baseline && yarn run test-ada-visual-update-baseline",
6264
"test-phy": "jest --config config/jest/jest.config.physics.js",
65+
"test-phy-visual": "python3 vrt-in-docker.py phy",
66+
"test-phy-visual-update-baseline": "python3 vrt-in-docker.py phy --update-baselines",
6367
"test-ada": "jest --config config/jest/jest.config.ada.js",
68+
"test-ada-visual": "python3 vrt-in-docker.py ada",
69+
"test-ada-visual-update-baseline": "python3 vrt-in-docker.py ada --update-baselines",
6470
"lint": "eslint --ext .ts,.tsx,.js,.jsx src/",
6571
"build-stats-phy": "webpack --env prod --config config/webpack.config.physics.js --profile --json > bundle-stats-phy.json",
6672
"analyse-bundle-phy": "webpack-bundle-analyzer bundle-stats-phy.json build-physics",
@@ -125,6 +131,7 @@
125131
"@babel/preset-env": "^7.24.3",
126132
"@babel/preset-react": "^7.24.1",
127133
"@babel/preset-typescript": "^7.24.1",
134+
"@frsource/cypress-plugin-visual-regression-diff": "^3.3.10",
128135
"@testing-library/dom": "^9.3.4",
129136
"@testing-library/jest-dom": "^6.4.2",
130137
"@testing-library/react": "^12.1.5",
@@ -137,6 +144,7 @@
137144
"@types/katex": "^0.16.7",
138145
"@types/leaflet": "^1.7.9",
139146
"@types/lodash": "^4.17.0",
147+
"@types/node": "^20.12.12",
140148
"@types/object-hash": "^3.0.6",
141149
"@types/qrcode": "^1.5.5",
142150
"@types/react-beautiful-dnd": "^13.1.8",
@@ -159,6 +167,7 @@
159167
"clean-webpack-plugin": "^4.0.0",
160168
"copy-webpack-plugin": "^10.2.4",
161169
"css-loader": "^6.10.0",
170+
"cypress": "13.6.4",
162171
"dotenv": "^10.0.0",
163172
"eslint": "^8.57.0",
164173
"eslint-plugin-jsx-a11y": "^6.8.0",

‎src/app/components/handlers/IsaacSpinner.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ export interface IsaacSpinnerProps {
1414
export const IsaacSpinner = ({size = "md", className, color = "primary", inline = false, displayText = "Loading..."} : IsaacSpinnerProps) => {
1515
const contents = <>
1616
<img style={siteSpecific({width: "auto", height: "5.5rem"}, {})} className={classNames(`isaac-spinner-${size}`, className)} alt="" src={siteSpecific("/assets/phy/isaac-phy-apple-grow.svg", "/assets/cs/icons/loading-spinner-placeholder.svg")}/>
17-
<span className="sr-only">{displayText}</span>
17+
<span data-testid={"loading"} className="sr-only">{displayText}</span>
1818
</>;
1919
return inline
2020
? <span role="status">{contents}</span>

0 commit comments

Comments
 (0)