Skip to content

Commit aace5a9

Browse files
gingerwizardgingerwizard
committed
visual tests
1 parent a9c14ad commit aace5a9

File tree

7 files changed

+247
-6
lines changed

7 files changed

+247
-6
lines changed

‎.github/workflows/test-preview.yml

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Playwright + Argos Tests
2+
3+
on:
4+
# Trigger on deployment event
5+
deployment_status:
6+
workflow_dispatch:
7+
# Trigger when a label is added to a PR
8+
pull_request:
9+
types:
10+
- labeled
11+
12+
jobs:
13+
test:
14+
# Run tests only if the deployment is successful
15+
if: |
16+
(
17+
github.event_name == 'deployment_status' &&
18+
github.event.deployment_status.state == 'success'
19+
) || (
20+
github.event_name == 'pull_request' &&
21+
github.event.action == 'labeled' &&
22+
github.event.label.name == 'visual-test'
23+
)
24+
runs-on: ubuntu-latest
25+
26+
steps:
27+
28+
# Step 1: Check out the repository
29+
- name: Check out docs repo
30+
uses: actions/checkout@v4
31+
with:
32+
repository: ClickHouse/clickhouse-docs
33+
path: ./
34+
35+
# Step 2: Log url
36+
- name: Log BASE_URL
37+
run: echo "BASE_URL=${{ github.event.deployment_status.environment_url }}"
38+
env:
39+
BASE_URL: ${{ github.event.deployment_status.environment_url }}
40+
41+
# Step 3: Setup node
42+
- name: Set up Node.js
43+
uses: actions/setup-node@v4
44+
with:
45+
node-version: '20.18.0'
46+
cache: 'yarn'
47+
48+
# Step 4: Install dependencies and build the site (needed for sitemap.xml)
49+
- name: Install and build
50+
run: |
51+
export NODE_OPTIONS="--max_old_space_size=4096"
52+
npm install -g yarn
53+
yarn install
54+
55+
- name: Configure Git safe.directory
56+
run: |
57+
git config --global --add safe.directory /workspace
58+
# Step 3: Run Playwright tests in Docker
59+
- name: Run Playwright tests
60+
run: |
61+
docker run --rm \
62+
-v ${{ github.workspace }}:/workspace \
63+
-w /workspace \
64+
-e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \
65+
-e BASE_URL=${{ github.event.deployment_status.environment_url }} \
66+
-e ARGOS_BRANCH="${{ github.event.deployment_status.environment == 'Production' && 'main' || github.ref_name }}" \
67+
-e CI=true \
68+
-e ARGOS_TOKEN=${{ secrets.ARGOS_TOKEN }} \
69+
mcr.microsoft.com/playwright:v1.49.1-noble \
70+
npm exec -- playwright test

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,7 @@ FormatFactorySettings.h
4141
Settings.cpp
4242

4343
.vscode
44+
45+
.vscode
46+
test-results/**
47+
tests/screenshot.spec.ts-snapshots/**

‎package.json

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@
2222
"write-heading-ids": "docusaurus write-heading-ids"
2323
},
2424
"dependencies": {
25-
"@docusaurus/core": "2.3.1",
26-
"@docusaurus/plugin-client-redirects": "2.3.1",
27-
"@docusaurus/preset-classic": "2.3.1",
28-
"@docusaurus/theme-mermaid": "2.3.1",
25+
"@docusaurus/core": "2.4.3",
26+
"@docusaurus/plugin-client-redirects": "2.4.3",
27+
"@docusaurus/preset-classic": "2.4.3",
28+
"@docusaurus/theme-mermaid": "2.4.3",
2929
"@mdx-js/react": "^1.6.22",
3030
"@radix-ui/react-navigation-menu": "^1.1.4",
3131
"axios": "^1.7.9",
@@ -44,7 +44,11 @@
4444
"sass": "^1.82.0"
4545
},
4646
"devDependencies": {
47-
"@docusaurus/module-type-aliases": "3.6.3"
47+
"@argos-ci/cli": "^2.5.3",
48+
"@argos-ci/playwright": "^3.9.4",
49+
"@docusaurus/module-type-aliases": "3.6.3",
50+
"@playwright/test": "^1.49.1",
51+
"cheerio": "^1.0.0"
4852
},
4953
"browserslist": {
5054
"production": [
@@ -59,6 +63,6 @@
5963
]
6064
},
6165
"engines": {
62-
"node": ">=16.14"
66+
"node": ">=20.18"
6367
}
6468
}

‎playwright.config.ts

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import {devices} from '@playwright/test';
2+
import type {PlaywrightTestConfig} from '@playwright/test';
3+
4+
const isCI = !!process.env.CI; // Check if running in CI
5+
const baseURL = isCI ? process.env.BASE_URL : "http://localhost:3000";
6+
7+
8+
const config: PlaywrightTestConfig = {
9+
fullyParallel: true,
10+
webServer: {
11+
port: 3000,
12+
command: 'yarn docusaurus serve',
13+
},
14+
projects: [
15+
{
16+
name: 'chromium',
17+
use: {
18+
...devices['Desktop Chrome'],
19+
},
20+
},
21+
],
22+
reporter: [
23+
// Use "dot" reporter on CI, "list" otherwise (Playwright default).
24+
process.env.CI ? ["dot"] : ["list"],
25+
// Add Argos reporter.
26+
[
27+
"@argos-ci/playwright/reporter",
28+
{
29+
// Upload to Argos on CI only.
30+
uploadToArgos: isCI,
31+
32+
// Set your Argos token.
33+
token: process.env.ARGOS_TOKEN,
34+
},
35+
],
36+
],
37+
timeout: 1200000,
38+
use: {
39+
// On CI, we will set `BASE_URL` from Vercel preview URL
40+
baseURL: baseURL,
41+
extraHTTPHeaders: {
42+
// Hide Vercel Toolbar in tests
43+
"x-vercel-skip-toolbar": "0",
44+
},
45+
},
46+
47+
};
48+
49+
export default config;

‎tests/screenshot.css

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
We need to hide some elements in Argos/Playwright screenshots
3+
Those elements are source of flakiness due to nondeterministic rendering
4+
They don't consistently render exactly the same across CI runs
5+
*/
6+
7+
/******* DOCUSAURUS GLOBAL / THEME *******/
8+
9+
/* Iframes can load lazily */
10+
iframe,
11+
/* Avatar images can be flaky due to using external sources: GitHub/Unavatar */
12+
.avatar__photo,
13+
/* Gifs load lazily and are animated */
14+
img[src$='.gif'],
15+
/* Algolia Keyboard shortcuts appear with a little delay */
16+
.DocSearch-Button-Keys > kbd,
17+
/* The live playground preview can often display dates/counters */
18+
[class*='playgroundPreview'] {
19+
visibility: hidden;
20+
}
21+
22+
/*
23+
Different docs last-update dates can alter layout
24+
"visibility: hidden" is not enough
25+
*/
26+
.theme-last-updated {
27+
display: none;
28+
}
29+
30+
/*
31+
Mermaid diagrams are rendered client-side and produce layout shifts
32+
*/
33+
.docusaurus-mermaid-container {
34+
display: none;
35+
}

‎tests/screenshot.spec.ts

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import * as fs from 'fs';
2+
import { argosScreenshot } from '@argos-ci/playwright';
3+
import { test } from '@playwright/test';
4+
import axios from 'axios';
5+
import { extractSitemapPathnames, pathnameToArgosName } from './utils';
6+
7+
// Constants
8+
const siteUrl = process.env.CI ? process.env.BASE_URL : 'http://localhost:3000';
9+
const sitemapUrl = `${siteUrl}/docs/sitemap.xml`;
10+
const stylesheetPath = './tests/screenshot.css';
11+
const stylesheet = fs.readFileSync(stylesheetPath).toString();
12+
13+
// Wait for hydration, requires Docusaurus v2.4.3+
14+
// Docusaurus adds a <html data-has-hydrated="true"> once hydrated
15+
function waitForDocusaurusHydration() {
16+
return document.documentElement.dataset.hasHydrated === 'true';
17+
}
18+
19+
test.describe('Docusaurus site screenshots', async () => {
20+
let pathnames: string[] = [];
21+
22+
test.beforeAll(async () => {
23+
// Fetch the sitemap dynamically
24+
try {
25+
const response = await axios.get(sitemapUrl);
26+
const sitemapContent = response.data;
27+
pathnames = extractSitemapPathnames(sitemapContent).filter((pathname) =>
28+
pathname.startsWith('/docs/en') // currently test en only
29+
);
30+
console.log(`${pathnames.length} paths to test`);
31+
} catch (error) {
32+
console.error(`Failed to fetch sitemap: ${error.message}`);
33+
throw error;
34+
}
35+
});
36+
37+
test('Generate and run screenshot tests', async ({ page }) => {
38+
const timeout = 60000; // 60 seconds timeout for navigation
39+
40+
for (const pathname of pathnames) {
41+
console.log(`Processing ${pathname}`);
42+
const url = siteUrl + pathname;
43+
44+
try {
45+
await page.goto(url, { timeout });
46+
console.log(`Successfully loaded ${url}`);
47+
48+
// Wait for hydration with a timeout
49+
await page.waitForFunction(waitForDocusaurusHydration, { timeout });
50+
console.log(`Hydration completed for ${url}`);
51+
52+
// Add custom stylesheet for screenshots
53+
await page.addStyleTag({ content: stylesheet });
54+
55+
// Take a screenshot
56+
await argosScreenshot(page, pathnameToArgosName(pathname));
57+
console.log(`Screenshot captured for ${pathname}`);
58+
} catch (error) {
59+
console.error(`Failed to process ${pathname}: ${error.message}`);
60+
}
61+
}
62+
});
63+
});

‎tests/utils.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import * as cheerio from "cheerio";
2+
import * as fs from "fs";
3+
4+
export function extractSitemapPathnames(sitemap: string): string[] {
5+
const $ = cheerio.load(sitemap, { xmlMode: true });
6+
const urls: string[] = [];
7+
$("loc").each(function handleLoc() {
8+
urls.push($(this).text());
9+
});
10+
return urls.map((url) => new URL(url).pathname);
11+
}
12+
13+
// Converts a pathname to a decent screenshot name
14+
export function pathnameToArgosName(pathname: string): string {
15+
return pathname.replace(/^\/|\/$/g, "") || "index";
16+
}

0 commit comments

Comments
 (0)