visual testing

.github/workflows/test-preview.yml

@@ -0,0 +1,71 @@
+name: Playwright + Argos Tests
+
+on:
+  # Trigger on deployment event
+  deployment_status:
+  workflow_dispatch:
+  # Trigger when a label is added to a PR
+  pull_request:
+    types:
+      - labeled
+
+jobs:
+  test:
+    # Run tests only if the deployment is successful
+    if: |
+      (
+        github.event_name == 'deployment_status' &&
+        github.event.deployment_status.state == 'success'
+      ) || (
+        github.event_name == 'pull_request' &&
+        github.event.action == 'labeled' &&
+        github.event.label.name == 'visual-test'
+      )
+    runs-on: ubuntu-latest
+
+    steps:
+
+      # Step 1: Check out the repository
+      - name: Check out docs repo
+        uses: actions/checkout@v4
+        with:
+          repository: ClickHouse/clickhouse-docs
+          path: ./
+
+      # Step 2: Log url
+      - name: Log BASE_URL
+        run: echo "BASE_URL=${{ github.event.deployment_status.environment_url }}"
+        env:
+          BASE_URL: ${{ github.event.deployment_status.environment_url }}
+
+      # Step 3: Setup node
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20.18.0'
+          cache: 'yarn'
+
+      # Step 4: Install dependencies and build the site (needed for sitemap.xml)
+      - name: Install and build
+        run: |
+          export NODE_OPTIONS="--max_old_space_size=4096"
+          npm install -g yarn
+          yarn install
+
+      - name: Configure Git safe.directory
+        run: |
+          git config --global --add safe.directory /workspace
+
+      # Step 3: Run Playwright tests in Docker
+      - name: Run Playwright tests
+        run: |
+          docker run --rm \
+            -v ${{ github.workspace }}:/workspace \
+            -w /workspace \
+            -e GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} \
+            -e BASE_URL=${{ github.event.deployment_status.environment_url }} \
+            -e ARGOS_BRANCH="${{ github.event.deployment_status.environment == 'Production' && 'main' || github.ref_name }}" \
+            -e CI=true \
+            -e ARGOS_TOKEN=${{ secrets.ARGOS_TOKEN }} \
+            mcr.microsoft.com/playwright:v1.49.1-noble \
+            npm exec -- playwright test

.gitignore

@@ -41,3 +41,5 @@ FormatFactorySettings.h
 Settings.cpp
 
 .vscode
+test-results/**
+tests/screenshot.spec.ts-snapshots/**

build-archive.log

@@ -0,0 +1,29 @@
+@storybook/core v8.4.7
+
+info => Cleaning outputDir: ../../var/folders/48/dc999rwj32s3nv1pkl1sdq4w0000gn/T/chromatic--78807-vB4WXYLSV7Sf
+info => Loading presets
+info => Building manager..
+WARN unable to find package.json for @chromatic-com/shared-e2e
+WARN unable to find package.json for @storybook/types
+info => Manager built (335 ms)
+info => Building preview..
+info Addon-docs: using MDX3
+info => Using implicit CSS loaders
+info => Using default Webpack5 setup
+info => Copying static files: test-results/chromatic-archives/archive at ../../var/folders/48/dc999rwj32s3nv1pkl1sdq4w0000gn/T/chromatic--78807-vB4WXYLSV7Sf
+WARN export 'useId' (imported as 'wl') was not found in 'react' (possible exports: Children, Component, Fragment, Profiler, PureComponent, StrictMode, Suspense, __SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED, cloneElement, createContext, createElement, createFactory, createRef, forwardRef, isValidElement, lazy, memo, useCallback, useContext, useDebugValue, useEffect, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useRef, useState, version)
+WARN export 'useInsertionEffect' (imported as 'Re') was not found in 'react' (possible exports: Children, Component, Fragment, Profiler, PureComponent, StrictMode, Suspense, __SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED, cloneElement, createContext, createElement, createFactory, createRef, forwardRef, isValidElement, lazy, memo, useCallback, useContext, useDebugValue, useEffect, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useRef, useState, version)
+WARN export 'useInsertionEffect' (imported as 'Re') was not found in 'react' (possible exports: Children, Component, Fragment, Profiler, PureComponent, StrictMode, Suspense, __SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED, cloneElement, createContext, createElement, createFactory, createRef, forwardRef, isValidElement, lazy, memo, useCallback, useContext, useDebugValue, useEffect, useImperativeHandle, useLayoutEffect, useMemo, useReducer, useRef, useState, version)
+WARN asset size limit: The following asset(s) exceed the recommended size limit (244 KiB).
+WARN This can impact web performance.
+WARN Assets: 
+WARN   7857.1207a2b9.iframe.bundle.js (1.14 MiB)
+WARN entrypoint size limit: The following entrypoint(s) combined asset size exceeds the recommended limit (244 KiB). This can impact web performance.
+WARN Entrypoints:
+WARN   main (261 KiB)
+WARN       runtime~main.e431fbf4.iframe.bundle.js
+WARN       878.4fc2c244.iframe.bundle.js
+WARN       main.e5f40e77.iframe.bundle.js
+WARN 
+info => Preview built (3.62 s)
+info => Output directory: /var/folders/48/dc999rwj32s3nv1pkl1sdq4w0000gn/T/chromatic--78807-vB4WXYLSV7Sf

package.json

@@ -19,13 +19,14 @@
     "new-build": "bash ./copyClickhouseRepoDocs.sh && bash ./scripts/settings/autogenerate-settings.sh && yarn build-api-doc && yarn build && yarn build-swagger",
     "start": "docusaurus start",
     "swizzle": "docusaurus swizzle",
-    "write-heading-ids": "docusaurus write-heading-ids"
+    "write-heading-ids": "docusaurus write-heading-ids",
+    "screenshot": "playwright test"
   },
   "dependencies": {
-    "@docusaurus/core": "2.3.1",
-    "@docusaurus/plugin-client-redirects": "2.3.1",
-    "@docusaurus/preset-classic": "2.3.1",
-    "@docusaurus/theme-mermaid": "2.3.1",
+    "@docusaurus/core": "2.4.3",
+    "@docusaurus/plugin-client-redirects": "2.4.3",
+    "@docusaurus/preset-classic": "2.4.3",
+    "@docusaurus/theme-mermaid": "2.4.3",
     "@mdx-js/react": "^1.6.22",
     "@radix-ui/react-navigation-menu": "^1.1.4",
     "axios": "^1.7.9",
@@ -44,7 +45,11 @@
     "sass": "^1.82.0"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "3.6.3"
+    "@argos-ci/cli": "^2.5.3",
+    "@argos-ci/playwright": "^3.9.4",
+    "@docusaurus/module-type-aliases": "3.6.3",
+    "@playwright/test": "^1.49.1",
+    "cheerio": "^1.0.0"
   },
   "browserslist": {
     "production": [
@@ -59,6 +64,6 @@
     ]
   },
   "engines": {
-    "node": ">=16.14"
+    "node": ">=20.18"
   }
 }

playwright.config.ts

@@ -0,0 +1,49 @@
+import {devices} from '@playwright/test';
+import type {PlaywrightTestConfig} from '@playwright/test';
+
+const isCI = !!process.env.CI; // Check if running in CI
+const baseURL = isCI ? process.env.BASE_URL : "http://localhost:3000";
+
+
+const config: PlaywrightTestConfig = {
+  fullyParallel: true,
+  webServer: {
+    port: 3000,
+    command: 'yarn docusaurus serve',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+  reporter: [
+    // Use "dot" reporter on CI, "list" otherwise (Playwright default).
+    process.env.CI ? ["dot"] : ["list"],
+    // Add Argos reporter.
+    [
+      "@argos-ci/playwright/reporter",
+      {
+        // Upload to Argos on CI only.
+        uploadToArgos: isCI,
+
+        // Set your Argos token.
+        token: process.env.ARGOS_TOKEN,
+      },
+    ],
+  ],
+  timeout: 1200000, 
+  use: {
+    // On CI, we will set `BASE_URL` from Vercel preview URL
+    baseURL: baseURL,
+    extraHTTPHeaders: {
+        // Hide Vercel Toolbar in tests
+        "x-vercel-skip-toolbar": "0",
+    },
+  },
+
+};
+
+export default config;

tests/screenshot.css

@@ -0,0 +1,35 @@
+/*
+We need to hide some elements in Argos/Playwright screenshots
+Those elements are source of flakiness due to nondeterministic rendering
+They don't consistently render exactly the same across CI runs
+ */
+
+/******* DOCUSAURUS GLOBAL / THEME *******/
+
+/* Iframes can load lazily */
+iframe,
+/* Avatar images can be flaky due to using external sources: GitHub/Unavatar */
+.avatar__photo,
+/* Gifs load lazily and are animated */
+img[src$='.gif'],
+/* Algolia Keyboard shortcuts appear with a little delay */
+.DocSearch-Button-Keys > kbd,
+/* The live playground preview can often display dates/counters */
+[class*='playgroundPreview'] {
+  visibility: hidden;
+}
+
+/*
+Different docs last-update dates can alter layout
+"visibility: hidden" is not enough
+ */
+.theme-last-updated {
+  display: none;
+}
+
+/*
+Mermaid diagrams are rendered client-side and produce layout shifts
+ */
+.docusaurus-mermaid-container {
+  display: none;
+}

tests/screenshot.spec.ts

@@ -0,0 +1,64 @@
+import * as fs from 'fs';
+import { argosScreenshot } from '@argos-ci/playwright';
+import { test } from '@playwright/test';
+import axios from 'axios';
+import { extractSitemapPathnames, pathnameToArgosName } from './utils';
+
+// Constants
+const siteUrl = process.env.CI ? process.env.BASE_URL : 'http://localhost:3000';
+const sitemapUrl = `${siteUrl}/docs/sitemap.xml`;
+const stylesheetPath = './tests/screenshot.css';
+const stylesheet = fs.readFileSync(stylesheetPath).toString();
+
+// Wait for hydration, requires Docusaurus v2.4.3+
+// Docusaurus adds a <html data-has-hydrated="true"> once hydrated
+function waitForDocusaurusHydration() {
+  return document.documentElement.dataset.hasHydrated === 'true';
+}
+
+test.describe.configure({ mode: 'parallel' });
+
+let pathnames: string[] = [];
+
+test.beforeAll(async () => {
+  // Fetch the sitemap dynamically
+  try {
+    const response = await axios.get(sitemapUrl);
+    const sitemapContent = response.data;
+    pathnames = extractSitemapPathnames(sitemapContent).filter((pathname) =>
+      pathname.startsWith('/docs/en') // currently test en only
+    );
+    console.log(`${pathnames.length} paths to test`);
+  } catch (error) {
+    console.error(`Failed to fetch sitemap: ${error.message}`);
+    throw error;
+  }
+});
+
+
+for (const pathname of pathnames) {
+  console.log(`Processing ${pathname}`);
+  test('Generate and run screenshot tests', async ({ page }) => {
+    const url = siteUrl + pathname;
+    const timeout = 60000; // 60 seconds timeout for navigation
+    try {
+      await page.goto(url, { timeout });
+      console.log(`Successfully loaded ${url}`);
+
+      // Wait for hydration with a timeout
+      await page.waitForFunction(waitForDocusaurusHydration, { timeout });
+      console.log(`Hydration completed for ${url}`);
+
+      // Add custom stylesheet for screenshots
+      await page.addStyleTag({ content: stylesheet });
+
+      // Take a screenshot
+      await argosScreenshot(page, pathnameToArgosName(pathname));
+      console.log(`Screenshot captured for ${pathname}`);
+    } catch (error) {
+      console.error(`Failed to process ${pathname}: ${error.message}`);
+    }
+  });
+}
+
+

tests/utils.ts

@@ -0,0 +1,16 @@
+import * as cheerio from "cheerio";
+import * as fs from "fs";
+
+export function extractSitemapPathnames(sitemap: string): string[] {
+  const $ = cheerio.load(sitemap, { xmlMode: true });
+  const urls: string[] = [];
+  $("loc").each(function handleLoc() {
+    urls.push($(this).text());
+  });
+  return urls.map((url) => new URL(url).pathname);
+}
+
+// Converts a pathname to a decent screenshot name
+export function pathnameToArgosName(pathname: string): string {
+  return pathname.replace(/^\/|\/$/g, "") || "index";
+}