Vitest reporter

.gitignore

@@ -2,6 +2,8 @@
 node_modules
 coverage
 lib
+allure-report
+allure-results
 /__image_snapshots__
 dist
 /src/test-utils/selectors

package.json

@@ -11,11 +11,13 @@
     "build": "npm-run-all build:pkg --parallel build:src:* --parallel build:pages:* build:themeable",
     "lint": "eslint --ignore-path .gitignore --ext ts,tsx,js .",
     "prepare": "husky install",
-    "test:unit": "vitest run --config vite.unit.config.js",
+    "test:unit": "vitest run --config vite.unit.config.mjs",
     "test:functional": "run-p -r preview test:functional:vitest",
-    "test:functional:vitest": "vitest run --config vite.e2e-functional.config.js",
+    "test:functional:vitest": "vitest run --config vite.e2e-functional.config.mjs",
+    "pretest:visual": "rm -rf allure-results",
     "test:visual": "run-p -r preview test:visual:vitest",
-    "test:visual:vitest": "vitest run --config vite.e2e-visual.config.js",
+    "posttest:visual": "allure generate --clean allure-results",
+    "test:visual:vitest": "vitest run --config vite.e2e-visual.config.mjs",
     "test:visual:update": "UPDATE_SCREENSHOTS=true npm run test:visual",
     "pretest": "tsc -p tsconfig.unit.json && tsc -p tsconfig.e2e.json",
     "test": "run-s lint test:unit test:functional",
@@ -69,13 +71,15 @@
     "@testing-library/react": "^13.4.0",
     "@types/jest-image-snapshot": "^6.1.0",
     "@types/lodash": "^4.14.191",
-    "@types/node": "^16.0.0",
+    "@types/node": "^18.19.4",
     "@types/react": "^18.0.26",
     "@types/react-dom": "^18.0.10",
     "@typescript-eslint/eslint-plugin": "^5.48.0",
     "@typescript-eslint/parser": "^5.48.0",
-    "@vitejs/plugin-react": "^3.0.0",
-    "@vitest/coverage-istanbul": "^0.28.3",
+    "@vitejs/plugin-react": "^4.2.1",
+    "@vitest/coverage-istanbul": "^1.1.1",
+    "allure-commandline": "^2.25.0",
+    "allure-js-commons": "^2.10.0",
     "chokidar-cli": "^3.0.0",
     "deep-freeze-es6": "^1.4.1",
     "eslint": "^8.31.0",
@@ -101,8 +105,8 @@
     "react-dom": "^18.2.0",
     "react-router-dom": "^6.6.1",
     "typescript": "^4.9.4",
-    "vite": "^4.0.5",
-    "vitest": "^0.25.8"
+    "vite": "^5.0.10",
+    "vitest": "^1.1.1"
   },
   "//": "ensure that typedoc uses latest typescript. It prints a warning, but works",
   "overrides": {

src/__tests__/__snapshots__/documenter.test.ts.snap

@@ -1,4 +1,4 @@
-// Vitest Snapshot v1
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
 exports[`definition for board matches the snapshot > board 1`] = `
 {
@@ -55,12 +55,12 @@ The change detail has the following properties:
     {
       "description": "An object containing all the necessary localized strings required by the component.
 Live announcements:
-* \`liveAnnouncementDndStarted(BoardProps.DndOperationType): string\` - the function to create a live announcement string to indicate start of DnD (\\"reorder\\", \\"resize\\" or \\"insert\\").
+* \`liveAnnouncementDndStarted(BoardProps.DndOperationType): string\` - the function to create a live announcement string to indicate start of DnD ("reorder", "resize" or "insert").
 * \`liveAnnouncementDndItemReordered(BoardProps.DndReorderState<D>): string\` - the function to create a live announcement string to indicate when DnD reorder is performed.
 * \`liveAnnouncementDndItemResized(BoardProps.DndResizeState<D>): string\` - the function to create a live announcement string to indicate when DnD resize is performed.
 * \`liveAnnouncementDndItemInserted(BoardProps.DndInsertState<D>): string\` - the function to create a live announcement string to indicate when DnD insert is performed.
-* \`liveAnnouncementDndDiscarded(BoardProps.DndOperationType): string\` - the function to create a live announcement string to indicate commit of DnD (\\"reorder\\", \\"resize\\" or \\"insert\\").
-* \`liveAnnouncementDndCommitted(BoardProps.DndOperationType): string\` - the function to create a live announcement string to indicate discard of DnD (\\"reorder\\", \\"resize\\" or \\"insert\\").
+* \`liveAnnouncementDndDiscarded(BoardProps.DndOperationType): string\` - the function to create a live announcement string to indicate commit of DnD ("reorder", "resize" or "insert").
+* \`liveAnnouncementDndCommitted(BoardProps.DndOperationType): string\` - the function to create a live announcement string to indicate discard of DnD ("reorder", "resize" or "insert").
 * \`liveAnnouncementItemRemoved(BoardProps.OperationStateRemove<D>): string\` - the function to create a live announcement string to indicate when item is removed.
 
 Navigation labels:

src/internal/environment.d.ts

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // This a placeholder for typescript. The real files are generated by scripts/environment.js.
-export const THEME: string;
-export const PACKAGE_SOURCE: string;
-export const PACKAGE_VERSION: string;
+export declare const THEME: string;
+export declare const PACKAGE_SOURCE: string;
+export declare const PACKAGE_VERSION: string;
 /** Indicates that the current theme is always in visual refresh mode. */
-export const ALWAYS_VISUAL_REFRESH: boolean;
+export declare const ALWAYS_VISUAL_REFRESH: boolean;

test/functional/board-layout/dnd-page-object.ts

@@ -38,18 +38,18 @@ export class DndPageObject extends BasePageObject {
         [...document.querySelectorAll(widgetsSelector)].map(
           (w) => [w.getAttribute("data-item-id"), w.getBoundingClientRect()] as [string, DOMRect]
         ),
-      `.${boardStyles.default.root} [data-item-id]`
+      `.${boardStyles.root} [data-item-id]`
     );
 
     const placeholderRects = await this.browser.execute(
       (placeholderSelector) =>
         [...document.querySelectorAll(placeholderSelector)].map((p) => p.getBoundingClientRect()),
-      `.${boardStyles.default.placeholder}`
+      `.${boardStyles.placeholder}`
     );
     const hoveredPlaceholderRects = await this.browser.execute(
       (placeholderSelector) =>
         [...document.querySelectorAll(placeholderSelector)].map((p) => p.getBoundingClientRect()),
-      `.${boardStyles.default["placeholder--hover"]}`
+      `.${boardStyles["placeholder--hover"]}`
     );
 
     function matchWidget(placeholderIndex: number): null | string {

test/functional/board-layout/layout.test.ts

@@ -155,7 +155,7 @@ test(
     ),
     DndPageObject,
     async (page) => {
-      const placeholderSelector = `.${boardStyles.default.placeholder}`;
+      const placeholderSelector = `.${boardStyles.placeholder}`;
 
       await page.setWindowSize({ width: 1600, height: 1800 });
       await expect(page.getElementsCount(placeholderSelector)).resolves.toBe(10 * 4);

test/utils.ts

@@ -1,12 +1,68 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { BasePageObject } from "@cloudscape-design/browser-test-tools/page-objects";
+import fs from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { cropAndCompare, cropByOffset, parsePng } from "@cloudscape-design/browser-test-tools/image-utils";
+import {
+  BasePageObject,
+  ScreenshotPageObject as BaseScreenshotPageObject,
+  ScreenshotWithOffset,
+} from "@cloudscape-design/browser-test-tools/page-objects";
 import useBrowser from "@cloudscape-design/browser-test-tools/use-browser";
+import { allureTest } from "../vitest-reporter/fixture";
 
 // Default window size to ensure 4-columns layout is used.
 const windowSize = { width: 1600, height: 800 };
 
+class ScreenshotPageObject extends BaseScreenshotPageObject {
+  async url(url: string) {
+    await this.browser.url(url);
+  }
+}
+
+interface CompareFixture {
+  (image: ScreenshotWithOffset): Promise<Awaited<ReturnType<typeof cropAndCompare>>>;
+}
+
+export const screenshotTest = allureTest.extend<{ page: ScreenshotPageObject; compare: CompareFixture }>({
+  // eslint-disable-next-line no-empty-pattern
+  page: ({}, use) => {
+    return useBrowser(windowSize, async (browser) => {
+      const page = new ScreenshotPageObject(browser);
+      await use(page);
+    })();
+  },
+  compare: async ({ task }, use) => {
+    let counter = 0;
+    const targetDir = "__image_snapshots__";
+    await use(async (actual) => {
+      const snapshotName = `${task.file!.name}-${task.suite.name}-${task.name}-${counter++}.png`.replaceAll("/", "-");
+      if (process.env.UPDATE_SCREENSHOTS) {
+        const image = await cropByOffset(actual.image, actual.offset);
+        fs.writeFileSync(path.join(targetDir, snapshotName), image);
+        return {
+          diffPixels: 0,
+          diffImage: null,
+          isEqual: true,
+          firstImage: image,
+          secondImage: image,
+        };
+      } else {
+        if (!fs.existsSync(path.join(targetDir, snapshotName))) {
+          throw new Error(`No reference image ${snapshotName}`);
+        }
+        const reference = {
+          ...actual,
+          image: await parsePng(fs.readFileSync(path.join(targetDir, snapshotName), "base64")),
+        };
+        return cropAndCompare(actual, reference);
+      }
+    });
+  },
+});
+
 export function setupTest<P extends BasePageObject & { init?(): Promise<void> }>(
   url: string,
   PageClass: new (browser: WebdriverIO.Browser) => P,

test/visual/dummy.test.ts

@@ -0,0 +1,16 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import { allureTest } from "../../vitest-reporter/fixture";
+// import { screenshotTest } from "../utils";
+
+allureTest("dummy", async ({ allure }) => {
+  allure.attachment("test.txt", "{works: true}", "application/json");
+
+  await allure.step("step 1", async () => {
+    await allure.step("sub-step 1", async () => {});
+    await allure.step("sub-step 2", () => {
+      allure.attachment("sub-step.txt", "{subStep: true}", "application/json");
+      return Promise.resolve();
+    });
+  });
+});

test/visual/index.test.ts

@@ -1,22 +1,45 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 import path from "path";
-import { ScreenshotPageObject } from "@cloudscape-design/browser-test-tools/page-objects";
-import { expect, test } from "vitest";
-import { setupTest } from "../utils";
+import { ContentType } from "allure-js-commons";
+import { expect } from "vitest";
+import { screenshotTest } from "../utils";
 
 const pagesMap = import.meta.glob("../../pages/**/*.page.tsx", { as: "raw" });
 const pages = Object.keys(pagesMap)
   .map((page) => page.replace(/\.page\.tsx$/, ""))
   .map((page) => "/#/" + path.relative("../../pages/", page));
 
-test.each(pages)("matches snapshot for %s", (route) =>
-  setupTest(route, ScreenshotPageObject, async (page) => {
+for (const route of pages) {
+  screenshotTest(`matches snapshot for ${route}`, async ({ task, allure, page, compare }) => {
+    await page.url(route);
     const hasScreenshotArea = await page.isExisting(".screenshot-area");
 
-    if (hasScreenshotArea) {
-      const pngString = await page.fullPageScreenshot();
-      expect(pngString).toMatchImageSnapshot();
+    if (!hasScreenshotArea) {
+      task.context.skip();
+      return;
     }
-  })()
-);
+    const screenshot = await page.captureViewport();
+    const { diffImage, diffPixels, firstImage, secondImage, isEqual } = await compare(screenshot);
+    allure.label("testType", "screenshotDiff");
+    allure.attachment(`compareResult`, JSON.stringify({ diffPixels, isEqual }, null, 2), ContentType.JSON);
+    allure.attachment(`actual`, firstImage, ContentType.PNG);
+    allure.attachment(`expected`, secondImage, ContentType.PNG);
+
+    if (diffImage) {
+      allure.attachment(
+        `diff`,
+        JSON.stringify({
+          expected: `data:image/png;base64,${secondImage.toString("base64")}`,
+          actual: `data:image/png;base64,${firstImage.toString("base64")}`,
+          diff: diffImage && `data:image/png;base64,${diffImage.toString("base64")}`,
+        }),
+        {
+          contentType: "application/vnd.allure.image.diff",
+          fileExtension: "json",
+        }
+      );
+    }
+    expect(isEqual).toBe(true);
+  });
+}

types/global.d.ts

@@ -5,9 +5,6 @@ declare module "*.css.js" {
   export default styles;
 }
 declare module "*.selectors.js" {
-  // this is how Node.js and Vitest receive ESM transpiled to CJS
-  const styles: {
-    default: Record<string, string>;
-  };
+  const styles: Record<string, string>;
   export default styles;
 }

vite.e2e-functional.config.js → vite.e2e-functional.config.mjs

@@ -1,7 +1,7 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 import { defineConfig } from "vite";
-import base from "./vite.config.js";
+import base from "./vite.config.mjs";
 
 // https://vitejs.dev/config/
 export default defineConfig({

vite.e2e-visual.config.js → vite.e2e-visual.config.mjs

@@ -2,14 +2,15 @@
 // SPDX-License-Identifier: Apache-2.0
 import process from "node:process";
 import { defineConfig } from "vite";
-import functional from "./vite.e2e-functional.config.js";
+import functional from "./vite.e2e-functional.config.mjs";
 
 // https://vitejs.dev/config/
 export default defineConfig({
   ...functional,
   test: {
     ...functional.test,
     update: process.env.UPDATE_SCREENSHOTS,
+    reporters: ["default", "./vitest-reporter/reporter.ts"],
     include: ["./test/visual/**/*.test.ts"],
     setupFiles: [...functional.test.setupFiles, "./test/visual-test-setup.ts"],
   },

vite.unit.config.js → vite.unit.config.mjs

@@ -3,7 +3,7 @@
 
 import process from "node:process";
 import { defineConfig } from "vite";
-import base from "./vite.config.js";
+import base from "./vite.config.mjs";
 
 // https://vitejs.dev/config/
 export default defineConfig({

vitest-reporter/fixture.ts

@@ -0,0 +1,40 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import { Status } from "allure-js-commons";
+import { test } from "vitest";
+import { AllureApi, AllureMeta, AllureStep } from "./interfaces";
+
+export const allureTest = test.extend<{ allure: AllureApi }>({
+  allure: async ({ task }, use) => {
+    const meta = task.meta as { allure: AllureMeta };
+    const currentTest = { name: "root-test", labels: [], attachments: [], steps: [] };
+    meta.allure = { currentStep: currentTest, currentTest: currentTest };
+    await use({
+      attachment: (name, content, options) => {
+        const parsedOptions = typeof options === "string" ? { contentType: options } : options;
+        meta.allure.currentStep.attachments.push({ name, content, ...parsedOptions });
+      },
+      label: (name, value) => meta.allure.currentTest.labels.push({ name, value }),
+      step: async (name, body) => {
+        const prevStep = meta.allure.currentStep;
+        const nextStep: AllureStep = { name, attachments: [], steps: [], start: Date.now() };
+        prevStep.steps.push(nextStep);
+        meta.allure.currentStep = nextStep;
+        try {
+          const result = await body();
+          nextStep.status = Status.PASSED;
+          return result;
+        } catch (error) {
+          nextStep.status = Status.FAILED;
+          if (error instanceof Error) {
+            nextStep.statusDetails = { message: error.message, trace: error.stack };
+          }
+          throw error;
+        } finally {
+          nextStep.stop = Date.now();
+          meta.allure.currentStep = prevStep;
+        }
+      },
+    });
+  },
+});

vitest-reporter/interfaces.ts

@@ -0,0 +1,35 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+import { AttachmentOptions, Status, StatusDetails } from "allure-js-commons";
+
+export interface AllureAttachment {
+  name: string;
+  contentType: string;
+  fileExtension?: string;
+  content: Buffer | string;
+}
+
+export interface AllureStep {
+  name: string;
+  start?: number;
+  stop?: number;
+  status?: Status;
+  attachments: Array<AllureAttachment>;
+  steps: Array<AllureStep>;
+  statusDetails?: StatusDetails;
+}
+
+export interface AllureTest extends AllureStep {
+  labels: Array<{ name: string; value: string }>;
+}
+
+export interface AllureMeta {
+  currentTest: AllureTest;
+  currentStep: AllureStep;
+}
+
+export interface AllureApi {
+  label: (name: string, value: string) => void;
+  attachment: (name: string, content: Buffer | string, options: string | AttachmentOptions) => void;
+  step: <T>(name: string, body: () => Promise<T>) => Promise<T>;
+}