Heavy tests: FFmpeg's FATE samples and benchmarks, with Vitest

`pnpm test:heavy` decodes every sample of FFmpeg's FATE suite (~2.5k files,
rsynced to .cache/fate-suite, or FATE_SUITE=<dir>) with each core in Node.js:
first video and audio stream, at most 5 s, to framemd5. Each result must
match the snapshot in tests/heavy/baseline/fate-{st,mt}.snap; `-u` updates
it and `-t h264/` narrows the run. FATE_NATIVE=1 also decodes each sample
with native ffmpeg and fails the ones only one of them decodes.

`pnpm bench` times common jobs on FATE samples (h264/vp9 decode, scale,
remux, x264, vp9, aac/mp3/opus) with each core and prints an ST vs MT table,
plus native ffmpeg with FATE_NATIVE=1.

CI runs both in a `heavy` job nightly, on workflow_dispatch, or on pull
requests labelled `heavy`, reusing the cores the build jobs produced.

Author: lucasgelfond

.github/workflows/CI.yml

@@ -5,6 +5,10 @@ on:
     branches:
       - main
   pull_request:
+    types: [opened, synchronize, reopened, labeled]
+  schedule:
+    - cron: "0 6 * * *"
+  workflow_dispatch:
 
 jobs:
   build-core:
@@ -108,3 +112,43 @@ jobs:
       - run: pnpm install --frozen-lockfile
       - run: pnpm test:install
       - run: pnpm test --project browser-st --project browser-mt
+  heavy:
+    # FATE samples + benchmarks: nightly, on demand, or with the "heavy" PR label.
+    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'heavy')
+    runs-on: ubuntu-latest
+    needs: [build-core, build-core-mt]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: ffmpeg-core
+          path: packages/core/dist
+      - uses: actions/download-artifact@v4
+        with:
+          name: ffmpeg-core-mt
+          path: packages/core-mt/dist
+      - uses: pnpm/action-setup@v6
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: pnpm
+      - uses: actions/cache@v4
+        with:
+          path: .cache/fate-suite
+          key: fate-suite-v1
+      - run: pnpm install --frozen-lockfile
+      - name: FATE samples
+        run: pnpm test:heavy --reporter=default --reporter=json --outputFile.json=tests/heavy/results/fate.json
+      - name: FATE baseline from this run, to commit if the changes are expected
+        if: failure()
+        run: pnpm test:heavy -u
+      - name: Benchmarks
+        if: always()
+        run: pnpm bench --reporter=default --reporter=json --outputFile.json=tests/heavy/results/bench.json
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: heavy-results
+          path: |
+            tests/heavy/results
+            tests/heavy/baseline

.gitignore

@@ -8,3 +8,5 @@ dist
 
 # Local Netlify folder
 .netlify
+.cache
+tests/heavy/results

package.json

@@ -14,7 +14,9 @@
     "postinstall": "pnpm build",
     "pretest": "pnpm build",
     "test": "vitest run",
-    "test:install": "playwright install --with-deps chromium"
+    "test:install": "playwright install --with-deps chromium",
+    "test:heavy": "vitest run --config tests/heavy/vitest.config.js",
+    "bench": "vitest bench --run --config tests/heavy/vitest.config.js"
   },
   "devDependencies": {
     "@ffmpeg/ffmpeg": "workspace:*",

tests/heavy/fate.js

@@ -0,0 +1,60 @@
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { extname, join } from "node:path";
+import { inject } from "vitest";
+import { createCore } from "../helpers/core.js";
+import { exec } from "../helpers/run.js";
+
+export const suite = inject("suite");
+
+const NOT_MEDIA = [".md5", ".sha1", ".sha256", ".txt", ".html", ".xml", ".json", ".lst", ".ffmeta", ".cfg", ".sub-ref"];
+
+export const samples = readdirSync(suite, { recursive: true })
+  .filter((sample) => statSync(join(suite, sample)).isFile())
+  .filter((sample) => !NOT_MEDIA.includes(extname(sample).toLowerCase()))
+  .sort();
+
+// Damaged streams whose error concealment differs between runs with frame threads.
+const UNSTABLE_OUTPUT = ["mov/empty_edit_5s.mp4"];
+
+// The first video and audio stream, at most 5 s, hashed per frame.
+const framemd5 = (input, output) => ["-i", input, "-map", "0:v:0?", "-map", "0:a:0?", "-t", "5", "-f", "framemd5", output];
+
+function summarize(sample, out) {
+  const frames = out.split("\n").filter((line) => line && !line.startsWith("#")).length;
+  const hash = createHash("sha256").update(out).digest("hex").slice(0, 16);
+  if (!frames) return { status: "error" };
+  return UNSTABLE_OUTPUT.includes(sample) ? { status: "ok", frames } : { status: "ok", frames, hash };
+}
+
+function decode(core, sample) {
+  // keep the extension: FFmpeg guesses some formats from the file name
+  const input = `/in${extname(sample)}`;
+  core.FS.writeFile(input, readFileSync(join(suite, sample)));
+  core.setTimeout(60000);
+  const { ret } = exec(core, ...framemd5(input, "/out"));
+  const out = ret === 0 ? core.FS.readFile("/out", { encoding: "utf8" }) : "";
+  [input, "/out"].filter((path) => core.FS.analyzePath(path).exists).forEach((path) => core.FS.unlink(path));
+  return summarize(sample, out);
+}
+
+// Decodes samples with one core; a crash leaves it unusable, so the next
+// sample gets a new one.
+export function decoder() {
+  const cores = {};
+  return async (sample) => {
+    cores.current ??= await createCore();
+    try {
+      return decode(cores.current, sample);
+    } catch {
+      delete cores.current;
+      return { status: "crash" };
+    }
+  };
+}
+
+export function decodeNatively(sample) {
+  const { status, stdout } = spawnSync("ffmpeg", ["-v", "error", ...framemd5(join(suite, sample), "-")], { maxBuffer: 1 << 26, timeout: 60000 });
+  return status === 0 ? summarize(sample, stdout.toString()) : { status: "error" };
+}

tests/heavy/fate.test.js

@@ -0,0 +1,11 @@
+// Decodes every sample of FFmpeg's FATE suite and compares the result with
+// tests/heavy/baseline. `pnpm test:heavy -u` updates the baseline.
+import { expect, test } from "vitest";
+import { decoder, samples } from "./fate.js";
+
+const decode = decoder();
+
+test.for(samples)("%s", async (sample) => {
+  const result = await decode(sample);
+  expect(result).toMatchSnapshot();
+});

tests/heavy/ffmpeg.bench.js

@@ -0,0 +1,37 @@
+// Times common jobs on FATE samples with each core; with FATE_NATIVE=1, also
+// with native `ffmpeg`.
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { test } from "vitest";
+import { createCore } from "../helpers/core.js";
+import { exec } from "../helpers/run.js";
+import { suite } from "./fate.js";
+
+const H264_1080P = "h264/h264refframeregression.mp4";
+const VP9_480P = "vp9-test-vectors/vp90-2-15-segkey_adpq.webm";
+const WAV = "audio-reference/luckynight_2ch_44kHz_s16.wav";
+
+const JOBS = [
+  ["decode h264 1080p", H264_1080P, ["-f", "null", "-"]],
+  ["decode vp9 480p", VP9_480P, ["-f", "null", "-"]],
+  ["scale 1080p to 480p", H264_1080P, ["-vf", "scale=854:480", "-f", "null", "-"]],
+  ["remux mp4 to mkv", H264_1080P, ["-c", "copy", "-f", "matroska", "/out"]],
+  ["x264 veryfast 720p", H264_1080P, ["-t", "2", "-vf", "scale=1280:720", "-c:v", "libx264", "-preset", "veryfast", "-an", "-f", "mp4", "/out"]],
+  ["vp9 realtime 480p", VP9_480P, ["-c:v", "libvpx-vp9", "-deadline", "realtime", "-cpu-used", "8", "-f", "webm", "/out"]],
+  ["aac encode", WAV, ["-c:a", "aac", "-f", "ipod", "/out"]],
+  ["mp3 encode", WAV, ["-f", "mp3", "/out"]],
+  ["opus encode", WAV, ["-c:a", "libopus", "-f", "ogg", "/out"]],
+];
+
+const RUNS = { iterations: 3, time: 0, warmupIterations: 1, warmupTime: 0 };
+
+const core = await createCore();
+[H264_1080P, VP9_480P, WAV].forEach((sample) => core.FS.writeFile(`/${basename(sample)}`, readFileSync(join(suite, sample))));
+
+test.for(JOBS)("%s", async ([, sample, args], { bench }) => {
+  const wasm = bench("ffmpeg.wasm", { perProject: true }, () => exec(core, "-i", `/${basename(sample)}`, ...args));
+  const nativeArgs = ["-v", "error", "-y", "-i", join(suite, sample), ...args.map((arg) => (arg === "/out" ? "/dev/null" : arg))];
+  const native = bench("native ffmpeg", () => execFileSync("ffmpeg", nativeArgs, { stdio: "ignore" }));
+  await (process.env.FATE_NATIVE ? bench.compare(wasm, native, RUNS) : wasm.run(RUNS));
+});

tests/heavy/native.test.js

@@ -0,0 +1,12 @@
+// With FATE_NATIVE=1, the samples native `ffmpeg` decodes and ffmpeg.wasm
+// doesn't (or the other way around) fail.
+import { expect, test } from "vitest";
+import { decodeNatively, decoder, samples } from "./fate.js";
+
+const decode = decoder();
+
+test.skipIf(!process.env.FATE_NATIVE).for(samples)("%s", async (sample) => {
+  const native = decodeNatively(sample);
+  const wasm = await decode(sample);
+  expect(wasm.status).to.equal(native.status);
+});

tests/heavy/setup.js

@@ -0,0 +1,14 @@
+import { execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+const cache = fileURLToPath(new URL("../../.cache/fate-suite", import.meta.url));
+
+// FFmpeg's FATE samples (~1.3 GB): FATE_SUITE=<dir>, or downloaded to .cache/fate-suite.
+export default function setup({ provide }) {
+  const suite = process.env.FATE_SUITE ?? cache;
+  if (!existsSync(suite)) {
+    execFileSync("rsync", ["-rlLt", "rsync://fate-suite.ffmpeg.org/fate-suite/", `${suite}/`], { stdio: "inherit" });
+  }
+  provide("suite", suite);
+}

tests/heavy/vitest.config.js

@@ -0,0 +1,24 @@
+import { basename } from "node:path";
+import { defineConfig } from "vitest/config";
+
+// FFmpeg's FATE samples and benchmarks, with @ffmpeg/core in Node.js like the
+// core tests. Too slow for `pnpm test`: run with `pnpm test:heavy` and `pnpm bench`.
+const heavy = (name) => ({
+  test: {
+    name: `heavy-${name}`,
+    root: import.meta.dirname,
+    include: ["*.test.js"],
+    globalSetup: ["setup.js"],
+    provide: { core: name },
+  },
+});
+
+export default defineConfig({
+  test: {
+    testTimeout: 120000,
+    projects: [heavy("st"), heavy("mt")],
+    // one baseline per core: baseline/fate-st.snap, baseline/fate-mt.snap
+    resolveSnapshotPath: (path, extension, { config }) =>
+      `${import.meta.dirname}/baseline/${basename(path, ".test.js")}-${config.name.replace("heavy-", "")}${extension}`,
+  },
+});