bench-examples.mjs

// Langium example benchmark harness — the counterpart of pegium's PegiumBench.
//
// This file is source-controlled in the pegium repository
// (tools/langium-bench/bench-examples.mjs) and installed into a Langium checkout
// at <langium>/scripts/bench-examples.mjs by tools/compare_langium_bench.py.
//
// It builds each example language through Langium's DocumentBuilder and reports
// per-phase timings in the exact same text format as PegiumBench, so
// compare_langium_bench.py can diff the two. Inputs are generated by the same
// logic as the pegium benches (tests/bench/*.cpp), so both sides parse identical
// source.
//
// Env knobs (mirrors PegiumBench):
//   LANGIUM_BENCH_BYTES        single-file target size in bytes (default 65536)
//   LANGIUM_BENCH_ITERATIONS   iterations per benchmark         (default 3)
//   LANGIUM_BENCH_WARMUP       warmup iterations                (default 0)
//   LANGIUM_BENCH_WS_SMALL     small-workspace target bytes     (default 262144)
//   LANGIUM_BENCH_WS_LARGE     large-workspace target bytes     (default 12582912)
//   LANGIUM_BENCH_FILTER       substring filter over bench names

import { performance } from 'node:perf_hooks';
import { EmptyFileSystem, URI, DocumentState } from 'langium';

import { createArithmeticsServices } from '../examples/arithmetics/out/language-server/arithmetics-module.js';
import { createDomainModelServices } from '../examples/domainmodel/out/language-server/domain-model-module.js';
import { createRequirementsAndTestsLangServices } from '../examples/requirements/out/language-server/requirements-and-tests-lang-module.js';
import { createStatemachineServices } from '../examples/statemachine/out/language-server/statemachine-module.js';

const envInt = (name, dflt, min) => {
  const raw = parseInt(process.env[name] ?? '', 10);
  return Math.max(Number.isFinite(raw) ? raw : dflt, min);
};

const TARGET_BYTES = envInt('LANGIUM_BENCH_BYTES', 64 * 1024, 16 * 1024);
const ITERATIONS = envInt('LANGIUM_BENCH_ITERATIONS', 3, 1);
const WARMUP = envInt('LANGIUM_BENCH_WARMUP', 0, 0);
const WS_SMALL_BYTES = envInt('LANGIUM_BENCH_WS_SMALL', 256 * 1024, 16 * 1024);
const WS_LARGE_BYTES = envInt('LANGIUM_BENCH_WS_LARGE', 12 * 1024 * 1024, 16 * 1024);
const FILTER = process.env.LANGIUM_BENCH_FILTER ?? '';

// One boundary state per pegium build phase, so both engines report the same
// three phases: parse+index (-> IndexedContent), scope+link (-> IndexedReferences),
// validate (-> Validated). full-build is the total.
const PHASES = [
  DocumentState.IndexedContent,
  DocumentState.IndexedReferences,
  DocumentState.Validated,
];
const STEP_NAMES = ['parse+index', 'scope+link', 'validate'];

// ---- input generators (byte-for-byte equivalents of tests/bench/*.cpp) ----

function arithmeticsSource(targetBytes) {
  let source = 'module Bench\n\n';
  let index = 0;
  while (source.length < targetBytes) {
    if (index === 0) {
      source += 'def value0: 1 + 2;\n';
    } else {
      source += `def value${index}: value${index - 1} + 1;\n`;
    }
    if (index % 16 === 0) {
      source += `value${index};\n`;
    }
    index++;
  }
  return source;
}

function domainmodelSource(targetBytes) {
  let source = 'datatype String\n';
  source += 'package bench {\n';
  let index = 0;
  while (source.length < targetBytes) {
    source += `  entity Entity${index}`;
    if (index > 0) {
      source += ` extends Entity${index - 1}`;
    }
    source += ' {\n';
    source += '    name: String\n';
    if (index > 0) {
      source += `    prev: Entity${index - 1}\n`;
    }
    source += '  }\n';
    index++;
  }
  source += '}\n';
  return source;
}

function requirementsSource(targetBytes) {
  const environmentCount = 24;
  let source = 'contact: "bench"\n';
  for (let index = 0; index < environmentCount; index++) {
    source += `environment Env${index}: "Environment ${index}"\n`;
  }
  let requirementIndex = 0;
  while (source.length < targetBytes) {
    source +=
      `req REQ${requirementIndex} "Requirement ${requirementIndex}" applicable for ` +
      `Env${requirementIndex % environmentCount}, ` +
      `Env${(requirementIndex + 1) % environmentCount}\n`;
    requirementIndex++;
  }
  return source;
}

function statemachineSource(targetBytes) {
  const eventCount = 64;
  const commandCount = 64;
  let source = 'statemachine Bench\n';
  source += 'events';
  for (let index = 0; index < eventCount; index++) {
    source += ` Event${index}`;
  }
  source += '\ncommands';
  for (let index = 0; index < commandCount; index++) {
    source += ` Command${index}`;
  }
  source += '\ninitialState State0\n';
  let stateIndex = 0;
  while (source.length < targetBytes) {
    source += `state State${stateIndex} actions { Command${stateIndex % commandCount} }\n`;
    source += `Event${stateIndex % eventCount} => State${stateIndex + 1}\n`;
    source += 'end\n';
    stateIndex++;
  }
  source += `state State${stateIndex} actions { Command0 }\n`;
  source += 'Event0 => State0\n';
  source += 'end\n';
  return source;
}

// Per-language workspace corpora: many self-contained files of one language with
// unique top-level names, built simultaneously at startup (the fastbelt model).
// The generators match tests/bench/Workspace.cpp byte-for-byte.
const WS_PER_FILE_BYTES = 8 * 1024;

function arithmeticsWorkspaceFile(fileIndex, perFileBytes) {
  let source = `module Bench${fileIndex}\n\n`;
  let index = 0;
  while (source.length < perFileBytes) {
    source += index === 0
      ? 'def value0: 1 + 2;\n'
      : `def value${index}: value${index - 1} + 1;\n`;
    if (index % 16 === 0) {
      source += `value${index};\n`;
    }
    index++;
  }
  return source;
}

function domainmodelWorkspaceFile(fileIndex, perFileBytes) {
  const suffix = String(fileIndex);
  let source = `datatype String${suffix}\npackage bench${suffix} {\n`;
  let index = 0;
  while (source.length < perFileBytes) {
    source += `  entity Entity${index}`;
    if (index > 0) {
      source += ` extends Entity${index - 1}`;
    }
    source += ` {\n    name: String${suffix}\n`;
    if (index > 0) {
      source += `    prev: Entity${index - 1}\n`;
    }
    source += '  }\n';
    index++;
  }
  source += '}\n';
  return source;
}

function requirementsWorkspaceFile(fileIndex, perFileBytes) {
  const environmentCount = 24;
  const suffix = String(fileIndex);
  let source = 'contact: "bench"\n';
  for (let index = 0; index < environmentCount; index++) {
    source += `environment E${suffix}_${index}: "Environment ${index}"\n`;
  }
  let requirementIndex = 0;
  while (source.length < perFileBytes) {
    source +=
      `req R${suffix}_${requirementIndex} "Requirement ${requirementIndex}" ` +
      `applicable for E${suffix}_${requirementIndex % environmentCount}, ` +
      `E${suffix}_${(requirementIndex + 1) % environmentCount}\n`;
    requirementIndex++;
  }
  return source;
}

function statemachineWorkspaceFile(fileIndex, perFileBytes) {
  const eventCount = 64;
  const commandCount = 64;
  let source = `statemachine Bench${fileIndex}\nevents`;
  for (let index = 0; index < eventCount; index++) {
    source += ` Event${index}`;
  }
  source += '\ncommands';
  for (let index = 0; index < commandCount; index++) {
    source += ` Command${index}`;
  }
  source += '\ninitialState State0\n';
  let stateIndex = 0;
  while (source.length < perFileBytes) {
    source += `state State${stateIndex} actions { Command${stateIndex % commandCount} }\n`;
    source += `Event${stateIndex % eventCount} => State${stateIndex + 1}\nend\n`;
    stateIndex++;
  }
  source += `state State${stateIndex} actions { Command0 }\nEvent0 => State0\nend\n`;
  return source;
}

function workspaceFiles(fileGen, targetBytes) {
  const files = [];
  let total = 0;
  let index = 0;
  while (total < targetBytes) {
    const text = fileGen(index, WS_PER_FILE_BYTES);
    total += Buffer.byteLength(text, 'utf8');
    files.push(text);
    index++;
  }
  return files;
}

// ---- timing helpers ----

function freshShared(createServices) {
  return createServices(EmptyFileSystem).shared;
}

function instrumentPhases(builder) {
  const times = new Array(PHASES.length).fill(undefined);
  const subscriptions = PHASES.map((state, index) =>
    builder.onBuildPhase(state, () => {
      times[index] = performance.now();
    })
  );
  return { times, dispose: () => subscriptions.forEach((s) => s.dispose()) };
}

// parseMs is the eager parse (Langium parses in fromString, before build runs);
// it is folded into the parse+index phase so both engines measure the same work.
function phaseTimings(start, end, times, parseMs) {
  return {
    'parse+index': parseMs + (times[0] - start),
    'scope+link': times[1] - times[0],
    'validate': times[2] - times[1],
    'full-build': parseMs + (end - start),
  };
}

function countErrors(documents) {
  let errors = 0;
  for (const doc of documents) {
    for (const d of doc.diagnostics ?? []) {
      if (d.severity === 1) {
        errors++;
      }
    }
  }
  return errors;
}

// Langium parses eagerly in `fromString`, before the build runs; PegiumBench's
// documents start unparsed and are parsed inside `build`. Time the parse
// explicitly and fold it into the parse+index phase so both engines measure the
// same parse → index → scope → link → validate work.
async function buildSingle(createServices, ext, source) {
  const shared = freshShared(createServices);
  const builder = shared.workspace.DocumentBuilder;
  const parseStart = performance.now();
  const doc = shared.workspace.LangiumDocumentFactory.fromString(
    source,
    URI.parse(`file:///bench/main${ext}`)
  );
  const parseMs = performance.now() - parseStart;
  shared.workspace.LangiumDocuments.addDocument(doc);
  const phases = instrumentPhases(builder);
  const start = performance.now();
  await builder.build([doc], { validation: true });
  const end = performance.now();
  phases.dispose();
  const timings = phaseTimings(start, end, phases.times, parseMs);
  return { timings, errors: countErrors([doc]) };
}

// Workspace build: hand the whole document set to Langium's DocumentBuilder and
// time the full build (parse folded in, as for buildSingle). Reports full-build
// only.
async function buildWorkspace(createServices, ext, files) {
  const shared = freshShared(createServices);
  const builder = shared.workspace.DocumentBuilder;
  const parseStart = performance.now();
  const docs = files.map((text, i) =>
    shared.workspace.LangiumDocumentFactory.fromString(
      text,
      URI.parse(`file:///bench/ws/${i}${ext}`)
    )
  );
  const parseMs = performance.now() - parseStart;
  for (const doc of docs) {
    shared.workspace.LangiumDocuments.addDocument(doc);
  }
  const buildStart = performance.now();
  await builder.build(docs, { validation: true });
  const buildEnd = performance.now();
  return {
    timings: { 'full-build': parseMs + (buildEnd - buildStart) },
    errors: countErrors(docs),
  };
}

function printBench(name, bytes, timings, order) {
  // Matches PegiumBench's column layout so tools/compare_langium_bench.py can
  // parse both with one regex.
  process.stdout.write(`[bench] ${name} size=${bytes}B iterations=${ITERATIONS}\n`);
  for (const step of order) {
    const ms = timings[step] ?? 0;
    const mibPerSecond = ms > 0 ? bytes / (1024 * 1024) / (ms / 1000) : 0;
    process.stdout.write(
      `  ${step.padEnd(18)}${ms.toFixed(2).padEnd(10)}ms  ${mibPerSecond
        .toFixed(2)
        .padEnd(10)}MiB/s\n`
    );
  }
}

async function runBench(name, bytes, once, fullBuildOnly = false) {
  if (FILTER && !name.includes(FILTER)) {
    return;
  }
  // Progress trace on stderr (results go to stdout): a long benchmark would
  // otherwise look like the run had hung.
  process.stderr.write(`  running ${name} (${ITERATIONS} iter)\n`);
  for (let w = 0; w < WARMUP; w++) {
    await once();
  }
  const order = fullBuildOnly ? ['full-build'] : [...STEP_NAMES, 'full-build'];
  const totals = Object.fromEntries(order.map((s) => [s, 0]));
  let errors = 0;
  for (let i = 0; i < ITERATIONS; i++) {
    const result = await once();
    errors = result.errors;
    for (const step of order) {
      totals[step] += result.timings[step] ?? 0;
    }
  }
  for (const step of order) {
    totals[step] /= ITERATIONS;
  }
  if (errors > 0) {
    process.stderr.write(`[warn] ${name}: ${errors} error diagnostic(s) in input\n`);
  }
  printBench(name, bytes, totals, order);
}

async function main() {
  const singles = [
    ['arithmetics', createArithmeticsServices, '.calc', arithmeticsSource],
    ['domainmodel', createDomainModelServices, '.dmodel', domainmodelSource],
    ['requirements', createRequirementsAndTestsLangServices, '.req', requirementsSource],
    ['statemachine', createStatemachineServices, '.statemachine', statemachineSource],
  ];
  for (const [name, createServices, ext, makeSource] of singles) {
    if (FILTER && !name.includes(FILTER)) continue;
    const source = makeSource(TARGET_BYTES);
    const bytes = Buffer.byteLength(source, 'utf8');
    await runBench(name, bytes, () => buildSingle(createServices, ext, source));
  }

  const workspaceLanguages = [
    ['arithmetics', createArithmeticsServices, '.calc', arithmeticsWorkspaceFile],
    ['domainmodel', createDomainModelServices, '.dmodel', domainmodelWorkspaceFile],
    ['requirements', createRequirementsAndTestsLangServices, '.req', requirementsWorkspaceFile],
    ['statemachine', createStatemachineServices, '.statemachine', statemachineWorkspaceFile],
  ];
  for (const [name, createServices, ext, fileGen] of workspaceLanguages) {
    for (const [suffix, target] of [['small', WS_SMALL_BYTES], ['large', WS_LARGE_BYTES]]) {
      // Skip generating the workspaces the filter excludes, so an isolated
      // per-config run's peak RSS reflects only that workspace.
      if (FILTER && !`${name}-workspace-${suffix}`.includes(FILTER)) continue;
      const files = workspaceFiles(fileGen, target);
      const bytes = files.reduce((sum, t) => sum + Buffer.byteLength(t, 'utf8'), 0);
      await runBench(`${name}-workspace-${suffix} files=${files.length}`, bytes,
                     () => buildWorkspace(createServices, ext, files), true);
    }
  }
}

main().catch((error) => {
  process.stderr.write(`Langium bench failed: ${error?.stack ?? error}\n`);
  process.exit(1);
});