Skip to content

Commit

Permalink
WIP: mininal reporter
Browse files Browse the repository at this point in the history
  • Loading branch information
Krinkle committed Jan 13, 2025
1 parent d029107 commit e8baf23
Show file tree
Hide file tree
Showing 5 changed files with 303 additions and 88 deletions.
25 changes: 22 additions & 3 deletions src/qtap.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
'use strict';

import { EventEmitter } from 'node:events';
import util from 'node:util';

import kleur from 'kleur';
import browsers from './browsers.js';
import reporters from './reporters.js';
import { ControlServer } from './server.js';

/**
Expand Down Expand Up @@ -37,7 +39,9 @@ function makeLogger (defaultChannel, printDebug, verbose = false) {
: function debug (messageCode, ...params) {
printDebug(kleur.grey(`[${prefix}] ${kleur.bold(messageCode)} ${paramsFmt(params)}`));
},
warning: function warning (messageCode, ...params) {
warning: !verbose
? function () {}
: function warning (messageCode, ...params) {
printDebug(kleur.yellow(`[${prefix}] WARNING ${kleur.bold(messageCode)}`) + ` ${paramsFmt(params)}`);
}
};
Expand Down Expand Up @@ -89,10 +93,13 @@ async function run (browserNames, files, options = {}) {
options.printDebug || console.error,
options.verbose
);
const eventbus = new EventEmitter();

// reporters.minimal(eventbus, logger.channel('reporter_minimal'));

const servers = [];
for (const file of files) {
servers.push(new ControlServer(options.root, file, logger, {
servers.push(new ControlServer(options.root, file, eventbus, logger, {
idleTimeout: options.timeout,
connectTimeout: options.connectTimeout
}));
Expand Down Expand Up @@ -138,8 +145,20 @@ async function run (browserNames, files, options = {}) {
}
}

// const promise = new Promise((resolve, reject) => {
// // TODO: The promise must be created on-demand so that Node.js built-in protection
// // against unhandled error events remains useful. That is, the caller must either
// // handle the error event -or- request the promise, at which point the user continues
// // to be protected by Node.js 'unhandledRejection' detection, e.g. if they forget to
// // await or catch the promise.
// eventbus.on('error', reject);
// // TODO: finish event should include exitCode and/or data that makes it trivial to construct
// eventbus.on('finish', resolve);
// });
// return { eventbus, promise };

// TODO: Set exit status to 1 on failures, to ease programmatic use and testing.
// TODO: Return an event emitter for custom reportering via programmatic use.
// TODO: Return an event emitter for custom reportin via programmatic use.
return 0;
}

Expand Down
88 changes: 88 additions & 0 deletions src/reporters.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
function dynamicMinimalReporter (eventbus, logger) {
/**
* @typedef {Object} ClientState
* @property {string} clientId
* @property {string} displayName
* @property {string} status
* @property {null|string} lastline
* @property {Array} failures
*/

/** @type {Object<string,ClientState[]> */
const clientsByFile = Object.create(null);
/** @type {Object<string,ClientState> */
const clientsById = Object.create(null);

let screen = '';

function render () {
const icons = { waiting: '⢎', progress: '⠧', success: '✔', failure: '✘' };
let str = '';
for (const testFile in clientsByFile) {
str += `\nRunning ${testFile}\n`;
for (const client of clientsByFile[testFile]) {
str += `* ${client.displayName} ${icons[client.status]} ${client.lastline || ''}\n`;
}
}

if (screen) {
const oldHeight = screen.split('\n').length;
for (let i = 1; i < oldHeight; i++) {
process.stdout.write('\x1b[A\x1b[K');
}
}

process.stdout.write(str);
screen = str;
}

eventbus.on('clientcreate', (event) => {
const client = {
clientId: event.clientId,
displayName: event.displayName,
status: 'waiting',
lastline: null,
failures: []
};

clientsByFile[event.testFile] ??= [];
clientsByFile[event.testFile].push(client);
clientsById[event.clientId] = client;
render();
});

eventbus.on('clientonline', (event) => {
clientsById[event.clientId].status = 'progress';
render();
});

eventbus.on('clientsample', (event) => {
clientsById[event.clientId].lastline = event.line;
render();
});

eventbus.on('clientend', (event) => {
clientsById[event.clientId].status = 'failure';
clientsById[event.clientId].lastline = event.reason;
render();
});
}

function flatMinimalReporter (eventbus) {
eventbus.on('createclient', (client) => {
console.log(`Running ${client.testFile}`);
console.log(`* ${client.displayName}`);
});
}

const isTTY = process.stdout.isTTY && process.env.TERM !== 'dumb';
const minimal = isTTY ? dynamicMinimalReporter : flatMinimalReporter;

export default { minimal };

// TODO: Default: TAP where each browser is 1 virtual test in case of success.
// TODO: Verbose: TAP forwarded, test names prepended with [browsername].
// TODO: Failures are shown either way, with prepended names.
// TODO: On "runEnd", report wall-clock runtime
// Default: No-op, as overall TAP line as single test (above) can contain runtime
// Verbose: Output comment indicatinh browser done, and test runtime.
121 changes: 36 additions & 85 deletions src/server.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import path from 'node:path';
import { fnToStr, qtapClientHead, qtapClientBody } from './client.js';
import * as util from './util.js';
import tapFinished from './tap-finished.js';
/** @import events from 'node:events' */
/** @import { Logger } from './qtap.js' */

const QTAP_DEBUG = process.env.QTAP_DEBUG === '1';
Expand All @@ -19,12 +20,13 @@ class ControlServer {
/**
* @param {string|undefined} root
* @param {string} testFile File path or URL
* @param {events.EventEmitter} eventbus
* @param {Logger} logger
* @param {Object} options
* @param {number|undefined} options.idleTimeout
* @param {number|undefined} options.connectTimeout
*/
constructor (root, testFile, logger, options) {
constructor (root, testFile, eventbus, logger, options) {
if (!root) {
// For `qtap test/index.html`, default root to cwd.
root = process.cwd();
Expand All @@ -38,6 +40,7 @@ class ControlServer {

this.root = root;
this.testFile = testFile;
this.eventbus = eventbus;
this.logger = logger.channel('qtap_server_' + ControlServer.nextServerId++);
this.idleTimeout = options.idleTimeout || 30;
this.connectTimeout = options.connectTimeout || 60;
Expand Down Expand Up @@ -108,9 +111,7 @@ class ControlServer {
async launchBrowser (browserFn, browserName, globalSignal) {
const clientId = 'client_' + ControlServer.nextClientId++;
const logger = this.logger.channel(`qtap_browser_${clientId}_${browserName}`);

// TODO: Remove `summary` in favour of `eventbus`
const summary = { ok: true };
let clientIdleTimer = null;

const controller = new AbortController();
let signal = controller.signal;
Expand All @@ -122,13 +123,6 @@ class ControlServer {
});
}

// TODO: Write test for --connect-timeout by using a no-op browser.
const TIMEOUT_CONNECT = this.connectTimeout;
const TIMEOUT_IDLE = this.idleTimeout;
const TIMEOUT_CHECK_MS = 1000;
const launchStart = performance.now();
let clientIdleTimer = null;

// Reasons to stop a browser, whichever comes first:
// 1. tap-finished.
// 2. tap-parser 'bailout' event (client knows it crashed).
Expand All @@ -144,19 +138,19 @@ class ControlServer {
controller.abort(reason);
};

const tapParser = tapFinished({ wait: 0 }, () => {
logger.debug('browser_tap_finished', 'Test has finished, stopping browser');
const tapParser = tapFinished({ wait: 0 }, (result) => {
logger.debug('browser_tap_finished', 'Test has finished, stopping browser', result.ok);
stopBrowser('QTap: browser_tap_finished');
});

tapParser.on('bailout', (reason) => {
logger.warning('browser_tap_bailout', `Test ended unexpectedly, stopping browser. Reason: ${reason}`);
summary.ok = false;
stopBrowser('QTap: browser_tap_bailout');
this.eventbus.emit('clientend', { clientId, reason: 'browser_tap_bailout' });
});
tapParser.once('fail', () => {
logger.debug('browser_tap_fail', 'Results indicate at least one test has failed assertions');
summary.ok = false;
this.eventbus.emit('clientend', { clientId, reason: 'browser_tap_fail' });
});
// Debugging
// tapParser.on('assert', logger.debug.bind(logger, 'browser_tap_assert'));
Expand All @@ -176,32 +170,47 @@ class ControlServer {
// in `handleTap()` or `tapParser.on('line')`. But that adds significant overhead from
// Node.js/V8 natively allocating many timers when processing large batches of test results.
// Instead, merely store performance.now() and check that periodically.
clientIdleTimer = setTimeout(function qtapCheckTimeout () {
// TODO: Report timeout failure to reporter/TAP/CLI.
// TODO: Write test for --connect-timeout by using a no-op browser.
const TIMEOUT_CHECK_MS = 1000;
const launchStart = performance.now();
const qtapCheckTimeout = () => {
if (!browser.clientIdleActive) {
if ((performance.now() - launchStart) > (TIMEOUT_CONNECT * 1000)) {
logger.warning('browser_connect_timeout', `Browser did not start within ${TIMEOUT_CONNECT}s, stopping browser`);
summary.ok = false;
if ((performance.now() - launchStart) > (this.connectTimeout * 1000)) {
logger.warning('browser_connect_timeout', `Browser did not start within ${this.connectTimeout}s, stopping browser`);
stopBrowser('QTap: browser_connect_timeout');
this.eventbus.emit('clientend', { clientId, reason: 'browser_connect_timeout' });
return;
}
} else {
if ((performance.now() - browser.clientIdleActive) > (TIMEOUT_IDLE * 1000)) {
logger.warning('browser_idle_timeout', `Browser idle for ${TIMEOUT_IDLE}s, stopping browser`);
summary.ok = false;
if ((performance.now() - browser.clientIdleActive) > (this.idleTimeout * 1000)) {
logger.warning('browser_idle_timeout', `Browser idle for ${this.idleTimeout}s, stopping browser`);
stopBrowser('QTap: browser_idle_timeout');
this.eventbus.emit('clientend', { clientId, reason: 'browser_idle_timeout' });
return;
}
}
clientIdleTimer = setTimeout(qtapCheckTimeout, TIMEOUT_CHECK_MS);
}, TIMEOUT_CHECK_MS);
};
clientIdleTimer = setTimeout(qtapCheckTimeout, TIMEOUT_CHECK_MS);

const url = await this.getProxyBase() + '/?qtap_clientId=' + clientId;
const signals = { client: signal, global: globalSignal };

try {
logger.debug('browser_launch_call');
await browserFn(url, signals, logger);

// Separate calling browserFn() from awaiting so that we can emit 'createclient'
// right after calling it (which may set Browser.displayName). If we await here,
// the event would be emitted when the client is done instead of when it starts.
const browerPromise = browserFn(url, signals, logger);
this.eventbus.emit('clientcreate', {
testFile: this.testFile,
clientId,
browserName,
displayName: browser.getDisplayName()
});
await browerPromise;

logger.debug('browser_launch_ended');
} finally {
// TODO: Report error to TAP. Eg. "No executable found"
Expand Down Expand Up @@ -265,6 +274,7 @@ class ControlServer {
if (browser) {
browser.clientIdleActive = performance.now();
browser.logger.debug('browser_connected', `${browser.getDisplayName()} connected! Serving test file.`);
this.eventbus.emit('clientonline', { clientId });
} else {
this.logger.debug('respond_static_testfile', clientId);
}
Expand Down Expand Up @@ -309,72 +319,13 @@ class ControlServer {
);

browser.clientIdleActive = performance.now();
this.eventbus.emit('clientsample', { clientId, line: body.trimEnd().split('\n').pop() });
} else {
this.logger.debug('browser_tap_unhandled', clientId, JSON.stringify(body.slice(0, 30) + '…'));
}
});
resp.writeHead(204);
resp.end();

// TODO: Pipe to one of two options, based on --reporter:
// - [tap | default in piped and CI?]: tap-parser + some kind of renumbering or prefixing.
// client_1> ok 40 foo > bar
// out> ok 1 - qtap > Firefox (client_1) connected! Running test/index.html.
// out> ok 42 - foo > bar
// -->
// client_1> ok 40 foo > bar
// client_2> ok 40 foo > bar
// out> ok 1 - qtap > Firefox (client_1) connected! Running test/index.html.
// out> ok 2 - qtap > Chromium (client_2) connected! Running test/index.html.
// out> ok 81 - foo > bar [Firefox client_1]
// out> ok 82 - foo > bar [Chromium client_2]
// - [minimal|default in interactive mode]
// out> Testing /test/index.html
// out>
// out> Firefox : SPINNER [blue]
// out> Running test 40.
// out> [Chromium] : [grey] [star] [grey] Launching...
// out> [Safari] : [grey] [star] [grey] Launching...
// -->
// out> Testing /test/index.html
// out>
// out> [Firefox client_1]: ✔ [green] Completed 123 tests in 42ms.
// out> [Chromium client2]: [blue*spinner] Running test 40.
// out> [Safari client_3] [grey] [star] [grey] Launching...
// -->
// out> Testing /test/index.html
// out>
// out> not ok 40 foo > bar # Chromium client_2
// out> ---
// out> message: failed
// out> actual : false
// out> expected: true
// out> stack: |
// out> @/example.js:46:12
// out> ...
// out>
// out> [Firefox client_1]: ✔ [green] Completed 123 tests in 42ms.
// out> [Chromium client_2]: ✘ [red] 2 failures.
//
// If minimal is selected explicilty in piped/non-interactive/CI mode,
// then it will have no spinners, and also lines won't overwrite each other.
// Test counting will be disabled along with the spinner so instead we'll print:
// out> Firefox client_1: Launching...
// out> Firefox client_1: Running tests... [= instead of spinner/counter]
// out> Firefox client_1: Completed 123 tets in 42ms.

// "▓", "▒", "░" // noise, 100
// "㊂", "㊀", "㊁" // toggle10, 100
// await new Promise(r=>setTimeout(r,100)); process.stdout.write('\r' + frames[i % frames.length] + ' ');
// writable.isTTY
// !process.env.CI

// Default: TAP where each browser is 1 virtual test in case of success.
// Verbose: TAP forwarded, test names prepended with [browsername].
// Failures are shown either way, with prepended names.
// TODO: On "runEnd", report runtime
// Default: No-op, as overall TAP line as single test (above) can contain runtime
// Verbose: Output comment indicatinh browser done, and test runtime.
}

/**
Expand Down
2 changes: 2 additions & 0 deletions src/util.js
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ export const LocalBrowser = {

return new Promise((resolve, reject) => {
spawned.on('error', error => {
// TODO: Move absorbing of expected error to server.js
if (signals.client.aborted) {
resolve();
} else {
Expand All @@ -148,6 +149,7 @@ export const LocalBrowser = {
+ (sig ? `\n signal: ${sig}` : '')
+ (stderr ? `\n stderr:\n${indent(stderr)}` : '')
+ (stdout ? `\n stdout:\n${indent(stdout)}` : '');
// TODO: Move absorbing of expected error to server.js
if (!signals.client.aborted) {
reject(new Error(details));
} else {
Expand Down
Loading

0 comments on commit e8baf23

Please sign in to comment.